From 88744f1089f689bcebb6c9aaa4c9fcc9f1aee549 Mon Sep 17 00:00:00 2001 From: mlmz <54172054+minleminzui@users.noreply.github.com> Date: Wed, 11 Oct 2023 19:43:33 +0800 Subject: [PATCH] fix: arrange output parser classes (#59) * fix: arrange output parser classes Put all the output parser classes in one place. Remove redundant output parser classes. Reuse output parser classes with the same functionality. * fix: arrange output parser classes Put all the output parser classes in one place. Remove redundant output parser classes. Reuse output parser classes with the same functionality. * fix: output parser error * fix: output parser error --------- Co-authored-by: chenweize1998 --- agentverse/__init__.py | 5 +- agentverse/agents/base.py | 2 +- .../agents/simulation_agent/reflection.py | 2 +- agentverse/initialization.py | 2 +- agentverse/output_parser/__init__.py | 5 + agentverse/output_parser/output_parser.py | 621 ++++++++++++++++++ agentverse/parser.py | 25 - agentverse/tasks/__init__.py | 28 +- .../simulation/alice_home/output_parser.py | 29 - .../tasks/simulation/db_diag/output_parser.py | 40 -- .../math_problem_2players_tools/config.yaml | 10 +- .../output_parser.py | 32 - .../nlp_classroom_3players/output_parser.py | 31 - .../output_parser.py | 39 -- .../nlp_classroom_9players/output_parser.py | 36 - .../output_parser.py | 34 - .../tasks/simulation/pokemon/output_parser.py | 36 - .../prisoner_dilemma/output_parser.py | 74 --- .../sde_team_2players/output_parser.py | 19 - .../sde_team_3players/output_parser.py | 19 - .../brainstorming/output_parser.py | 77 --- .../tasksolving/commongen/output_parser.py | 93 --- .../tasksolving/humaneval/output_parser.py | 311 --------- .../tasksolving/logic_grid/output_parser.py | 13 - .../tasks/tasksolving/mgsm/output_parser.py | 166 ----- .../pythoncalculator/output_parser.py | 19 - .../tasksolving/responsegen/output_parser.py | 104 --- .../tool_using/24point/config.yaml | 2 +- .../tasksolving/tool_using/bmi/config.yaml | 2 +- .../tool_using/bookclub/config.yaml | 2 +- .../tasksolving/tool_using/car/config.yaml | 2 +- .../tasksolving/tool_using/date/config.yaml | 2 +- .../tasksolving/tool_using/diy/config.yaml | 2 +- .../tasksolving/tool_using/output_parser.py | 77 --- .../tasksolving/tool_using/party/config.yaml | 2 +- .../tasksolving/tool_using/sudoku/config.yaml | 2 +- .../tool_using/trending/config.yaml | 2 +- .../tool_using/vacation/config.yaml | 2 +- 38 files changed, 646 insertions(+), 1323 deletions(-) create mode 100644 agentverse/output_parser/__init__.py create mode 100644 agentverse/output_parser/output_parser.py delete mode 100644 agentverse/parser.py delete mode 100644 agentverse/tasks/simulation/alice_home/output_parser.py delete mode 100644 agentverse/tasks/simulation/db_diag/output_parser.py delete mode 100644 agentverse/tasks/simulation/math_problem_2players_tools/output_parser.py delete mode 100644 agentverse/tasks/simulation/nlp_classroom_3players/output_parser.py delete mode 100644 agentverse/tasks/simulation/nlp_classroom_3players_withtool/output_parser.py delete mode 100644 agentverse/tasks/simulation/nlp_classroom_9players/output_parser.py delete mode 100644 agentverse/tasks/simulation/nlp_classroom_9players_group/output_parser.py delete mode 100644 agentverse/tasks/simulation/pokemon/output_parser.py delete mode 100644 agentverse/tasks/simulation/prisoner_dilemma/output_parser.py delete mode 100644 agentverse/tasks/simulation/sde_team/sde_team_2players/output_parser.py delete mode 100644 agentverse/tasks/simulation/sde_team/sde_team_3players/output_parser.py delete mode 100644 agentverse/tasks/tasksolving/brainstorming/output_parser.py delete mode 100644 agentverse/tasks/tasksolving/commongen/output_parser.py delete mode 100644 agentverse/tasks/tasksolving/humaneval/output_parser.py delete mode 100644 agentverse/tasks/tasksolving/logic_grid/output_parser.py delete mode 100644 agentverse/tasks/tasksolving/mgsm/output_parser.py delete mode 100644 agentverse/tasks/tasksolving/pythoncalculator/output_parser.py delete mode 100644 agentverse/tasks/tasksolving/responsegen/output_parser.py delete mode 100644 agentverse/tasks/tasksolving/tool_using/output_parser.py diff --git a/agentverse/__init__.py b/agentverse/__init__.py index af88fdd9e..4512a99dd 100644 --- a/agentverse/__init__.py +++ b/agentverse/__init__.py @@ -1,7 +1,4 @@ -from .tasks import * - - -# from .agents import Agent +from .output_parser import output_parser_registry from .environments import env_registry from .environments.simulation_env.rules.order import order_registry from .environments.simulation_env.rules.describer import describer_registry diff --git a/agentverse/agents/base.py b/agentverse/agents/base.py index 084bc5135..4f118e08c 100644 --- a/agentverse/agents/base.py +++ b/agentverse/agents/base.py @@ -8,7 +8,7 @@ from agentverse.llms import BaseLLM from agentverse.memory import BaseMemory, ChatHistoryMemory from agentverse.message import Message -from agentverse.parser import OutputParser +from agentverse.output_parser import OutputParser from agentverse.memory_manipulator import BaseMemoryManipulator diff --git a/agentverse/agents/simulation_agent/reflection.py b/agentverse/agents/simulation_agent/reflection.py index e9303032b..bbbcf9f10 100644 --- a/agentverse/agents/simulation_agent/reflection.py +++ b/agentverse/agents/simulation_agent/reflection.py @@ -14,7 +14,7 @@ from agentverse.llms import BaseLLM from agentverse.memory import BaseMemory, ChatHistoryMemory from agentverse.message import Message -from agentverse.parser import OutputParser +from agentverse.output_parser import OutputParser from agentverse.message import Message from agentverse.agents.base import BaseAgent diff --git a/agentverse/initialization.py b/agentverse/initialization.py index ed21733e0..13ef54e77 100644 --- a/agentverse/initialization.py +++ b/agentverse/initialization.py @@ -19,7 +19,7 @@ from agentverse.memory import memory_registry from agentverse.memory_manipulator import memory_manipulator_registry -from agentverse.parser import output_parser_registry +from agentverse.output_parser import output_parser_registry if TYPE_CHECKING: from agentverse.agents import BaseAgent diff --git a/agentverse/output_parser/__init__.py b/agentverse/output_parser/__init__.py new file mode 100644 index 000000000..8b54c8edd --- /dev/null +++ b/agentverse/output_parser/__init__.py @@ -0,0 +1,5 @@ +from agentverse.registry import Registry + +output_parser_registry = Registry(name="OutputParserRegistry") + +from .output_parser import * \ No newline at end of file diff --git a/agentverse/output_parser/output_parser.py b/agentverse/output_parser/output_parser.py new file mode 100644 index 000000000..556d9ff6e --- /dev/null +++ b/agentverse/output_parser/output_parser.py @@ -0,0 +1,621 @@ +from __future__ import annotations + +import re +from abc import abstractmethod +import json +from typing import Union, List, Tuple, NamedTuple, TYPE_CHECKING + +from . import output_parser_registry + +from agentverse.utils import AgentAction, AgentFinish, AgentCriticism + +from agentverse.llms import LLMResult +from agentverse.logging import logger + +from pydantic import BaseModel + +if TYPE_CHECKING: + from agentverse.agents.base import BaseAgent + from agentverse.environments.base import BaseEnvironment + +class OutputParserError(Exception): + """Exception raised when parsing output from a command fails.""" + + def __init__(self, message): + self.message = message + + def __str__(self): + return "Failed to parse output of the model:%s\n " % self.message + + +class OutputParser(BaseModel): + """Base class for output parsers.""" + + @abstractmethod + def parse(self, output: LLMResult) -> NamedTuple: + pass + + +@output_parser_registry.register("alice_home") +class AliceHomeParser(OutputParser): + def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: + text = output.content + cleaned_output = text.strip() + cleaned_output = re.sub(r"\n+", "\n", cleaned_output) + cleaned_output = cleaned_output.split("\n") + if not ( + len(cleaned_output) == 2 + and cleaned_output[0].startswith("Thought:") + and cleaned_output[1].startswith("Action:") + ): + raise OutputParserError(text) + + action = cleaned_output[1][len("Action:") :].strip() + + return AgentFinish({"output": action}, text) + + +@output_parser_registry.register("db_diag") +@output_parser_registry.register("nlp_classroom_3players_withtool") +class CommonParser1(OutputParser): + def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: + text = output.content + cleaned_output = text.strip() + cleaned_output = re.sub(r"\n+", "\n", cleaned_output) + cleaned_output = cleaned_output.split("\n") + if not ( + len(cleaned_output) == 3 + and cleaned_output[0].startswith("Thought:") + and cleaned_output[1].startswith("Action:") + and cleaned_output[2].startswith("Action Input:") + ): + raise OutputParserError(text) + action = cleaned_output[1][len("Action:") :].strip() + action_input = cleaned_output[2][len("Action Input:") :].strip() + if action in ["Speak"]: + return AgentFinish({"output": action_input}, text) + elif action == "CallOn": + return AgentFinish({"output": "[CallOn] " + action_input}, text) + elif action == "RaiseHand": + return AgentFinish({"output": "[RaiseHand] " + action_input}, text) + elif action == "Listen": + return AgentFinish({"output": ""}, text) + else: + return AgentAction(action.lower(), action_input, text) + + +@output_parser_registry.register("math_problem_2players_tools") +class MathProblem2PlayersToolsParser(OutputParser): + def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: + text = output.content + cleaned_output = text.strip() + cleaned_output = re.sub(r"\n+", "\n", cleaned_output) + cleaned_output = cleaned_output.split("\n") + if not ( + len(cleaned_output) == 2 + and cleaned_output[0].startswith("Action:") + and cleaned_output[1].startswith("Action Input:") + ): + raise OutputParserError(text) + action = cleaned_output[0][len("Action:") :].strip() + action_input = cleaned_output[1][len("Action Input:") :].strip() + if action == "Speak": + return AgentFinish({"output": action_input}, text) + else: + return AgentAction(action, action_input, text) + + +@output_parser_registry.register("nlp_classroom_3players") +class NlpClassroom3PlayersParser(OutputParser): + def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: + text = output.content + cleaned_output = text.strip() + cleaned_output = re.sub(r"\n+", "\n", cleaned_output) + cleaned_output = cleaned_output.split("\n") + if not ( + len(cleaned_output) == 2 + and cleaned_output[0].startswith("Action:") + and cleaned_output[1].startswith("Action Input:") + ): + raise OutputParserError(text) + action = cleaned_output[0][len("Action:") :].strip() + action_input = cleaned_output[1][len("Action Input:") :].strip() + if action == "Speak": + return AgentFinish({"output": action_input}, text) + else: + raise OutputParserError(text) + + +@output_parser_registry.register("nlp_classroom_9players") +class NlpClassroom9PlayersParser(OutputParser): + def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: + text = output.content + cleaned_output = text.strip() + cleaned_output = re.sub(r"\n+", "\n", cleaned_output) + cleaned_output = cleaned_output.split("\n") + if not ( + len(cleaned_output) == 2 + and cleaned_output[0].startswith("Action:") + and cleaned_output[1].startswith("Action Input:") + ): + raise OutputParserError(text) + action = cleaned_output[0][len("Action:") :].strip() + action_input = cleaned_output[1][len("Action Input:") :].strip() + if action == "Speak": + return AgentFinish({"output": action_input}, text) + elif action == "CallOn": + return AgentFinish({"output": "[CallOn] " + action_input}, text) + elif action == "RaiseHand": + return AgentFinish({"output": "[RaiseHand] " + action_input}, text) + elif action == "Listen": + return AgentFinish({"output": ""}, text) + else: + return AgentAction(action, action_input, text) + + +@output_parser_registry.register("nlp_classroom_9players_group") +class NlpClassroom9PlayersGroupParser(OutputParser): + def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: + text = output.content + cleaned_output = text.strip() + cleaned_output = re.sub(r"\n+", "\n", cleaned_output) + cleaned_output = cleaned_output.split("\n") + if not ( + len(cleaned_output) == 2 + and cleaned_output[0].startswith("Action:") + and cleaned_output[1].startswith("Action Input:") + ): + raise OutputParserError(text) + action = cleaned_output[0][len("Action:") :].strip() + action_input = cleaned_output[1][len("Action Input:") :].strip() + if action == "Speak": + return AgentFinish({"output": action_input}, text) + elif action in ["CallOn", "RaiseHand", "GroupDiscuss"]: + return AgentFinish({"output": f"[{action}] {action_input}"}, text) + elif action == "Listen": + return AgentFinish({"output": ""}, text) + else: + return AgentAction(action, action_input, text) + + +@output_parser_registry.register("pokemon") +class PokemonParser(OutputParser): + def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: + text = output.content + cleaned_output = text.strip() + cleaned_output = re.sub(r"\n+", "\n", cleaned_output) + cleaned_output = cleaned_output.split("\n") + if not ( + len(cleaned_output) == 3 + and cleaned_output[0].startswith("Thought:") + and cleaned_output[1].startswith("Action:") + and cleaned_output[2].startswith("Action Input:") + ): + raise OutputParserError(text) + action = cleaned_output[1][len("Action:") :].strip() + action_input = cleaned_output[2][len("Action Input:") :].strip() + try: + action_input = json.loads(action_input) + except json.JSONDecodeError: + raise OutputParserError(text) + action_input["action"] = action + return AgentFinish({"output": json.dumps(action_input)}, text) + + +@output_parser_registry.register("prisoner_dilemma") +class PrisonerDilemmaParser(OutputParser): + # make sure 1 1 2 2 3 3 + cur_round: int = 1 + encounter_cur_round: bool = False + + def parse( + self, agent: "BaseAgent", environment: "BaseEnvironment", output: LLMResult + ) -> Union[AgentAction, AgentFinish]: + text = output.content + cleaned_output = text.strip() + cleaned_output = re.sub(r"\n+", "\n", cleaned_output) + cleaned_output = cleaned_output.split("\n") + if not ( + len(cleaned_output) == 2 + and cleaned_output[0].startswith("Action:") + and cleaned_output[1].startswith("Action Input:") + ): + raise OutputParserError(text) + action = cleaned_output[0][len("Action:") :].strip() + action_input = cleaned_output[1][len("Action Input:") :].strip() + + if action == "Speak": + # make sure the police count the round right + # if agent.name == "Police": + # action_input = re.sub(r'Round (\d+)', f'Round {self.cur_round}', action_input) + # self.cur_round += 1 + # if self.encounter_cur_round: + # self.encounter_cur_round = False + # self.cur_round += 1 + # else: + # self.encounter_cur_round = True + + # each time police speak is a new round + if agent.name == "Police": + if environment.cnt_turn == (environment.max_turns - 4): + action_input = ( + "Attention! You are now required to made your final decision and I will made the " + "final judgement to both of you based on this time, Please Answer now !" + ) + + elif environment.cnt_turn == (environment.max_turns - 2): + action_input = "Attention! Suspect2, it's now your time to make your final decision, Please Answer now !" + + # elif self.cur_round == 1: + # action_input = "Hey Listen! You are both arrested, and I am going to give you both a chance to walk out of here," \ + # "But you should comply with the following rules:" \ + # "- If one of you are willing to testifies against the other and the other one remains silent, then the one who testifies will be released IMMEDIATELY, while the silent one will be sentenced to TEN years in prison." \ + # "- If both of you remain silent, you will each receive a sentence of ONE year in prison." \ + # "- It seems that always testifying is a goog strategy, So! if you both choose to testify against each other, you will each receive a sentence of FIVE years in prison." \ + # "Now, it's your time to consider testifying or remaining silent. Remember this is a best chance you might ever have to walk out of here without guilty." \ + # "I will noticed both of you WHEN you have to make your final decision! Before that, try to make your best!" \ + + self.cur_round += 1 + + return AgentFinish({"output": action_input}, text) + else: + raise OutputParserError(text) + + +@output_parser_registry.register("sde_team/sde_team_2players") +@output_parser_registry.register("sde_team/sde_team_3players") +@output_parser_registry.register("commongen") +@output_parser_registry.register("humaneval-manager") +@output_parser_registry.register("mgsm") +@output_parser_registry.register("dummy") +@output_parser_registry.register("responsegen") +class CommonParser2(OutputParser): + # def parse(self, agent, env, output: LLMResult) -> Union[AgentAction, AgentFinish]: + def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: + return AgentFinish({"output": output.content}, output.content) + + +@output_parser_registry.register("role_assigner") +class RoleAssignerParser(OutputParser): + cnt_critic_agents: int = 0 + + def parse(self, output: LLMResult) -> List[str]: + text = output.content + pattern = re.compile(r"\d\.\s*(.+)") + roles = pattern.findall(text) + if len(roles) < self.cnt_critic_agents: + logger.error( + f"Role assigner failed to assign roles to {self.cnt_critic_agents} critics!" + ) + raise OutputParserError(text) + return roles + + +@output_parser_registry.register("evaluator") +class EvaluatorParser(OutputParser): + dimensions: List[str] = None + + def parse(self, output: LLMResult) -> Tuple[List[int], str]: + text = output.content + cleaned_output = re.sub(r"\n+", "\n", text.strip()) + checks = cleaned_output.split("\n") + patterns = [ + re.compile(r"(?:\d\.\s*)?" + dimension + r":\s*(\d)") + for dimension in self.dimensions + ] + try: + # find score and advice + score = [ + int(pattern.findall(checks[i])[0]) for i, pattern in enumerate(patterns) + ] + advice_text = "".join(checks[len(self.dimensions) :]) + advice = re.findall(r"(?:\d\.\s*)?Advice:\s*(.+)", advice_text)[0] + # logger.info("Evaluator give the following advice:\n" + advice) + except (IndexError, ValueError): + # logger.error("Bad response from evaluator!") + raise OutputParserError(text) + return score, advice + + +@output_parser_registry.register("humaneval-solver") +class HumanevalSolverParser(OutputParser): + def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: + text = output.content + # start_pos = text.find("```") + # end_pos = text.rfind("```") + # if end_pos == -1: + # raise OutputParserError(text) + # text = text[start_pos:end_pos] + # cleaned_output = text.strip().strip("```").strip() + # if cleaned_output.startswith("python"): + # cleaned_output = cleaned_output[6:].strip() + # elif cleaned_output.startswith("python3"): + # cleaned_output = cleaned_output[7:].strip() + code = re.findall(r"```.*?\n(.+?)```", text, re.DOTALL)[-1] + + return AgentFinish({"output": code}, text) + + +@output_parser_registry.register("humaneval-executor") +class HumanevalSolverParser(OutputParser): + def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: + text = output.content + try: + parsed_result = re.findall( + r"Thought:(.+?)Reasoning:(.+?)Criticism:(.+?)File Path:(.+?)Code:(.+?)Command:(.+)", + text, + re.DOTALL, + )[0] + cleaned_output = { + "thought": parsed_result[0].strip(), + "reasoning": parsed_result[1].strip(), + "criticism": parsed_result[2].strip(), + "file_path": parsed_result[3].strip().strip("`"), + "code": parsed_result[4] + .strip() + .strip("```") + .strip("python") + .strip("python3"), + "command": parsed_result[5].strip().strip("`"), + } + except BaseException as e: + raise OutputParserError(text) + + return AgentFinish({"output": cleaned_output}, text) + + +@output_parser_registry.register("humaneval-evaluator") +class HumanevalEvaluatorParser(OutputParser): + dimensions: List[str] = None + + def parse(self, output: LLMResult) -> Tuple[List[int], str]: + text = output.content + cleaned_output = re.sub(r"\n+", "\n", text.strip()) + checks = cleaned_output.split("\n") + + patterns = [ + re.compile(r"(?:\d.\s*)?" + dimension + r":\s*(\d)") + for dimension in self.dimensions + ] + + advice = "" + for check in reversed(checks): + advice = check + advice + if check.startswith("Advice:"): + break + checks[-1] = advice + try: + # find score and advice + score = [] + for pattern in patterns: + for check in checks[:-1]: + if pattern.findall(check): + score.append(bool(int(pattern.findall(check)[0]))) + break + advice = re.findall(r"(?:\d.\s*)?Advice:\s*(.+)", checks[-1])[0] + # logger.info("Evaluator give the following advice:\n" + advice) + except (IndexError, ValueError): + # logger.error("Bad response from evaluator!") + raise OutputParserError(text) + return score[0], advice + + +@output_parser_registry.register("humaneval-critic-agree") +class HumanevalyCriticParser(OutputParser): + def parse(self, output: LLMResult) -> AgentCriticism: + text = output.content + if "[Agree]" in text: + return AgentCriticism(True, "") + else: + return AgentCriticism(False, text) + + +@output_parser_registry.register("mgsm-evaluator") +class MGSMEvaluatorParser(OutputParser): + dimensions: List[str] = None + + def parse(self, output: LLMResult) -> Tuple[List[int], str]: + text = output.content + cleaned_output = re.sub(r"\n+", "\n", text.strip()) + # checks = cleaned_output.split("\n") + + patterns = [ + re.compile(r"(?:\d.\s*)?" + dimension + r":\s*(\d)") + for dimension in self.dimensions + ] + try: + # find score and advice + score_num = [ + int(pattern.findall(cleaned_output)[0]) + for i, pattern in enumerate(patterns) + ][0] + if score_num == 0: + score = False + elif score_num == 1: + score = True + else: + raise ValueError("Bad score!") + pat = re.compile(r"(?:\d.\s*)?Response:\s*(.+)", re.DOTALL) + advice = pat.findall(cleaned_output)[0] + # logger.info("Evaluator give the following advice:\n" + advice) + except (IndexError, ValueError): + # logger.error("Bad response from evaluator!") + raise OutputParserError(text) + return score, advice + + +@output_parser_registry.register("mgsm-critic-agree") +class MGSMCriticAgreeParser(OutputParser): + def parse(self, output: LLMResult) -> AgentCriticism: + text = output.content + text = re.sub(r"\n+", "\n", text.strip()) + # checks = text.split("\n") + # if not text.startswith("Thought:"): + # raise OutputParserError(text) + # if not (checks[0].startswith("Action:")): + # raise OutputParserError(text) + # if checks[0].strip(". ") == "Action: Agree": + # return AgentCriticism(True, "") + if "[Agree]" in text: + return AgentCriticism(True, "") + else: + # pattern = re.compile(r"Action Input: ([\S\n ]+)") + # try: + # criticism = pattern.findall(text)[0].strip() + # criticism = ( + # re.findall(r"Output:\S?(.+)", text)[0].replace("[Wrong]", "") + # ).strip() + criticism = text.replace("[Disagree]", "").strip() + # except IndexError: + # logger.error("Bad response from critic!") + # raise OutputParserError(text) + # criticism = "I think the solution is not correct. Please think carefully and correct it." + return AgentCriticism(False, criticism) + # else: + # raise OutputParserError(text) + + +@output_parser_registry.register("responsegen-evaluator") +class ResponseGenEvaluatorParser(OutputParser): + dimensions: List[str] = None + + def parse(self, output: LLMResult) -> Tuple[List[int], str]: + text = output.content + cleaned_output = re.sub(r"\n+", "\n", text.strip()) + checks = cleaned_output.split("\n") + + patterns = [ + re.compile(r"(?:\d.\s*)?" + dimension + r":\s*(\d+)") + for dimension in self.dimensions + ] + + advice = "" + for check in reversed(checks): + advice = check + advice + if check.startswith("Advice:"): + break + checks[-1] = advice + try: + # find score and advice + score = [ + int(pattern.findall(checks[i])[0]) for i, pattern in enumerate(patterns) + ] + advice = re.findall(r"(?:\d.\s*)?Advice:\s*(.+)", checks[-1])[0] + # logger.info("Evaluator give the following advice:\n" + advice) + except (IndexError, ValueError): + # logger.error("Bad response from evaluator!") + raise OutputParserError(text) + return score, advice + + +@output_parser_registry.register("responsegen-critic") +@output_parser_registry.register("critic") +class CommonParser3(OutputParser): + def parse(self, output: LLMResult) -> AgentCriticism: + text = output.content + text = re.sub(r"\n+", "\n", text.strip()) + checks = text.split("\n") + if not (checks[0].startswith("Action:")): + raise OutputParserError(text) + if checks[0].strip(". ") == "Action: Agree": + return AgentCriticism(True, "") + elif checks[0].strip(". ") == "Action: Disagree": + pattern = re.compile(r"Action Input: ([\S\n ]+)") + try: + criticism = pattern.findall(text)[0].strip() + except IndexError: + criticism = ( + "I think it is not correct. Please think carefully and improve it." + ) + # raise OutputParserError(text) + return AgentCriticism(False, criticism) + else: + raise OutputParserError(text) + + +@output_parser_registry.register("responsegen-critic-2") +class ResponseGenCriticParser(OutputParser): + def parse(self, output: LLMResult) -> AgentCriticism: + text = output.content + # text = re.sub(r"\n+", "\n", text.strip()) + # checks = text.split("\n") + # if not (checks[0].startswith("Action:")): + # raise OutputParserError(text) + # if checks[0].strip(". ") == "Action: Agree": + # return AgentCriticism(True, "") + # elif checks[0].strip(". ") == "Action: Disagree": + # pattern = re.compile(r"Action Input: ([\S\n ]+)") + # try: + # criticism = pattern.findall(text)[0].strip() + # except IndexError: + # # criticism = "I think the solution is not correct. Please think carefully and correct it." + # raise OutputParserError(text) + # return AgentCriticism(False, criticism) + # else: + # raise OutputParserError(text) + result = re.findall(r"Decision:(.+?)Response:(.+)", text, re.DOTALL) + if len(result) == 0: + result = ["Disagree", "I think the response can be further improved."] + else: + result = result[0] + if "Agree" in result[0]: + return AgentCriticism(True, "") + else: + return AgentCriticism(False, result[1].strip()) + + +@output_parser_registry.register("role-description-name-assigner") +class RoleAssignerParser(OutputParser): + cnt_critic_agents: int = 0 + + def parse(self, output: LLMResult) -> List[str]: + text = output.content + pattern = re.compile(r"\d+?\.\s*(.+?) - (.+)") + roles = pattern.findall(text) + if len(roles) < self.cnt_critic_agents: + logger.error( + f"Role assigner failed to assign roles to {self.cnt_critic_agents} critics!" + ) + raise OutputParserError(text) + res = [] + for role in roles: + res.append({"name": role[0], "description": role[1]}) + return res + + +@output_parser_registry.register("tool-using-solver") +class SolverParser(OutputParser): + def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: + text = output.content + pattern = re.compile(r"\d+?\.\s*(.+?) - (.+)") + tasks = pattern.findall(text) + if len(tasks) == 0: + raise OutputParserError(text) + return AgentFinish({"output": tasks}, text) + + +@output_parser_registry.register("tool-using-executor") +class ToolUsingSolverParser(OutputParser): + def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: + if output.function_name != "": + return AgentAction( + tool=output.function_name, + tool_input=output.function_arguments, + log=output.content, + ) + else: + return AgentFinish({"output": output.content}, output.content) + + +@output_parser_registry.register("tool-using-evaluator") +class HumanevalEvaluatorParser(OutputParser): + def parse(self, output: LLMResult) -> Tuple[List[int], str]: + text = output.content + try: + result = re.findall(r"Status:(.+?)Speak:(.+)", text, re.DOTALL)[0] + score = bool(int(result[0])) + words = result[1].strip() + except (IndexError, ValueError): + # logger.error("Bad response from evaluator!") + raise OutputParserError(text) + return score, words diff --git a/agentverse/parser.py b/agentverse/parser.py deleted file mode 100644 index abe7ae14e..000000000 --- a/agentverse/parser.py +++ /dev/null @@ -1,25 +0,0 @@ -from agentverse.registry import Registry -from typing import NamedTuple -from abc import abstractmethod -from agentverse.llms.base import LLMResult -from pydantic import BaseModel - -output_parser_registry = Registry(name="OutputParserRegistry") - - -class OutputParserError(Exception): - """Exception raised when parsing output from a command fails.""" - - def __init__(self, message): - self.message = message - - def __str__(self): - return "Failed to parse output of the model:%s\n " % self.message - - -class OutputParser(BaseModel): - """Base class for output parsers.""" - - @abstractmethod - def parse(self, output: LLMResult) -> NamedTuple: - pass diff --git a/agentverse/tasks/__init__.py b/agentverse/tasks/__init__.py index 77f33815d..426bc0f3a 100644 --- a/agentverse/tasks/__init__.py +++ b/agentverse/tasks/__init__.py @@ -1,30 +1,4 @@ import os import yaml -from .simulation.math_problem_2players_tools.output_parser import ( - MathProblem2PlayersToolsParser, -) -from .simulation.nlp_classroom_3players.output_parser import NlpClassroom3PlayersParser -from .simulation.nlp_classroom_9players.output_parser import NlpClassroom9PlayersParser -from .simulation.nlp_classroom_3players_withtool.output_parser import ( - NlpClassroom3PlayersWithtoolParser, -) -from .simulation.nlp_classroom_9players_group.output_parser import ( - NlpClassroom9PlayersGroupParser, -) -from .simulation.db_diag.output_parser import DBDiag - -from .simulation.prisoner_dilemma.output_parser import PrisonerDilemmaParser - -from .simulation.pokemon.output_parser import PokemonParser -from .simulation.sde_team.sde_team_3players.output_parser import SdeTeamParser -from .simulation.sde_team.sde_team_2players.output_parser import SdeTeamGivenTestsParser - -from .tasksolving.pythoncalculator.output_parser import PipelinePythoncalculatorParser -from .tasksolving.brainstorming.output_parser import * -from .tasksolving.humaneval.output_parser import * -from .tasksolving.tool_using.output_parser import * -from .tasksolving.mgsm.output_parser import * -from .tasksolving.responsegen.output_parser import * -from .tasksolving.logic_grid.output_parser import * -from .tasksolving.commongen.output_parser import * +from agentverse.output_parser import * diff --git a/agentverse/tasks/simulation/alice_home/output_parser.py b/agentverse/tasks/simulation/alice_home/output_parser.py deleted file mode 100644 index 32d6a5faa..000000000 --- a/agentverse/tasks/simulation/alice_home/output_parser.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union - -from agentverse.parser import OutputParser, LLMResult - -from agentverse.utils import AgentAction, AgentFinish - -from agentverse.parser import OutputParserError, output_parser_registry - - -@output_parser_registry.register("alice_home") -class AliceHomeParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - cleaned_output = text.strip() - cleaned_output = re.sub(r"\n+", "\n", cleaned_output) - cleaned_output = cleaned_output.split("\n") - if not ( - len(cleaned_output) == 2 - and cleaned_output[0].startswith("Thought:") - and cleaned_output[1].startswith("Action:") - ): - raise OutputParserError(text) - - action = cleaned_output[1][len("Action:"):].strip() - - return AgentFinish({"output": action}, text) diff --git a/agentverse/tasks/simulation/db_diag/output_parser.py b/agentverse/tasks/simulation/db_diag/output_parser.py deleted file mode 100644 index 9fad13f78..000000000 --- a/agentverse/tasks/simulation/db_diag/output_parser.py +++ /dev/null @@ -1,40 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union - - -# from langchain.schema import AgentAction, AgentFinish -from agentverse.utils import AgentAction, AgentFinish - -from agentverse.parser import OutputParserError, output_parser_registry -from agentverse.parser import OutputParser -from agentverse.llms.base import LLMResult - - -@output_parser_registry.register("db_diag") -class DBDiag(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - cleaned_output = text.strip() - cleaned_output = re.sub(r"\n+", "\n", cleaned_output) - cleaned_output = cleaned_output.split("\n") - if not ( - len(cleaned_output) == 3 - and cleaned_output[0].startswith("Thought:") - and cleaned_output[1].startswith("Action:") - and cleaned_output[2].startswith("Action Input:") - ): - raise OutputParserError(text) - action = cleaned_output[1][len("Action:") :].strip() - action_input = cleaned_output[2][len("Action Input:") :].strip() - if action in ["Speak"]: - return AgentFinish({"output": action_input}, text) - elif action == "CallOn": - return AgentFinish({"output": "[CallOn] " + action_input}, text) - elif action == "RaiseHand": - return AgentFinish({"output": "[RaiseHand] " + action_input}, text) - elif action == "Listen": - return AgentFinish({"output": ""}, text) - else: - return AgentAction(action.lower(), action_input, text) diff --git a/agentverse/tasks/simulation/math_problem_2players_tools/config.yaml b/agentverse/tasks/simulation/math_problem_2players_tools/config.yaml index 1599d8356..6e154b9d2 100644 --- a/agentverse/tasks/simulation/math_problem_2players_tools/config.yaml +++ b/agentverse/tasks/simulation/math_problem_2players_tools/config.yaml @@ -12,12 +12,12 @@ prompts: When responding, please use the following two-line format: [Option 1]: When you need to use a tool, output in the following format (omit the "[]" bracket when responding) - ACTION: (a tool name, it can be one of [${tool_names}]) - ACTION INPUT: (input arguments for the tool) + Action: (a tool name, it can be one of [${tool_names}]) + Action Input: (input arguments for the tool) [Option 2]: When you want to speak, you can use the following format: - ACTION: Speak - ACTION INPUT: (what you want to say in a single line) + Action: Speak + Action Input: (what you want to say in a single line) Here is the conversation history ${chat_history} @@ -25,7 +25,7 @@ prompts: Here is the observations from tool execution: ${tool_observation} - Now the game starts! ${role_description} You should give your action based on the above history. Remember, you should ALWAYS give your response STRICTLY in the above response format with the TWO lines start with "ACTION:" and "ACTION INPUT:" respectively! + Now the game starts! ${role_description} You should give your action based on the above history. Remember, you should ALWAYS give your response STRICTLY in the above response format with the TWO lines start with "Action:" and "Action Input:" respectively! summary_prompt: &summary_prompt | Progressively summarize the lines of a record that you uses tools, which contains inputs for certain tools and the results returned by these tools. Based on the current summary, you need to summarize from the record the goals that the you intended to solve with each call to the tool, add it onto the previous summary, and eventually return a new summary. diff --git a/agentverse/tasks/simulation/math_problem_2players_tools/output_parser.py b/agentverse/tasks/simulation/math_problem_2players_tools/output_parser.py deleted file mode 100644 index 24729b307..000000000 --- a/agentverse/tasks/simulation/math_problem_2players_tools/output_parser.py +++ /dev/null @@ -1,32 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms.base import LLMResult -from agentverse.utils import AgentAction, AgentFinish - - -@output_parser_registry.register("math_problem_2players_tools") -class MathProblem2PlayersToolsParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - cleaned_output = text.strip() - cleaned_output = re.sub(r"\n+", "\n", cleaned_output) - cleaned_output = cleaned_output.split("\n") - if not ( - len(cleaned_output) == 2 - and - # cleaned_output[0].startswith("THOUGHT:") and - cleaned_output[0].startswith("ACTION:") - and cleaned_output[1].startswith("ACTION INPUT:") - ): - print(text) - raise OutputParserError("Output Format Error") - action = cleaned_output[0][len("ACTION:") :].strip() - action_input = cleaned_output[1][len("ACTION INPUT:") :].strip() - if action == "Speak": - return AgentFinish({"output": action_input}, text) - else: - return AgentAction(action, action_input, text) diff --git a/agentverse/tasks/simulation/nlp_classroom_3players/output_parser.py b/agentverse/tasks/simulation/nlp_classroom_3players/output_parser.py deleted file mode 100644 index 4c4563bb4..000000000 --- a/agentverse/tasks/simulation/nlp_classroom_3players/output_parser.py +++ /dev/null @@ -1,31 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union - -from agentverse.parser import OutputParser, LLMResult - -from agentverse.utils import AgentAction, AgentFinish - -from agentverse.parser import OutputParserError, output_parser_registry - - -@output_parser_registry.register("nlp_classroom_3players") -class NlpClassroom3PlayersParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - cleaned_output = text.strip() - cleaned_output = re.sub(r"\n+", "\n", cleaned_output) - cleaned_output = cleaned_output.split("\n") - if not ( - len(cleaned_output) == 2 - and cleaned_output[0].startswith("Action:") - and cleaned_output[1].startswith("Action Input:") - ): - raise OutputParserError(text) - action = cleaned_output[0][len("Action:") :].strip() - action_input = cleaned_output[1][len("Action Input:") :].strip() - if action == "Speak": - return AgentFinish({"output": action_input}, text) - else: - raise OutputParserError(text) diff --git a/agentverse/tasks/simulation/nlp_classroom_3players_withtool/output_parser.py b/agentverse/tasks/simulation/nlp_classroom_3players_withtool/output_parser.py deleted file mode 100644 index 2bb9a0aef..000000000 --- a/agentverse/tasks/simulation/nlp_classroom_3players_withtool/output_parser.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union - - -from agentverse.utils import AgentAction, AgentFinish - -from agentverse.parser import OutputParserError, output_parser_registry -from agentverse.parser import OutputParser -from agentverse.llms.base import LLMResult - - -@output_parser_registry.register("nlp_classroom_3players_withtool") -class NlpClassroom3PlayersWithtoolParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - cleaned_output = text.strip() - cleaned_output = re.sub(r"\n+", "\n", cleaned_output) - cleaned_output = cleaned_output.split("\n") - if not ( - len(cleaned_output) == 3 - and cleaned_output[0].startswith("Thought:") - and cleaned_output[1].startswith("Action:") - and cleaned_output[2].startswith("Action Input:") - ): - raise OutputParserError(text) - action = cleaned_output[1][len("Action:") :].strip() - action_input = cleaned_output[2][len("Action Input:") :].strip() - if action in ["Speak"]: - return AgentFinish({"output": action_input}, text) - elif action == "CallOn": - return AgentFinish({"output": "[CallOn] " + action_input}, text) - elif action == "RaiseHand": - return AgentFinish({"output": "[RaiseHand] " + action_input}, text) - elif action == "Listen": - return AgentFinish({"output": ""}, text) - else: - return AgentAction(action.lower(), action_input, text) diff --git a/agentverse/tasks/simulation/nlp_classroom_9players/output_parser.py b/agentverse/tasks/simulation/nlp_classroom_9players/output_parser.py deleted file mode 100644 index 6a10c7f6d..000000000 --- a/agentverse/tasks/simulation/nlp_classroom_9players/output_parser.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union - -from agentverse.utils import AgentAction, AgentFinish - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms import LLMResult - - -@output_parser_registry.register("nlp_classroom_9players") -class NlpClassroom9PlayersParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - cleaned_output = text.strip() - cleaned_output = re.sub(r"\n+", "\n", cleaned_output) - cleaned_output = cleaned_output.split("\n") - if not ( - len(cleaned_output) == 2 - and cleaned_output[0].startswith("Action:") - and cleaned_output[1].startswith("Action Input:") - ): - raise OutputParserError(text) - action = cleaned_output[0][len("Action:") :].strip() - action_input = cleaned_output[1][len("Action Input:") :].strip() - if action == "Speak": - return AgentFinish({"output": action_input}, text) - elif action == "CallOn": - return AgentFinish({"output": "[CallOn] " + action_input}, text) - elif action == "RaiseHand": - return AgentFinish({"output": "[RaiseHand] " + action_input}, text) - elif action == "Listen": - return AgentFinish({"output": ""}, text) - else: - return AgentAction(action, action_input, text) diff --git a/agentverse/tasks/simulation/nlp_classroom_9players_group/output_parser.py b/agentverse/tasks/simulation/nlp_classroom_9players_group/output_parser.py deleted file mode 100644 index 91c87c89b..000000000 --- a/agentverse/tasks/simulation/nlp_classroom_9players_group/output_parser.py +++ /dev/null @@ -1,34 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union - -from agentverse.utils import AgentAction, AgentFinish - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms import LLMResult - - -@output_parser_registry.register("nlp_classroom_9players_group") -class NlpClassroom9PlayersGroupParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - cleaned_output = text.strip() - cleaned_output = re.sub(r"\n+", "\n", cleaned_output) - cleaned_output = cleaned_output.split("\n") - if not ( - len(cleaned_output) == 2 - and cleaned_output[0].startswith("Action:") - and cleaned_output[1].startswith("Action Input:") - ): - raise OutputParserError(text) - action = cleaned_output[0][len("Action:") :].strip() - action_input = cleaned_output[1][len("Action Input:") :].strip() - if action == "Speak": - return AgentFinish({"output": action_input}, text) - elif action in ["CallOn", "RaiseHand", "GroupDiscuss"]: - return AgentFinish({"output": f"[{action}] {action_input}"}, text) - elif action == "Listen": - return AgentFinish({"output": ""}, text) - else: - return AgentAction(action, action_input, text) diff --git a/agentverse/tasks/simulation/pokemon/output_parser.py b/agentverse/tasks/simulation/pokemon/output_parser.py deleted file mode 100644 index e96f7e3cf..000000000 --- a/agentverse/tasks/simulation/pokemon/output_parser.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -import re -import json -from typing import Union - -from agentverse.parser import OutputParser, LLMResult - -# from langchain.schema import AgentAction, AgentFinish -from agentverse.utils import AgentAction, AgentFinish - -from agentverse.parser import OutputParserError, output_parser_registry - - -@output_parser_registry.register("pokemon") -class PokemonParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - cleaned_output = text.strip() - cleaned_output = re.sub(r"\n+", "\n", cleaned_output) - cleaned_output = cleaned_output.split("\n") - if not ( - len(cleaned_output) == 3 - and cleaned_output[0].startswith("Thought:") - and cleaned_output[1].startswith("Action:") - and cleaned_output[2].startswith("Action Input:") - ): - raise OutputParserError(text) - action = cleaned_output[1][len("Action:") :].strip() - action_input = cleaned_output[2][len("Action Input:") :].strip() - try: - action_input = json.loads(action_input) - except json.JSONDecodeError: - raise OutputParserError(text) - action_input["action"] = action - return AgentFinish({"output": json.dumps(action_input)}, text) diff --git a/agentverse/tasks/simulation/prisoner_dilemma/output_parser.py b/agentverse/tasks/simulation/prisoner_dilemma/output_parser.py deleted file mode 100644 index 05a41e0f8..000000000 --- a/agentverse/tasks/simulation/prisoner_dilemma/output_parser.py +++ /dev/null @@ -1,74 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union, TYPE_CHECKING - -# from langchain.agents import AgentOutputParser -from agentverse.parser import OutputParser, LLMResult -#from langchain.schema import AgentAction, AgentFinish -from agentverse.utils import AgentAction, AgentFinish -from agentverse.parser import OutputParserError, output_parser_registry - -if TYPE_CHECKING: - from agentverse.agents.base import BaseAgent - from agentverse.environments.base import BaseEnvironment - - -@output_parser_registry.register("prisoner_dilemma") -class PrisonerDilemmaParser(OutputParser): - # make sure 1 1 2 2 3 3 - cur_round: int = 1 - encounter_cur_round: bool = False - - def parse( - self, agent: "BaseAgent", environment: "BaseEnvironment", output: LLMResult - ) -> Union[AgentAction, AgentFinish]: - text = output.content - cleaned_output = text.strip() - cleaned_output = re.sub(r"\n+", "\n", cleaned_output) - cleaned_output = cleaned_output.split("\n") - if not ( - len(cleaned_output) == 2 - and cleaned_output[0].startswith("Action:") - and cleaned_output[1].startswith("Action Input:") - ): - raise OutputParserError(text) - action = cleaned_output[0][len("Action:") :].strip() - action_input = cleaned_output[1][len("Action Input:") :].strip() - - if action == "Speak": - # make sure the police count the round right - # if agent.name == "Police": - # action_input = re.sub(r'Round (\d+)', f'Round {self.cur_round}', action_input) - # self.cur_round += 1 - # if self.encounter_cur_round: - # self.encounter_cur_round = False - # self.cur_round += 1 - # else: - # self.encounter_cur_round = True - - # each time police speak is a new round - if agent.name == "Police": - if environment.cnt_turn == (environment.max_turns - 4): - action_input = ( - "Attention! You are now required to made your final decision and I will made the " - "final judgement to both of you based on this time, Please Answer now !" - ) - - elif environment.cnt_turn == (environment.max_turns - 2): - action_input = "Attention! Suspect2, it's now your time to make your final decision, Please Answer now !" - - # elif self.cur_round == 1: - # action_input = "Hey Listen! You are both arrested, and I am going to give you both a chance to walk out of here," \ - # "But you should comply with the following rules:" \ - # "- If one of you are willing to testifies against the other and the other one remains silent, then the one who testifies will be released IMMEDIATELY, while the silent one will be sentenced to TEN years in prison." \ - # "- If both of you remain silent, you will each receive a sentence of ONE year in prison." \ - # "- It seems that always testifying is a goog strategy, So! if you both choose to testify against each other, you will each receive a sentence of FIVE years in prison." \ - # "Now, it's your time to consider testifying or remaining silent. Remember this is a best chance you might ever have to walk out of here without guilty." \ - # "I will noticed both of you WHEN you have to make your final decision! Before that, try to make your best!" \ - - self.cur_round += 1 - - return AgentFinish({"output": action_input}, text) - else: - raise OutputParserError(text) diff --git a/agentverse/tasks/simulation/sde_team/sde_team_2players/output_parser.py b/agentverse/tasks/simulation/sde_team/sde_team_2players/output_parser.py deleted file mode 100644 index f7cea629c..000000000 --- a/agentverse/tasks/simulation/sde_team/sde_team_2players/output_parser.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union - -#from langchain.agents import AgentOutputParser - -# from langchain.schema import AgentAction, AgentFinish - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms.base import LLMResult -from agentverse.utils import AgentAction, AgentFinish - - -@output_parser_registry.register("sde_team/sde_team_2players") -class SdeTeamGivenTestsParser(OutputParser): - #def parse(self, agent, env, output: LLMResult) -> Union[AgentAction, AgentFinish]: - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - return AgentFinish({"output": output.content}, output.content) diff --git a/agentverse/tasks/simulation/sde_team/sde_team_3players/output_parser.py b/agentverse/tasks/simulation/sde_team/sde_team_3players/output_parser.py deleted file mode 100644 index 304dd7f99..000000000 --- a/agentverse/tasks/simulation/sde_team/sde_team_3players/output_parser.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union - -# from langchain.agents import AgentOutputParser - -# from langchain.schema import AgentAction, AgentFinish - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms.base import LLMResult -from agentverse.utils import AgentAction, AgentFinish - - -@output_parser_registry.register("sde_team/sde_team_3players") -class SdeTeamParser(OutputParser): - #def parse(self, agent, env, output: LLMResult) -> Union[AgentAction, AgentFinish]: - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - return AgentFinish({"output": output.content}, output.content) diff --git a/agentverse/tasks/tasksolving/brainstorming/output_parser.py b/agentverse/tasks/tasksolving/brainstorming/output_parser.py deleted file mode 100644 index d0041ec46..000000000 --- a/agentverse/tasks/tasksolving/brainstorming/output_parser.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union, List, Tuple - -from agentverse.utils import AgentAction, AgentFinish, AgentCriticism - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms import LLMResult -from agentverse.logging import get_logger - - -logger = get_logger() - - -@output_parser_registry.register("role_assigner") -class RoleAssignerParser(OutputParser): - cnt_critic_agents: int = 0 - - def parse(self, output: LLMResult) -> List[str]: - text = output.content - pattern = re.compile(r"\d\.\s*(.+)") - roles = pattern.findall(text) - if len(roles) < self.cnt_critic_agents: - logger.error( - f"Role assigner failed to assign roles to {self.cnt_critic_agents} critics!" - ) - raise OutputParserError(text) - return roles - - -@output_parser_registry.register("critic") -class CriticParser(OutputParser): - def parse(self, output: LLMResult) -> AgentCriticism: - text = output.content - text = re.sub(r"\n+", "\n", text.strip()) - checks = text.split("\n") - if not (checks[0].startswith("Action:")): - raise OutputParserError(text) - if checks[0].strip(". ") == "Action: Agree": - return AgentCriticism(True, "") - elif checks[0].strip(". ") == "Action: Disagree": - pattern = re.compile(r"Action Input: ([\S\n ]+)") - try: - criticism = pattern.findall(text)[0].strip() - except IndexError: - logger.error("Bad response from critic!") - raise OutputParserError(text) - return AgentCriticism(False, criticism) - else: - raise OutputParserError(text) - - -@output_parser_registry.register("evaluator") -class EvaluatorParser(OutputParser): - dimensions: List[str] = None - - def parse(self, output: LLMResult) -> Tuple[List[int], str]: - text = output.content - cleaned_output = re.sub(r"\n+", "\n", text.strip()) - checks = cleaned_output.split("\n") - patterns = [ - re.compile(r"(?:\d\.\s*)?" + dimension + r":\s*(\d)") - for dimension in self.dimensions - ] - try: - # find score and advice - score = [ - int(pattern.findall(checks[i])[0]) for i, pattern in enumerate(patterns) - ] - advice_text = "".join(checks[len(self.dimensions) :]) - advice = re.findall(r"(?:\d\.\s*)?Advice:\s*(.+)", advice_text)[0] - # logger.info("Evaluator give the following advice:\n" + advice) - except (IndexError, ValueError): - # logger.error("Bad response from evaluator!") - raise OutputParserError(text) - return score, advice diff --git a/agentverse/tasks/tasksolving/commongen/output_parser.py b/agentverse/tasks/tasksolving/commongen/output_parser.py deleted file mode 100644 index 88d3e166a..000000000 --- a/agentverse/tasks/tasksolving/commongen/output_parser.py +++ /dev/null @@ -1,93 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union, List, Tuple - -from agentverse.utils import AgentAction, AgentFinish, AgentCriticism - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms import LLMResult -from agentverse.logging import get_logger - -logger = get_logger() - - -@output_parser_registry.register("commongen") -class CommongenParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - return AgentFinish({"output": output.content}, output.content) - - -@output_parser_registry.register("commongen-solver") -class CommongenSolverParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - end_pos = text.rfind("```") - if end_pos == -1: - raise OutputParserError(text) - text = text[:end_pos] - cleaned_output = text.strip().strip("```").strip() - if cleaned_output.startswith("python"): - cleaned_output = cleaned_output[6:].strip() - elif cleaned_output.startswith("python3"): - cleaned_output = cleaned_output[7:].strip() - return AgentFinish({"output": cleaned_output}, text) - - -@output_parser_registry.register("commongen-evaluator") -class CommongenEvaluatorParser(OutputParser): - dimensions: List[str] = None - - def parse(self, output: LLMResult) -> Tuple[List[int], str]: - text = output.content - cleaned_output = re.sub(r"\n+", "\n", text.strip()) - checks = cleaned_output.split("\n") - - patterns = [ - re.compile(r"(?:\d.\s*)?" + dimension + r":\s*(\d)") - for dimension in self.dimensions - ] - - advice = "" - for check in reversed(checks): - advice = check + advice - if check.startswith("Advice:"): - break - checks[-1] = advice - try: - # find score and advice - score = [] - for pattern in patterns: - for check in checks[:-1]: - if pattern.findall(check): - score.append(bool(pattern.findall(check)[0])) - break - advice = re.findall(r"(?:\d.\s*)?Advice:\s*(.+)", checks[-1])[0] - # logger.info("Evaluator give the following advice:\n" + advice) - except (IndexError, ValueError): - # logger.error("Bad response from evaluator!") - raise OutputParserError(text) - return score[0], advice - - -@output_parser_registry.register("commongen-critic") -class CommongenCriticParser(OutputParser): - def parse(self, output: LLMResult) -> AgentCriticism: - text = output.content - text = re.sub(r"\n+", "\n", text.strip()) - checks = text.split("\n") - if not (checks[0].startswith("Action:")): - raise OutputParserError(text) - if checks[0].strip(". ") == "Action: Agree": - return AgentCriticism(True, "") - elif checks[0].strip(". ") == "Action: Disagree": - pattern = re.compile(r"Action Input: ([\S\n ]+)") - try: - criticism = pattern.findall(text)[0].strip() - except IndexError: - # logger.error("Bad response from critic!") - # raise OutputParserError(text) - criticism = "I think the solution is not correct. Please think carefully and correct it." - return AgentCriticism(False, criticism) - else: - raise OutputParserError(text) diff --git a/agentverse/tasks/tasksolving/humaneval/output_parser.py b/agentverse/tasks/tasksolving/humaneval/output_parser.py deleted file mode 100644 index 1eb9c023d..000000000 --- a/agentverse/tasks/tasksolving/humaneval/output_parser.py +++ /dev/null @@ -1,311 +0,0 @@ -from __future__ import annotations - -import re -import json -import ast -from typing import Union, List, Tuple - -from agentverse.utils import AgentAction, AgentFinish, AgentCriticism - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms import LLMResult -from agentverse.logging import get_logger - -logger = get_logger() - - -@output_parser_registry.register("humaneval") -class HumanevalParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - return AgentFinish({"output": output.content}, output.content) - - -@output_parser_registry.register("humaneval-solver") -class HumanevalSolverParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - # start_pos = text.find("```") - # end_pos = text.rfind("```") - # if end_pos == -1: - # raise OutputParserError(text) - # text = text[start_pos:end_pos] - # cleaned_output = text.strip().strip("```").strip() - # if cleaned_output.startswith("python"): - # cleaned_output = cleaned_output[6:].strip() - # elif cleaned_output.startswith("python3"): - # cleaned_output = cleaned_output[7:].strip() - code = re.findall(r"```.*?\n(.+?)```", text, re.DOTALL)[-1] - - return AgentFinish({"output": code}, text) - - -@output_parser_registry.register("humaneval-critic-central") -class HumanevalCriticParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - return AgentCriticism(False, output.content) - - -@output_parser_registry.register("humaneval-solver-autogpt") -class HumanevalSolverParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - json_dict = re.findall(r"```.*?\n(.+?)```", text, re.DOTALL)[-1] - try: - cleaned_output = ast.literal_eval(json_dict) - except BaseException as e: - raise OutputParserError(text) - if "code" not in json_dict: - raise OutputParserError(text) - return AgentFinish({"output": cleaned_output["code"]}, text) - - -@output_parser_registry.register("humaneval-solver-autogpt-2") -class HumanevalSolverParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - try: - parsed_result = re.findall( - r"Text:(.+?)Reasoning:(.+?)Criticism:(.+?)Code:(.+)", text, re.DOTALL - )[0] - except BaseException as e: - raise OutputParserError(text) - code = parsed_result[-1].strip() - if code.startswith("```"): - try: - code = re.findall(r"```.*?\n(.+?)```", code, re.DOTALL)[0].strip() - except BaseException as e: - raise OutputParserError(text) - return AgentFinish({"output": code}, text) - - -@output_parser_registry.register("humaneval-manager") -class HumanevalManagerParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - return AgentFinish({"output": output.content}, output.content) - - -# @output_parser_registry.register("humaneval-solver") -# class HumanevalSolverParser(OutputParser): -# def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: -# text = output.content -# end_pos = text.rfind("```") -# if end_pos == -1: -# raise OutputParserError(text) -# text = text[:end_pos] -# cleaned_output = text.strip().strip("```").strip() -# if cleaned_output.startswith("python"): -# cleaned_output = cleaned_output[6:].strip() -# elif cleaned_output.startswith("python3"): -# cleaned_output = cleaned_output[7:].strip() -# return AgentFinish({"output": cleaned_output}, text) - - -@output_parser_registry.register("humaneval-executor-autogpt") -class HumanevalSolverParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - json_dict = re.findall(r"```.*?\n(.+?)```", text, re.DOTALL)[-1] - try: - cleaned_output = ast.literal_eval(json_dict) - except BaseException as e: - raise OutputParserError(text) - if not ( - "code" in json_dict and "file_path" in json_dict and "command" in json_dict - ): - raise OutputParserError(text) - return AgentFinish({"output": cleaned_output}, text) - - -@output_parser_registry.register("humaneval-executor") -class HumanevalSolverParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - try: - parsed_result = re.findall( - r"Thought:(.+?)Reasoning:(.+?)Criticism:(.+?)File Path:(.+?)Code:(.+?)Command:(.+)", - text, - re.DOTALL, - )[0] - cleaned_output = { - "thought": parsed_result[0].strip(), - "reasoning": parsed_result[1].strip(), - "criticism": parsed_result[2].strip(), - "file_path": parsed_result[3].strip().strip("`"), - "code": parsed_result[4] - .strip() - .strip("```") - .strip("python") - .strip("python3"), - "command": parsed_result[5].strip().strip("`"), - } - except BaseException as e: - raise OutputParserError(text) - - return AgentFinish({"output": cleaned_output}, text) - - - -@output_parser_registry.register("humaneval-executor-fc") -class HumanevalSolverParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - - #print("======") - #print(output) - #print("======") - - try: - #output_dict = eval(text) - output_dict = json.loads(text, strict=False) #The control characters (character codes in the 0-31 range, including '\t' (tab), '\n', '\r' and '\0'.") will be allowed inside strings - ''' - cleaned_output = { - "thought": output_dict["thought"].strip(), - "file_path": output_dict["file_path"].strip().strip("`"), - "code": output_dict["code"] - .strip() - .strip("```") - .strip("python") - .strip("python3"), - "command": output_dict["command"].strip().strip("`"), - } - ''' - cleaned_output = output_dict - except BaseException as e: - raise OutputParserError(text) - - return AgentFinish({"output": cleaned_output}, text) - - - -@output_parser_registry.register("humaneval-evaluator") -class HumanevalEvaluatorParser(OutputParser): - dimensions: List[str] = None - - def parse(self, output: LLMResult) -> Tuple[List[int], str]: - text = output.content - cleaned_output = re.sub(r"\n+", "\n", text.strip()) - checks = cleaned_output.split("\n") - - patterns = [ - re.compile(r"(?:\d.\s*)?" + dimension + r":\s*(\d)") - for dimension in self.dimensions - ] - - advice = "" - for check in reversed(checks): - advice = check + advice - if check.startswith("Advice:"): - break - checks[-1] = advice - try: - # find score and advice - score = [] - for pattern in patterns: - for check in checks[:-1]: - if pattern.findall(check): - score.append(bool(int(pattern.findall(check)[0]))) - break - advice = re.findall(r"(?:\d.\s*)?Advice:\s*(.+)", checks[-1])[0] - # logger.info("Evaluator give the following advice:\n" + advice) - except (IndexError, ValueError): - # logger.error("Bad response from evaluator!") - raise OutputParserError(text) - return score[0], advice - - -@output_parser_registry.register("humaneval-evaluator-2") -class HumanevalEvaluatorParser(OutputParser): - dimensions: List[str] = None - - def parse(self, output: LLMResult) -> Tuple[List[int], str]: - text = output.content - pattern = re.compile( - r"Response:(.+?)" - + "".join( - [ - f"{dimension}:(.+?)" - if i != len(self.dimensions) - 1 - else f"{dimension}:(.+)" - for i, dimension in enumerate(self.dimensions) - ] - ), - re.DOTALL, - ) - try: - parsed_result = pattern.findall(text)[0] - score = [bool(int(x.strip())) for x in parsed_result[1:]] - advice = parsed_result[0].strip() - except BaseException as e: - raise OutputParserError(text) - return score[0], advice - - -@output_parser_registry.register("humaneval-critic") -class HumanevalyCriticParser(OutputParser): - def parse(self, output: LLMResult) -> AgentCriticism: - text = output.content - text = re.sub(r"\n+", "\n", text.strip()) - checks = text.split("\n") - if not (checks[0].startswith("Action:")): - raise OutputParserError(text) - if checks[0].strip(". ") == "Action: Agree": - return AgentCriticism(True, "") - elif checks[0].strip(". ") == "Action: Disagree": - pattern = re.compile(r"Action Input: ([\S\n ]+)") - try: - criticism = pattern.findall(text)[0].strip() - except IndexError: - # logger.error("Bad response from critic!") - # raise OutputParserError(text) - criticism = "I think the solution is not correct. Please think carefully and correct it." - return AgentCriticism(False, criticism) - else: - raise OutputParserError(text) - - -@output_parser_registry.register("humaneval-critic-agree") -class HumanevalyCriticParser(OutputParser): - def parse(self, output: LLMResult) -> AgentCriticism: - text = output.content - if "[Agree]" in text: - return AgentCriticism(True, "") - else: - return AgentCriticism(False, text) - - -@output_parser_registry.register("humaneval-critic-autogpt") -class HumanevalCriticParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - try: - parsed_result = re.findall( - r"Text:(.+?)Reasoning:(.+?)Criticism:(.+?)Speak:(.+?)Final Decision:(.+)", - text, - re.DOTALL, - )[0] - except BaseException as e: - raise OutputParserError(text) - decision = parsed_result[-1].strip() - if "[Agree]" in decision: - return AgentCriticism(True, "") - else: - return AgentCriticism(False, parsed_result[-2].strip()) - - -@output_parser_registry.register("humaneval-critic-autogpt-2") -class HumanevalCriticParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - try: - parsed_result = re.findall( - r"Problem Analysis:(.+?)Solution Analysis:(.+?)Decision:(.+?)Suggestion:(.+)", - text, - re.DOTALL, - )[0] - except BaseException as e: - raise OutputParserError(text) - decision = parsed_result[-2].strip() - if "[Agree]" in decision: - return AgentCriticism(True, "") - else: - return AgentCriticism(False, parsed_result[-1].strip()) diff --git a/agentverse/tasks/tasksolving/logic_grid/output_parser.py b/agentverse/tasks/tasksolving/logic_grid/output_parser.py deleted file mode 100644 index 7f8a0d10d..000000000 --- a/agentverse/tasks/tasksolving/logic_grid/output_parser.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union, List, Tuple - -from agentverse.utils import AgentAction, AgentFinish, AgentCriticism - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms import LLMResult -from agentverse.logging import get_logger - - -logger = get_logger() diff --git a/agentverse/tasks/tasksolving/mgsm/output_parser.py b/agentverse/tasks/tasksolving/mgsm/output_parser.py deleted file mode 100644 index f7cd2c0f4..000000000 --- a/agentverse/tasks/tasksolving/mgsm/output_parser.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union, List, Tuple - -from agentverse.utils import AgentAction, AgentFinish, AgentCriticism - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms import LLMResult -from agentverse.logging import get_logger - - -logger = get_logger() - - -@output_parser_registry.register("mgsm") -class MGSMParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - return AgentFinish({"output": output.content}, output.content) - - -@output_parser_registry.register("mgsm-solver-autogpt") -class MGSMSolverParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - try: - parsed_result = re.findall( - r"Thought:(.+?)Reasoning:(.+?)Criticism:(.+?)Solution:(.+)", - text, - re.DOTALL, - )[0] - except BaseException as e: - raise OutputParserError(text) - return AgentFinish({"output": re.sub(r"\n+", "\n", text.strip())}, text) - - -@output_parser_registry.register("mgsm-evaluator") -class MGSMEvaluatorParser(OutputParser): - dimensions: List[str] = None - - def parse(self, output: LLMResult) -> Tuple[List[int], str]: - text = output.content - cleaned_output = re.sub(r"\n+", "\n", text.strip()) - # checks = cleaned_output.split("\n") - - patterns = [ - re.compile(r"(?:\d.\s*)?" + dimension + r":\s*(\d)") - for dimension in self.dimensions - ] - try: - # find score and advice - score_num = [ - int(pattern.findall(cleaned_output)[0]) - for i, pattern in enumerate(patterns) - ][0] - if score_num == 0: - score = False - elif score_num == 1: - score = True - else: - raise ValueError("Bad score!") - pat = re.compile(r"(?:\d.\s*)?Response:\s*(.+)", re.DOTALL) - advice = pat.findall(cleaned_output)[0] - # logger.info("Evaluator give the following advice:\n" + advice) - except (IndexError, ValueError): - # logger.error("Bad response from evaluator!") - raise OutputParserError(text) - return score, advice - - -@output_parser_registry.register("mgsm-evaluator-autogpt") -class MGSMCriticParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - try: - parsed_result = re.findall( - r"Thought:(.+?)Reasoning:(.+?)Criticism:(.+?)Speak:(.+?)Correctness:(.+)", - text, - re.DOTALL, - )[0] - score = int(parsed_result[-1]) - advice = parsed_result[-2] - except BaseException as e: - raise OutputParserError(text) - return score, advice - - -@output_parser_registry.register("mgsm-critic") -class MGSMCriticParser(OutputParser): - def parse(self, output: LLMResult) -> AgentCriticism: - text = output.content - return AgentCriticism(False, text) - # text = re.sub(r"\n+", "\n", text.strip()) - # checks = text.split("\n") - # if not text.startswith("Thought:"): - # raise OutputParserError(text) - # if not (checks[0].startswith("Action:")): - # raise OutputParserError(text) - # if checks[0].strip(". ") == "Action: Agree": - # return AgentCriticism(True, "") - if "[Correct]" in text: - return AgentCriticism(True, "") - else: - # pattern = re.compile(r"Action Input: ([\S\n ]+)") - # try: - # criticism = pattern.findall(text)[0].strip() - # criticism = ( - # re.findall(r"Output:\S?(.+)", text)[0].replace("[Wrong]", "") - # ).strip() - criticism = text.replace("[Wrong]", "").strip() - # except IndexError: - # logger.error("Bad response from critic!") - # raise OutputParserError(text) - # criticism = "I think the solution is not correct. Please think carefully and correct it." - return AgentCriticism(False, criticism) - # else: - # raise OutputParserError(text) - - -@output_parser_registry.register("mgsm-critic-autogpt") -class MGSMCriticParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - try: - parsed_result = re.findall( - r"Thought:(.+?)Reasoning:(.+?)Criticism:(.+?)Speak:(.+?)Decision:(.+)", - text, - re.DOTALL, - )[0] - except BaseException as e: - raise OutputParserError(text) - if "[Agree]" in parsed_result[-1]: - return AgentCriticism(True, "") - else: - return AgentCriticism(False, parsed_result[-2]) - - -@output_parser_registry.register("mgsm-critic-agree") -class MGSMCriticAgreeParser(OutputParser): - def parse(self, output: LLMResult) -> AgentCriticism: - text = output.content - text = re.sub(r"\n+", "\n", text.strip()) - # checks = text.split("\n") - # if not text.startswith("Thought:"): - # raise OutputParserError(text) - # if not (checks[0].startswith("Action:")): - # raise OutputParserError(text) - # if checks[0].strip(". ") == "Action: Agree": - # return AgentCriticism(True, "") - if "[Agree]" in text: - return AgentCriticism(True, "") - else: - # pattern = re.compile(r"Action Input: ([\S\n ]+)") - # try: - # criticism = pattern.findall(text)[0].strip() - # criticism = ( - # re.findall(r"Output:\S?(.+)", text)[0].replace("[Wrong]", "") - # ).strip() - criticism = text.replace("[Disagree]", "").strip() - # except IndexError: - # logger.error("Bad response from critic!") - # raise OutputParserError(text) - # criticism = "I think the solution is not correct. Please think carefully and correct it." - return AgentCriticism(False, criticism) - # else: - # raise OutputParserError(text) diff --git a/agentverse/tasks/tasksolving/pythoncalculator/output_parser.py b/agentverse/tasks/tasksolving/pythoncalculator/output_parser.py deleted file mode 100644 index e37a808f6..000000000 --- a/agentverse/tasks/tasksolving/pythoncalculator/output_parser.py +++ /dev/null @@ -1,19 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union, List, Tuple - -from agentverse.utils import AgentAction, AgentFinish, AgentCriticism - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms import LLMResult -from agentverse.logging import get_logger - - -logger = get_logger() - - -@output_parser_registry.register("dummy") -class PipelinePythoncalculatorParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - return AgentFinish({"output": output.content}, output.content) diff --git a/agentverse/tasks/tasksolving/responsegen/output_parser.py b/agentverse/tasks/tasksolving/responsegen/output_parser.py deleted file mode 100644 index e24009f78..000000000 --- a/agentverse/tasks/tasksolving/responsegen/output_parser.py +++ /dev/null @@ -1,104 +0,0 @@ -from __future__ import annotations - -import re -from typing import Union, List, Tuple - -from agentverse.utils import AgentAction, AgentFinish, AgentCriticism - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms import LLMResult -from agentverse.logging import get_logger - -logger = get_logger() - - -@output_parser_registry.register("responsegen") -class ResponseGenParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - return AgentFinish({"output": output.content}, output.content) - - -@output_parser_registry.register("responsegen-evaluator") -class ResponseGenEvaluatorParser(OutputParser): - dimensions: List[str] = None - - def parse(self, output: LLMResult) -> Tuple[List[int], str]: - text = output.content - cleaned_output = re.sub(r"\n+", "\n", text.strip()) - checks = cleaned_output.split("\n") - - patterns = [ - re.compile(r"(?:\d.\s*)?" + dimension + r":\s*(\d+)") - for dimension in self.dimensions - ] - - advice = "" - for check in reversed(checks): - advice = check + advice - if check.startswith("Advice:"): - break - checks[-1] = advice - try: - # find score and advice - score = [ - int(pattern.findall(checks[i])[0]) for i, pattern in enumerate(patterns) - ] - advice = re.findall(r"(?:\d.\s*)?Advice:\s*(.+)", checks[-1])[0] - # logger.info("Evaluator give the following advice:\n" + advice) - except (IndexError, ValueError): - # logger.error("Bad response from evaluator!") - raise OutputParserError(text) - return score, advice - - -@output_parser_registry.register("responsegen-critic") -class ResponseGenCriticParser(OutputParser): - def parse(self, output: LLMResult) -> AgentCriticism: - text = output.content - text = re.sub(r"\n+", "\n", text.strip()) - checks = text.split("\n") - if not (checks[0].startswith("Action:")): - raise OutputParserError(text) - if checks[0].strip(". ") == "Action: Agree": - return AgentCriticism(True, "") - elif checks[0].strip(". ") == "Action: Disagree": - pattern = re.compile(r"Action Input: ([\S\n ]+)") - try: - criticism = pattern.findall(text)[0].strip() - except IndexError: - criticism = "I think the can be further improved." - # raise OutputParserError(text) - return AgentCriticism(False, criticism) - else: - raise OutputParserError(text) - - -@output_parser_registry.register("responsegen-critic-2") -class ResponseGenCriticParser(OutputParser): - def parse(self, output: LLMResult) -> AgentCriticism: - text = output.content - # text = re.sub(r"\n+", "\n", text.strip()) - # checks = text.split("\n") - # if not (checks[0].startswith("Action:")): - # raise OutputParserError(text) - # if checks[0].strip(". ") == "Action: Agree": - # return AgentCriticism(True, "") - # elif checks[0].strip(". ") == "Action: Disagree": - # pattern = re.compile(r"Action Input: ([\S\n ]+)") - # try: - # criticism = pattern.findall(text)[0].strip() - # except IndexError: - # # criticism = "I think the solution is not correct. Please think carefully and correct it." - # raise OutputParserError(text) - # return AgentCriticism(False, criticism) - # else: - # raise OutputParserError(text) - result = re.findall(r"Decision:(.+?)Response:(.+)", text, re.DOTALL) - if len(result) == 0: - result = ["Disagree", "I think the response can be further improved."] - else: - result = result[0] - if "Agree" in result[0]: - return AgentCriticism(True, "") - else: - return AgentCriticism(False, result[1].strip()) diff --git a/agentverse/tasks/tasksolving/tool_using/24point/config.yaml b/agentverse/tasks/tasksolving/tool_using/24point/config.yaml index 4e65b3f5b..2ac61f385 100644 --- a/agentverse/tasks/tasksolving/tool_using/24point/config.yaml +++ b/agentverse/tasks/tasksolving/tool_using/24point/config.yaml @@ -153,7 +153,7 @@ agents: temperature: 0 max_tokens: 512 output_parser: - type: role_description_name_assigner + type: role-description-name-assigner - #solver_agent: agent_type: solver diff --git a/agentverse/tasks/tasksolving/tool_using/bmi/config.yaml b/agentverse/tasks/tasksolving/tool_using/bmi/config.yaml index 97411d782..d950f8a29 100644 --- a/agentverse/tasks/tasksolving/tool_using/bmi/config.yaml +++ b/agentverse/tasks/tasksolving/tool_using/bmi/config.yaml @@ -153,7 +153,7 @@ agents: temperature: 0 max_tokens: 512 output_parser: - type: role_description_name_assigner + type: role-description-name-assigner - #solver_agent: agent_type: solver diff --git a/agentverse/tasks/tasksolving/tool_using/bookclub/config.yaml b/agentverse/tasks/tasksolving/tool_using/bookclub/config.yaml index abc03fe11..381bb99e2 100644 --- a/agentverse/tasks/tasksolving/tool_using/bookclub/config.yaml +++ b/agentverse/tasks/tasksolving/tool_using/bookclub/config.yaml @@ -153,7 +153,7 @@ agents: temperature: 0 max_tokens: 512 output_parser: - type: role_description_name_assigner + type: role-description-name-assigner - #solver_agent: agent_type: solver diff --git a/agentverse/tasks/tasksolving/tool_using/car/config.yaml b/agentverse/tasks/tasksolving/tool_using/car/config.yaml index 4344c707e..c2a3ddddf 100644 --- a/agentverse/tasks/tasksolving/tool_using/car/config.yaml +++ b/agentverse/tasks/tasksolving/tool_using/car/config.yaml @@ -153,7 +153,7 @@ agents: temperature: 0 max_tokens: 512 output_parser: - type: role_description_name_assigner + type: role-description-name-assigner - #solver_agent: agent_type: solver diff --git a/agentverse/tasks/tasksolving/tool_using/date/config.yaml b/agentverse/tasks/tasksolving/tool_using/date/config.yaml index 6e12f1746..46865cde2 100644 --- a/agentverse/tasks/tasksolving/tool_using/date/config.yaml +++ b/agentverse/tasks/tasksolving/tool_using/date/config.yaml @@ -155,7 +155,7 @@ agents: temperature: 0 max_tokens: 512 output_parser: - type: role_description_name_assigner + type: role-description-name-assigner - #solver_agent: agent_type: solver diff --git a/agentverse/tasks/tasksolving/tool_using/diy/config.yaml b/agentverse/tasks/tasksolving/tool_using/diy/config.yaml index 8fa2f173c..ca012db37 100644 --- a/agentverse/tasks/tasksolving/tool_using/diy/config.yaml +++ b/agentverse/tasks/tasksolving/tool_using/diy/config.yaml @@ -155,7 +155,7 @@ agents: temperature: 0 max_tokens: 512 output_parser: - type: role_description_name_assigner + type: role-description-name-assigner - #solver_agent: agent_type: solver diff --git a/agentverse/tasks/tasksolving/tool_using/output_parser.py b/agentverse/tasks/tasksolving/tool_using/output_parser.py deleted file mode 100644 index f3b38450d..000000000 --- a/agentverse/tasks/tasksolving/tool_using/output_parser.py +++ /dev/null @@ -1,77 +0,0 @@ -from __future__ import annotations - -import re -import ast -from typing import Union, List, Tuple - -from agentverse.utils import AgentAction, AgentFinish, AgentCriticism - -from agentverse.parser import OutputParserError, output_parser_registry, OutputParser -from agentverse.llms import LLMResult -from agentverse.logging import get_logger - -logger = get_logger() - - -@output_parser_registry.register("role_description_name_assigner") -class RoleAssignerParser(OutputParser): - cnt_critic_agents: int = 0 - - def parse(self, output: LLMResult) -> List[str]: - text = output.content - pattern = re.compile(r"\d+?\.\s*(.+?) - (.+)") - roles = pattern.findall(text) - if len(roles) < self.cnt_critic_agents: - logger.error( - f"Role assigner failed to assign roles to {self.cnt_critic_agents} critics!" - ) - raise OutputParserError(text) - res = [] - for role in roles: - res.append({"name": role[0], "description": role[1]}) - return res - - -@output_parser_registry.register("tool-using-solver") -class SolverParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - text = output.content - pattern = re.compile(r"\d+?\.\s*(.+?) - (.+)") - tasks = pattern.findall(text) - if len(tasks) == 0: - raise OutputParserError(text) - return AgentFinish({"output": tasks}, text) - - -@output_parser_registry.register("tool-using-executor") -class ToolUsingSolverParser(OutputParser): - def parse(self, output: LLMResult) -> Union[AgentAction, AgentFinish]: - if output.function_name != "": - return AgentAction( - tool=output.function_name, - tool_input=output.function_arguments, - log=output.content, - ) - else: - return AgentFinish({"output": output.content}, output.content) - - -@output_parser_registry.register("tool-using-evaluator") -class HumanevalEvaluatorParser(OutputParser): - def parse(self, output: LLMResult) -> Tuple[List[int], str]: - text = output.content - try: - result = re.findall(r"Status:(.+?)Speak:(.+)", text, re.DOTALL)[0] - score = bool(int(result[0])) - words = result[1].strip() - except (IndexError, ValueError): - # logger.error("Bad response from evaluator!") - raise OutputParserError(text) - return score, words - - -@output_parser_registry.register("tool-using-critic") -class ToolUsingCriticParser(OutputParser): - def parse(self, output: LLMResult) -> AgentCriticism: - text = output.content - return AgentCriticism(False, text) diff --git a/agentverse/tasks/tasksolving/tool_using/party/config.yaml b/agentverse/tasks/tasksolving/tool_using/party/config.yaml index df7fad0bb..22b4be5e6 100644 --- a/agentverse/tasks/tasksolving/tool_using/party/config.yaml +++ b/agentverse/tasks/tasksolving/tool_using/party/config.yaml @@ -155,7 +155,7 @@ agents: temperature: 0 max_tokens: 512 output_parser: - type: role_description_name_assigner + type: role-description-name-assigner - #solver_agent: agent_type: solver diff --git a/agentverse/tasks/tasksolving/tool_using/sudoku/config.yaml b/agentverse/tasks/tasksolving/tool_using/sudoku/config.yaml index 4d1202028..3fc1a615a 100644 --- a/agentverse/tasks/tasksolving/tool_using/sudoku/config.yaml +++ b/agentverse/tasks/tasksolving/tool_using/sudoku/config.yaml @@ -153,7 +153,7 @@ agents: temperature: 0 max_tokens: 512 output_parser: - type: role_description_name_assigner + type: role-description-name-assigner - #solver_agent: agent_type: solver diff --git a/agentverse/tasks/tasksolving/tool_using/trending/config.yaml b/agentverse/tasks/tasksolving/tool_using/trending/config.yaml index 101612774..25a778fd3 100644 --- a/agentverse/tasks/tasksolving/tool_using/trending/config.yaml +++ b/agentverse/tasks/tasksolving/tool_using/trending/config.yaml @@ -153,7 +153,7 @@ agents: temperature: 0 max_tokens: 512 output_parser: - type: role_description_name_assigner + type: role-description-name-assigner - #solver_agent: agent_type: solver diff --git a/agentverse/tasks/tasksolving/tool_using/vacation/config.yaml b/agentverse/tasks/tasksolving/tool_using/vacation/config.yaml index c10dd1ed0..04925ff89 100644 --- a/agentverse/tasks/tasksolving/tool_using/vacation/config.yaml +++ b/agentverse/tasks/tasksolving/tool_using/vacation/config.yaml @@ -153,7 +153,7 @@ agents: temperature: 0 max_tokens: 512 output_parser: - type: role_description_name_assigner + type: role-description-name-assigner - #solver_agent: agent_type: solver