From 8069b6d705892ad6d6dde94fc6a37171ce725eab Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Sat, 6 May 2023 16:47:15 +0000 Subject: [PATCH 01/32] add human agent and chat agent --- flaml/autogen/agent/chat_agent.py | 28 +++++++++++++++++ flaml/autogen/agent/human_agent.py | 50 ++++++++++++++++++++++++++++++ test/autogen/test_human_agent.py | 40 ++++++++++++++++++++++++ 3 files changed, 118 insertions(+) create mode 100644 flaml/autogen/agent/chat_agent.py create mode 100644 flaml/autogen/agent/human_agent.py create mode 100644 test/autogen/test_human_agent.py diff --git a/flaml/autogen/agent/chat_agent.py b/flaml/autogen/agent/chat_agent.py new file mode 100644 index 0000000000..1efad5228b --- /dev/null +++ b/flaml/autogen/agent/chat_agent.py @@ -0,0 +1,28 @@ +from .agent import Agent +from flaml.autogen.code_utils import DEFAULT_MODEL +from flaml import oai + + +class ChatAgent(Agent): + """Chat.""" + + DEFAULT_SYSTEM_MESSAGE = """You are a chat agent. + """ + + DEFAULT_CONFIG = { + "model": DEFAULT_MODEL, + } + + def __init__(self, name, system_message=DEFAULT_SYSTEM_MESSAGE, work_dir=None, **config): + super().__init__(name, system_message) + self._work_dir = work_dir + self._config = self.DEFAULT_CONFIG.copy() + self._config.update(config) + self._sender_dict = {} + + def receive(self, message, sender): + super().receive(message, sender) + responses = oai.ChatCompletion.create(messages=self._conversations[sender.name], **self._config) + # cost = oai.ChatCompletion.cost(responses) + response = oai.ChatCompletion.extract_text(responses)[0] + self._send(response, sender) diff --git a/flaml/autogen/agent/human_agent.py b/flaml/autogen/agent/human_agent.py new file mode 100644 index 0000000000..a2575a1cfd --- /dev/null +++ b/flaml/autogen/agent/human_agent.py @@ -0,0 +1,50 @@ +from .agent import Agent +from flaml.autogen.code_utils import extract_code, execute_code + + +class HumanAgent(Agent): + """A proxy agent for human, that can execute code and provide feedback to the other agents.""" + + DEFAULT_SYSTEM_MESSAGE = """You are human agent. You can execute_code or give feedback to the sender. + """ + + def __init__(self, name, system_message="", work_dir=None, **context): + super().__init__(name, system_message) + self._work_dir = work_dir + self._context = context + + def _is_termination_msg(self, message): + """Check if the message is a termination message.""" + if "_is_termination_msg" in self._context: + return self._context["_is_termination_msg"](message) + return True + + def receive(self, message, sender): + """Receive a message from the sender agent. + Every time a message is received, the human agent will give feedback. + """ + super().receive(message, sender) + # try to execute the code + code, lang = extract_code(message) + # no code block is found, lang should be "unknown" + print("lang: ", lang) + if lang == "unknown": + # to determine if the message is a termination message using a function + terminate = self._is_termination_msg(message) + feedback = input() + if feedback: + self._send(feedback, sender) + elif terminate: + return + else: + self._send("Continue", sender) + elif lang == "bash": + assert code.startswith("python ") + file_name = code[len("python ") :] + exitcode, logs = execute_code(filename=file_name, work_dir=self._work_dir) + self._send(f"exitcode: {exitcode}\n{logs.decode('utf-8')}", sender) + elif lang == "python": + # TODO: is there such a case? + exitcode, logs = execute_code(code, work_dir=self._work_dir) + print(exitcode, logs) + self._send(f"exitcode: {exitcode}\n{logs.decode('utf-8')}", sender) diff --git a/test/autogen/test_human_agent.py b/test/autogen/test_human_agent.py new file mode 100644 index 0000000000..5c8b7d37a2 --- /dev/null +++ b/test/autogen/test_human_agent.py @@ -0,0 +1,40 @@ +from flaml.autogen.code_utils import extract_code +from flaml import oai + + +def test_human_agent(): + try: + import openai + except ImportError: + return + from flaml.autogen.agent.chat_agent import ChatAgent + from flaml.autogen.agent.human_agent import HumanAgent + + conversations = {} + oai.ChatCompletion.start_logging(conversations) + agent = ChatAgent("chat_agent") + user = HumanAgent("human_user") + agent.receive( + """Write python code to solve the equation x^3=125. You must write code in the following format. You must always print the result. + Wait for me to return the result. + ```python + # your code + print(your_result) + ``` + """, + user, + ) + print(conversations) + + +if __name__ == "__main__": + import openai + + openai.api_key_path = "test/openai/key.txt" + # if you use Azure OpenAI, comment the above line and uncomment the following lines + # openai.api_type = "azure" + # openai.api_base = "https://.openai.azure.com/" + # openai.api_version = "2023-03-15-preview" # change if necessary + # openai.api_key = "" + # test_extract_code() + test_human_agent() From 38c84bfa91fa28432f6499c1fc084c5765c4f876 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Sat, 6 May 2023 12:57:17 -0400 Subject: [PATCH 02/32] feedback msg --- flaml/autogen/agent/human_agent.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/flaml/autogen/agent/human_agent.py b/flaml/autogen/agent/human_agent.py index a2575a1cfd..779ddd7bc7 100644 --- a/flaml/autogen/agent/human_agent.py +++ b/flaml/autogen/agent/human_agent.py @@ -31,6 +31,7 @@ def receive(self, message, sender): if lang == "unknown": # to determine if the message is a termination message using a function terminate = self._is_termination_msg(message) + print("Please give feedback to the sender (press enter to skip): ") feedback = input() if feedback: self._send(feedback, sender) @@ -38,13 +39,14 @@ def receive(self, message, sender): return else: self._send("Continue", sender) - elif lang == "bash": - assert code.startswith("python ") - file_name = code[len("python ") :] - exitcode, logs = execute_code(filename=file_name, work_dir=self._work_dir) - self._send(f"exitcode: {exitcode}\n{logs.decode('utf-8')}", sender) - elif lang == "python": - # TODO: is there such a case? - exitcode, logs = execute_code(code, work_dir=self._work_dir) - print(exitcode, logs) + else: + if lang == "bash": + assert code.startswith("python ") + file_name = code[len("python ") :] + exitcode, logs = execute_code(filename=file_name, work_dir=self._work_dir) + elif lang == "python": + exitcode, logs = execute_code(code, work_dir=self._work_dir) + else: + # TODO: could this happen? + raise NotImplementedError self._send(f"exitcode: {exitcode}\n{logs.decode('utf-8')}", sender) From fdcd0794637ce0fbeb4c91cbc4dad2478deaae2e Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Sat, 6 May 2023 13:09:06 -0400 Subject: [PATCH 03/32] clean print --- flaml/autogen/agent/human_agent.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flaml/autogen/agent/human_agent.py b/flaml/autogen/agent/human_agent.py index 779ddd7bc7..f560b20ca0 100644 --- a/flaml/autogen/agent/human_agent.py +++ b/flaml/autogen/agent/human_agent.py @@ -27,7 +27,6 @@ def receive(self, message, sender): # try to execute the code code, lang = extract_code(message) # no code block is found, lang should be "unknown" - print("lang: ", lang) if lang == "unknown": # to determine if the message is a termination message using a function terminate = self._is_termination_msg(message) From a42f50e2bcbe6371ac584f782d03a57f2aa65027 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Sat, 6 May 2023 13:16:12 -0400 Subject: [PATCH 04/32] remove redundant import --- test/autogen/test_human_agent.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/autogen/test_human_agent.py b/test/autogen/test_human_agent.py index 5c8b7d37a2..40d154bc98 100644 --- a/test/autogen/test_human_agent.py +++ b/test/autogen/test_human_agent.py @@ -1,12 +1,7 @@ -from flaml.autogen.code_utils import extract_code from flaml import oai def test_human_agent(): - try: - import openai - except ImportError: - return from flaml.autogen.agent.chat_agent import ChatAgent from flaml.autogen.agent.human_agent import HumanAgent From 969fbd84b24d4425bc22596a0c9d606d70b1c191 Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Sun, 7 May 2023 00:23:33 +0000 Subject: [PATCH 05/32] make coding agent work --- flaml/autogen/agent/agent.py | 2 +- flaml/autogen/agent/coding_agent.py | 50 ++++++++++++++++------------- flaml/autogen/agent/human_agent.py | 13 ++++---- test/autogen/test_agent.py | 18 +++++------ 4 files changed, 43 insertions(+), 40 deletions(-) diff --git a/flaml/autogen/agent/agent.py b/flaml/autogen/agent/agent.py index be7743b6ce..39e7501374 100644 --- a/flaml/autogen/agent/agent.py +++ b/flaml/autogen/agent/agent.py @@ -31,7 +31,7 @@ def _send(self, message, recipient): def _receive(self, message, sender): """Receive a message from another agent.""" - # print(self.name, "received message from", sender.name, ":", message) + print(self.name, "received message from", sender.name, ":", message) self._conversations[sender.name].append({"content": message, "role": "user"}) def receive(self, message, sender): diff --git a/flaml/autogen/agent/coding_agent.py b/flaml/autogen/agent/coding_agent.py index 159b603ac5..a666cd3de0 100644 --- a/flaml/autogen/agent/coding_agent.py +++ b/flaml/autogen/agent/coding_agent.py @@ -7,14 +7,17 @@ class PythonAgent(Agent): """(Experimental) Suggest code blocks.""" - DEFAULT_SYSTEM_MESSAGE = """You are a coding agent. You suggest python code for a user to execute for a given task. Don't suggest shell command. Output the code in a coding block. Check the execution result. If the result indicates there is an error, fix the error and output the code again. + DEFAULT_SYSTEM_MESSAGE = """You are a coding agent. You suggest python code for a user to execute for a given task. Don't suggest shell command. Output the code in a coding block. Check the execution result. The execution result is in the following format: +exitcode: + + If the result indicates there is an error, fix the error and output the code again. If you don't intend for the user to execute the code, do not use a coding block. """ DEFAULT_CONFIG = { "model": DEFAULT_MODEL, } - EXECUTION_AGENT_PREFIX = "execution_agent4" - SUCCESS_EXIT_CODE = "exitcode: 0\n" + # EXECUTION_AGENT_PREFIX = "execution_agent4" + # SUCCESS_EXIT_CODE = "exitcode: 0\n" def __init__(self, name, system_message=DEFAULT_SYSTEM_MESSAGE, work_dir=None, **config): super().__init__(name, system_message) @@ -28,26 +31,27 @@ def receive(self, message, sender): self._sender_dict[sender.name] = sender self._conversations[sender.name] = [{"content": self._system_message, "role": "system"}] super().receive(message, sender) - if sender.name.startswith(self.EXECUTION_AGENT_PREFIX) and message.startswith(self.SUCCESS_EXIT_CODE): - # the code is correct, respond to the original sender - name = sender.name[len(self.EXECUTION_AGENT_PREFIX) :] - original_sender = self._sender_dict[name] - output = message[len(self.SUCCESS_EXIT_CODE) :] - if output: - self._send(f"{output}", original_sender) - else: - self._send("Done. No output.", original_sender) - return + # if sender.name.startswith(self.EXECUTION_AGENT_PREFIX) and message.startswith(self.SUCCESS_EXIT_CODE): + # # the code is correct, respond to the original sender + # name = sender.name[len(self.EXECUTION_AGENT_PREFIX) :] + # original_sender = self._sender_dict[name] + # output = message[len(self.SUCCESS_EXIT_CODE) :] + # if output: + # self._send(f"{output}", original_sender) + # else: + # self._send("Done. No output.", original_sender) + # return responses = oai.ChatCompletion.create(messages=self._conversations[sender.name], **self._config) # cost = oai.ChatCompletion.cost(responses) response = oai.ChatCompletion.extract_text(responses)[0] - if sender.name.startswith(self.EXECUTION_AGENT_PREFIX): - execution_agent = sender - else: - # create an execution agent - execution_agent = ExecutionAgent(f"{self.EXECUTION_AGENT_PREFIX}{sender.name}", work_dir=self._work_dir) - # initialize the conversation - self._conversations[execution_agent.name] = self._conversations[sender.name].copy() - self._sender_dict[execution_agent.name] = execution_agent - # send the response to the execution agent - self._send(response, execution_agent) + # if sender.name.startswith(self.EXECUTION_AGENT_PREFIX): + # execution_agent = sender + # else: + # # create an execution agent + # execution_agent = ExecutionAgent(f"{self.EXECUTION_AGENT_PREFIX}{sender.name}", work_dir=self._work_dir) + # # initialize the conversation + # self._conversations[execution_agent.name] = self._conversations[sender.name].copy() + # self._sender_dict[execution_agent.name] = execution_agent + # # send the response to the execution agent + # self._send(response, execution_agent) + self._send(response, sender) diff --git a/flaml/autogen/agent/human_agent.py b/flaml/autogen/agent/human_agent.py index f560b20ca0..c4a0969822 100644 --- a/flaml/autogen/agent/human_agent.py +++ b/flaml/autogen/agent/human_agent.py @@ -8,9 +8,10 @@ class HumanAgent(Agent): DEFAULT_SYSTEM_MESSAGE = """You are human agent. You can execute_code or give feedback to the sender. """ - def __init__(self, name, system_message="", work_dir=None, **context): + def __init__(self, name, system_message="", work_dir=None, interactive_mode=True, **context): super().__init__(name, system_message) self._work_dir = work_dir + self._interactive_mode = interactive_mode self._context = context def _is_termination_msg(self, message): @@ -29,15 +30,13 @@ def receive(self, message, sender): # no code block is found, lang should be "unknown" if lang == "unknown": # to determine if the message is a termination message using a function - terminate = self._is_termination_msg(message) - print("Please give feedback to the sender (press enter to skip): ") - feedback = input() + feedback = ( + input("Please give feedback to the sender (press enter to skip): ") if self._interactive_mode else "" + ) if feedback: self._send(feedback, sender) - elif terminate: + elif self._is_termination_msg(message): return - else: - self._send("Continue", sender) else: if lang == "bash": assert code.startswith("python ") diff --git a/test/autogen/test_agent.py b/test/autogen/test_agent.py index 921cd0c63c..083dc4502d 100644 --- a/test/autogen/test_agent.py +++ b/test/autogen/test_agent.py @@ -6,18 +6,18 @@ def test_extract_code(): print(extract_code("```bash\npython temp.py\n```")) -def test_coding_agent(): +def test_coding_agent(interactive_mode=False): try: import openai except ImportError: return from flaml.autogen.agent.coding_agent import PythonAgent - from flaml.autogen.agent.agent import Agent + from flaml.autogen.agent.human_agent import HumanAgent conversations = {} oai.ChatCompletion.start_logging(conversations) agent = PythonAgent("coding_agent") - user = Agent("user") + user = HumanAgent("user", interactive_mode=interactive_mode) agent.receive( """Create a temp.py file with the following content: ``` @@ -32,13 +32,13 @@ def test_coding_agent(): oai.ChatCompletion.stop_logging() -def test_tsp(): +def test_tsp(interactive_mode=False): try: import openai except ImportError: return from flaml.autogen.agent.coding_agent import PythonAgent - from flaml.autogen.agent.agent import Agent + from flaml.autogen.agent.human_agent import HumanAgent hard_questions = [ "What if we must go from node 1 to node 2?", @@ -47,8 +47,8 @@ def test_tsp(): ] oai.ChatCompletion.start_logging() - agent = PythonAgent("coding_agent", work_dir="test/autogen", temperature=0) - user = Agent("user") + agent = PythonAgent("coding_agent", temperature=0) + user = HumanAgent("user", work_dir="test/autogen", interactive_mode=interactive_mode) with open("test/autogen/tsp_prompt.txt", "r") as f: prompt = f.read() # agent.receive(prompt.format(question=hard_questions[0]), user) @@ -68,5 +68,5 @@ def test_tsp(): # openai.api_version = "2023-03-15-preview" # change if necessary # openai.api_key = "" # test_extract_code() - test_coding_agent() - test_tsp() + test_coding_agent(interactive_mode=True) + test_tsp(interactive_mode=True) From 3e6063619a15b49aab0310081817aec9d7d50813 Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Sun, 7 May 2023 01:49:33 +0000 Subject: [PATCH 06/32] import check --- test/autogen/test_human_agent.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/autogen/test_human_agent.py b/test/autogen/test_human_agent.py index 40d154bc98..ff60285b7d 100644 --- a/test/autogen/test_human_agent.py +++ b/test/autogen/test_human_agent.py @@ -2,6 +2,10 @@ def test_human_agent(): + try: + import openai + except ImportError: + return from flaml.autogen.agent.chat_agent import ChatAgent from flaml.autogen.agent.human_agent import HumanAgent From 76e393a425c12e5dac7dd18c066da675d0f34aca Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Sun, 7 May 2023 18:30:21 +0000 Subject: [PATCH 07/32] terminate condition --- flaml/autogen/agent/agent.py | 3 ++- flaml/autogen/agent/coding_agent.py | 12 ++++++--- flaml/autogen/agent/human_agent.py | 39 ++++++++++++++++------------- test/autogen/test_agent.py | 21 +++++++++++++--- 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/flaml/autogen/agent/agent.py b/flaml/autogen/agent/agent.py index 39e7501374..55b261c52c 100644 --- a/flaml/autogen/agent/agent.py +++ b/flaml/autogen/agent/agent.py @@ -31,7 +31,8 @@ def _send(self, message, recipient): def _receive(self, message, sender): """Receive a message from another agent.""" - print(self.name, "received message from", sender.name, ":", message) + print(self.name, "received message from", sender.name, ":") + print(message) self._conversations[sender.name].append({"content": message, "role": "user"}) def receive(self, message, sender): diff --git a/flaml/autogen/agent/coding_agent.py b/flaml/autogen/agent/coding_agent.py index a666cd3de0..4c332e5b52 100644 --- a/flaml/autogen/agent/coding_agent.py +++ b/flaml/autogen/agent/coding_agent.py @@ -1,16 +1,16 @@ from .agent import Agent -from .execution_agent import ExecutionAgent -from flaml.autogen.code_utils import generate_code, DEFAULT_MODEL +from flaml.autogen.code_utils import DEFAULT_MODEL from flaml import oai class PythonAgent(Agent): """(Experimental) Suggest code blocks.""" - DEFAULT_SYSTEM_MESSAGE = """You are a coding agent. You suggest python code for a user to execute for a given task. Don't suggest shell command. Output the code in a coding block. Check the execution result. The execution result is in the following format: + DEFAULT_SYSTEM_MESSAGE = """You suggest python code (in a python coding block) for a user to execute for a given task. Don't suggest shell command. Use 'print' function for the output when relevant. Check the execution result. The execution result is in the following format: exitcode: - If the result indicates there is an error, fix the error and output the code again. If you don't intend for the user to execute the code, do not use a coding block. + If the result indicates there is an error, fix the error and output the code again. + Reply "TERMINATE" in the end when the task is done. """ DEFAULT_CONFIG = { @@ -55,3 +55,7 @@ def receive(self, message, sender): # # send the response to the execution agent # self._send(response, execution_agent) self._send(response, sender) + + def reset(self): + self._sender_dict.clear() + self._conversations.clear() diff --git a/flaml/autogen/agent/human_agent.py b/flaml/autogen/agent/human_agent.py index c4a0969822..3ccbf5943e 100644 --- a/flaml/autogen/agent/human_agent.py +++ b/flaml/autogen/agent/human_agent.py @@ -8,38 +8,41 @@ class HumanAgent(Agent): DEFAULT_SYSTEM_MESSAGE = """You are human agent. You can execute_code or give feedback to the sender. """ - def __init__(self, name, system_message="", work_dir=None, interactive_mode=True, **context): + def __init__( + self, name, system_message="", work_dir=None, interactive_mode=True, is_termination_msg=None, **config + ): super().__init__(name, system_message) self._work_dir = work_dir self._interactive_mode = interactive_mode - self._context = context - - def _is_termination_msg(self, message): - """Check if the message is a termination message.""" - if "_is_termination_msg" in self._context: - return self._context["_is_termination_msg"](message) - return True + self._is_termination_msg = ( + is_termination_msg if is_termination_msg is not None else (lambda x: x == "TERMINATE") + ) + self._config = config def receive(self, message, sender): """Receive a message from the sender agent. Every time a message is received, the human agent will give feedback. """ super().receive(message, sender) + # to determine if the message is a termination message using a function + terminate = self._is_termination_msg(message) + feedback = ( + input("Please give feedback to the sender (press enter to skip): ") + if self._interactive_mode == "ALWAYS" or terminate and self._interactive_mode == "TERMINATE" + else "" + ) + if feedback: + self._send(feedback, sender) + elif terminate: + return # try to execute the code code, lang = extract_code(message) - # no code block is found, lang should be "unknown" if lang == "unknown": - # to determine if the message is a termination message using a function - feedback = ( - input("Please give feedback to the sender (press enter to skip): ") if self._interactive_mode else "" - ) - if feedback: - self._send(feedback, sender) - elif self._is_termination_msg(message): - return + # no code block is found, lang should be "unknown" + self._send(feedback, sender) else: if lang == "bash": - assert code.startswith("python ") + assert code.startswith("python "), code file_name = code[len("python ") :] exitcode, logs = execute_code(filename=file_name, work_dir=self._work_dir) elif lang == "python": diff --git a/test/autogen/test_agent.py b/test/autogen/test_agent.py index 083dc4502d..a213aae0af 100644 --- a/test/autogen/test_agent.py +++ b/test/autogen/test_agent.py @@ -16,8 +16,21 @@ def test_coding_agent(interactive_mode=False): conversations = {} oai.ChatCompletion.start_logging(conversations) - agent = PythonAgent("coding_agent") - user = HumanAgent("user", interactive_mode=interactive_mode) + agent = PythonAgent("coding_agent", request_timeout=600, seed=42) + user = HumanAgent( + "user", interactive_mode=interactive_mode, is_termination_msg=lambda x: x.rstrip().endswith("TERMINATE") + ) + # agent.receive("""Find $a+b+c$, given that $x+y\\neq -1$ and \\begin{align*} + # ax+by+c&=x+7,\\\\ + # a+bx+cy&=2x+6y,\\\\ + # ay+b+cx&=4x+y. + # \end{align*} + # Solve the problem smartly.""", user) + # agent.reset() + # agent.receive("""Let $a_1,a_2,a_3,\\dots$ be an arithmetic sequence. If $\\frac{a_4}{a_2} = 3$, what is $\\frac{a_5}{a_3}$? Solve the problem smartly.""", user) + # agent.reset() + # agent.receive("""The product of the first and the third terms of an arithmetic sequence is $5$. If all terms of the sequence are positive integers, what is the fourth term? Solve the problem smartly.""", user) + agent.reset() agent.receive( """Create a temp.py file with the following content: ``` @@ -68,5 +81,5 @@ def test_tsp(interactive_mode=False): # openai.api_version = "2023-03-15-preview" # change if necessary # openai.api_key = "" # test_extract_code() - test_coding_agent(interactive_mode=True) - test_tsp(interactive_mode=True) + test_coding_agent(interactive_mode="TERMINATE") + # test_tsp(interactive_mode=True) From d752d94b6b2a7ab05a25e109a3766f26124a18f9 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Sun, 7 May 2023 23:33:49 -0400 Subject: [PATCH 08/32] rename --- flaml/autogen/agent/chat_agent.py | 2 +- flaml/autogen/agent/human_agent.py | 4 ++-- test/autogen/test_agent.py | 8 ++++---- .../{test_human_agent.py => test_human_proxy_agent.py} | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) rename test/autogen/{test_human_agent.py => test_human_proxy_agent.py} (91%) diff --git a/flaml/autogen/agent/chat_agent.py b/flaml/autogen/agent/chat_agent.py index 1efad5228b..a5d813b733 100644 --- a/flaml/autogen/agent/chat_agent.py +++ b/flaml/autogen/agent/chat_agent.py @@ -4,7 +4,7 @@ class ChatAgent(Agent): - """Chat.""" + """(Experimental) Chat.""" DEFAULT_SYSTEM_MESSAGE = """You are a chat agent. """ diff --git a/flaml/autogen/agent/human_agent.py b/flaml/autogen/agent/human_agent.py index 3ccbf5943e..963aa5e8ce 100644 --- a/flaml/autogen/agent/human_agent.py +++ b/flaml/autogen/agent/human_agent.py @@ -2,8 +2,8 @@ from flaml.autogen.code_utils import extract_code, execute_code -class HumanAgent(Agent): - """A proxy agent for human, that can execute code and provide feedback to the other agents.""" +class HumanProxyAgent(Agent): + """(Experimental) A proxy agent for human, that can execute code and provide feedback to the other agents.""" DEFAULT_SYSTEM_MESSAGE = """You are human agent. You can execute_code or give feedback to the sender. """ diff --git a/test/autogen/test_agent.py b/test/autogen/test_agent.py index a213aae0af..e6e9328d71 100644 --- a/test/autogen/test_agent.py +++ b/test/autogen/test_agent.py @@ -12,12 +12,12 @@ def test_coding_agent(interactive_mode=False): except ImportError: return from flaml.autogen.agent.coding_agent import PythonAgent - from flaml.autogen.agent.human_agent import HumanAgent + from flaml.autogen.agent.human_agent import HumanProxyAgent conversations = {} oai.ChatCompletion.start_logging(conversations) agent = PythonAgent("coding_agent", request_timeout=600, seed=42) - user = HumanAgent( + user = HumanProxyAgent( "user", interactive_mode=interactive_mode, is_termination_msg=lambda x: x.rstrip().endswith("TERMINATE") ) # agent.receive("""Find $a+b+c$, given that $x+y\\neq -1$ and \\begin{align*} @@ -51,7 +51,7 @@ def test_tsp(interactive_mode=False): except ImportError: return from flaml.autogen.agent.coding_agent import PythonAgent - from flaml.autogen.agent.human_agent import HumanAgent + from flaml.autogen.agent.human_agent import HumanProxyAgent hard_questions = [ "What if we must go from node 1 to node 2?", @@ -61,7 +61,7 @@ def test_tsp(interactive_mode=False): oai.ChatCompletion.start_logging() agent = PythonAgent("coding_agent", temperature=0) - user = HumanAgent("user", work_dir="test/autogen", interactive_mode=interactive_mode) + user = HumanProxyAgent("user", work_dir="test/autogen", interactive_mode=interactive_mode) with open("test/autogen/tsp_prompt.txt", "r") as f: prompt = f.read() # agent.receive(prompt.format(question=hard_questions[0]), user) diff --git a/test/autogen/test_human_agent.py b/test/autogen/test_human_proxy_agent.py similarity index 91% rename from test/autogen/test_human_agent.py rename to test/autogen/test_human_proxy_agent.py index ff60285b7d..9c463c6825 100644 --- a/test/autogen/test_human_agent.py +++ b/test/autogen/test_human_proxy_agent.py @@ -7,12 +7,12 @@ def test_human_agent(): except ImportError: return from flaml.autogen.agent.chat_agent import ChatAgent - from flaml.autogen.agent.human_agent import HumanAgent + from flaml.autogen.agent.human_agent import HumanProxyAgent conversations = {} oai.ChatCompletion.start_logging(conversations) agent = ChatAgent("chat_agent") - user = HumanAgent("human_user") + user = HumanProxyAgent("human_user") agent.receive( """Write python code to solve the equation x^3=125. You must write code in the following format. You must always print the result. Wait for me to return the result. From 210efffdb72ac0bd8dd45bdf96ff50fb969c0367 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Mon, 8 May 2023 10:18:48 -0400 Subject: [PATCH 09/32] add docstr --- flaml/autogen/agent/agent.py | 5 +++++ flaml/autogen/agent/chat_agent.py | 7 +++++++ flaml/autogen/agent/coding_agent.py | 7 +++++++ .../{human_agent.py => human_proxy_agent.py} | 20 ++++++++++++++++--- test/autogen/test_agent.py | 16 +++++++-------- test/autogen/test_human_proxy_agent.py | 2 +- 6 files changed, 45 insertions(+), 12 deletions(-) rename flaml/autogen/agent/{human_agent.py => human_proxy_agent.py} (62%) diff --git a/flaml/autogen/agent/agent.py b/flaml/autogen/agent/agent.py index 55b261c52c..56e75ce9c4 100644 --- a/flaml/autogen/agent/agent.py +++ b/flaml/autogen/agent/agent.py @@ -8,6 +8,11 @@ class Agent: """ def __init__(self, name, system_message=""): + """ + Args: + name (str): name of the agent + system_message (str): system message to be sent to the agent + """ # empty memory self._memory = [] # a dictionary of conversations, default value is list diff --git a/flaml/autogen/agent/chat_agent.py b/flaml/autogen/agent/chat_agent.py index a5d813b733..a6d487a711 100644 --- a/flaml/autogen/agent/chat_agent.py +++ b/flaml/autogen/agent/chat_agent.py @@ -14,6 +14,13 @@ class ChatAgent(Agent): } def __init__(self, name, system_message=DEFAULT_SYSTEM_MESSAGE, work_dir=None, **config): + """ + Args: + name (str): agent name + system_message (str): system message to be sent to the agent + work_dir (str): working directory for the agent to execute code + config (dict): other configurations. + """ super().__init__(name, system_message) self._work_dir = work_dir self._config = self.DEFAULT_CONFIG.copy() diff --git a/flaml/autogen/agent/coding_agent.py b/flaml/autogen/agent/coding_agent.py index 4c332e5b52..1246cb25bc 100644 --- a/flaml/autogen/agent/coding_agent.py +++ b/flaml/autogen/agent/coding_agent.py @@ -20,6 +20,13 @@ class PythonAgent(Agent): # SUCCESS_EXIT_CODE = "exitcode: 0\n" def __init__(self, name, system_message=DEFAULT_SYSTEM_MESSAGE, work_dir=None, **config): + """ + Args: + name (str): agent name + system_message (str): system message to be sent to the agent + work_dir (str): working directory for the agent to execute code + config (dict): other configurations. + """ super().__init__(name, system_message) self._work_dir = work_dir self._config = self.DEFAULT_CONFIG.copy() diff --git a/flaml/autogen/agent/human_agent.py b/flaml/autogen/agent/human_proxy_agent.py similarity index 62% rename from flaml/autogen/agent/human_agent.py rename to flaml/autogen/agent/human_proxy_agent.py index 963aa5e8ce..2a02fcf6e2 100644 --- a/flaml/autogen/agent/human_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -9,11 +9,25 @@ class HumanProxyAgent(Agent): """ def __init__( - self, name, system_message="", work_dir=None, interactive_mode=True, is_termination_msg=None, **config + self, name, system_message="", work_dir=None, human_input_mode="ALWAYS", is_termination_msg=None, **config ): + """ + Args: + name (str): name of the agent + system_message (str): system message to be sent to the agent + work_dir (str): working directory for the agent to execute code + human_input_mode (bool): whether to ask for human inputs every time a message is received. + Possible values are "ALWAYS", "TERMINATE", "NEVER". + When "ALWAYS", the agent will ask for human input every time a message is received. + When "TERMINATE", the agent will ask for human input only when a termination message is received. + When "NEVER", the agent will never ask for human input. + is_termination_msg (function): a function that takes a message and returns a boolean value. + This function is used to determine if a received message is a termination message. + config (dict): other configurations. + """ super().__init__(name, system_message) self._work_dir = work_dir - self._interactive_mode = interactive_mode + self._human_input_mode = human_input_mode self._is_termination_msg = ( is_termination_msg if is_termination_msg is not None else (lambda x: x == "TERMINATE") ) @@ -28,7 +42,7 @@ def receive(self, message, sender): terminate = self._is_termination_msg(message) feedback = ( input("Please give feedback to the sender (press enter to skip): ") - if self._interactive_mode == "ALWAYS" or terminate and self._interactive_mode == "TERMINATE" + if self._human_input_mode == "ALWAYS" or terminate and self._human_input_mode == "TERMINATE" else "" ) if feedback: diff --git a/test/autogen/test_agent.py b/test/autogen/test_agent.py index e6e9328d71..c716764e39 100644 --- a/test/autogen/test_agent.py +++ b/test/autogen/test_agent.py @@ -6,19 +6,19 @@ def test_extract_code(): print(extract_code("```bash\npython temp.py\n```")) -def test_coding_agent(interactive_mode=False): +def test_coding_agent(human_input_mode="NEVER"): try: import openai except ImportError: return from flaml.autogen.agent.coding_agent import PythonAgent - from flaml.autogen.agent.human_agent import HumanProxyAgent + from flaml.autogen.agent.human_proxy_agent import HumanProxyAgent conversations = {} oai.ChatCompletion.start_logging(conversations) agent = PythonAgent("coding_agent", request_timeout=600, seed=42) user = HumanProxyAgent( - "user", interactive_mode=interactive_mode, is_termination_msg=lambda x: x.rstrip().endswith("TERMINATE") + "user", human_input_mode=human_input_mode, is_termination_msg=lambda x: x.rstrip().endswith("TERMINATE") ) # agent.receive("""Find $a+b+c$, given that $x+y\\neq -1$ and \\begin{align*} # ax+by+c&=x+7,\\\\ @@ -45,13 +45,13 @@ def test_coding_agent(interactive_mode=False): oai.ChatCompletion.stop_logging() -def test_tsp(interactive_mode=False): +def test_tsp(human_input_mode="NEVER"): try: import openai except ImportError: return from flaml.autogen.agent.coding_agent import PythonAgent - from flaml.autogen.agent.human_agent import HumanProxyAgent + from flaml.autogen.agent.human_proxy_agent import HumanProxyAgent hard_questions = [ "What if we must go from node 1 to node 2?", @@ -61,7 +61,7 @@ def test_tsp(interactive_mode=False): oai.ChatCompletion.start_logging() agent = PythonAgent("coding_agent", temperature=0) - user = HumanProxyAgent("user", work_dir="test/autogen", interactive_mode=interactive_mode) + user = HumanProxyAgent("user", work_dir="test/autogen", human_input_mode=human_input_mode) with open("test/autogen/tsp_prompt.txt", "r") as f: prompt = f.read() # agent.receive(prompt.format(question=hard_questions[0]), user) @@ -81,5 +81,5 @@ def test_tsp(interactive_mode=False): # openai.api_version = "2023-03-15-preview" # change if necessary # openai.api_key = "" # test_extract_code() - test_coding_agent(interactive_mode="TERMINATE") - # test_tsp(interactive_mode=True) + test_coding_agent(human_input_mode="TERMINATE") + # test_tsp(human_input_mode="ALWAYS") diff --git a/test/autogen/test_human_proxy_agent.py b/test/autogen/test_human_proxy_agent.py index 9c463c6825..bb7c3ef52b 100644 --- a/test/autogen/test_human_proxy_agent.py +++ b/test/autogen/test_human_proxy_agent.py @@ -7,7 +7,7 @@ def test_human_agent(): except ImportError: return from flaml.autogen.agent.chat_agent import ChatAgent - from flaml.autogen.agent.human_agent import HumanProxyAgent + from flaml.autogen.agent.human_proxy_agent import HumanProxyAgent conversations = {} oai.ChatCompletion.start_logging(conversations) From 464193cb2c6b0ceee25eebcd0c380a962c8c258b Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Mon, 8 May 2023 10:30:11 -0400 Subject: [PATCH 10/32] exitcode to str --- flaml/autogen/agent/human_proxy_agent.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index 2a02fcf6e2..bdcced6cb2 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -63,5 +63,7 @@ def receive(self, message, sender): exitcode, logs = execute_code(code, work_dir=self._work_dir) else: # TODO: could this happen? + exitcode = 1 raise NotImplementedError - self._send(f"exitcode: {exitcode}\n{logs.decode('utf-8')}", sender) + exitcode2str = "execution succeeded" if exitcode == 0 else "execution failed" + self._send(f"exitcode: {exitcode} ({exitcode2str})\nCode output: {logs.decode('utf-8')}", sender) From 04c3235ae9ac1965e44546592be4c9b2f38f6c23 Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Mon, 8 May 2023 16:18:58 +0000 Subject: [PATCH 11/32] print --- flaml/autogen/agent/agent.py | 2 +- flaml/autogen/agent/coding_agent.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/flaml/autogen/agent/agent.py b/flaml/autogen/agent/agent.py index 55b261c52c..d1515cf9cf 100644 --- a/flaml/autogen/agent/agent.py +++ b/flaml/autogen/agent/agent.py @@ -31,7 +31,7 @@ def _send(self, message, recipient): def _receive(self, message, sender): """Receive a message from another agent.""" - print(self.name, "received message from", sender.name, ":") + print("****", self.name, "received message from", sender.name, "****") print(message) self._conversations[sender.name].append({"content": message, "role": "user"}) diff --git a/flaml/autogen/agent/coding_agent.py b/flaml/autogen/agent/coding_agent.py index 4c332e5b52..6048950273 100644 --- a/flaml/autogen/agent/coding_agent.py +++ b/flaml/autogen/agent/coding_agent.py @@ -6,9 +6,7 @@ class PythonAgent(Agent): """(Experimental) Suggest code blocks.""" - DEFAULT_SYSTEM_MESSAGE = """You suggest python code (in a python coding block) for a user to execute for a given task. Don't suggest shell command. Use 'print' function for the output when relevant. Check the execution result. The execution result is in the following format: -exitcode: - + DEFAULT_SYSTEM_MESSAGE = """You suggest python code (in a python coding block) for a user to execute for a given task. Finish the task smartly. Don't suggest shell command. Use 'print' function for the output when relevant. Check the execution result returned by the user. If the result indicates there is an error, fix the error and output the code again. Reply "TERMINATE" in the end when the task is done. """ From 16788230b945af802328d66fa4f1701e27138ff1 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Mon, 8 May 2023 16:48:17 +0000 Subject: [PATCH 12/32] feedback --- flaml/autogen/agent/human_proxy_agent.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index bdcced6cb2..c1a334d1bd 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -41,14 +41,19 @@ def receive(self, message, sender): # to determine if the message is a termination message using a function terminate = self._is_termination_msg(message) feedback = ( - input("Please give feedback to the sender (press enter to skip): ") + input("Please give feedback to the sender (press enter to skip and type exit to exit): ") if self._human_input_mode == "ALWAYS" or terminate and self._human_input_mode == "TERMINATE" else "" ) - if feedback: - self._send(feedback, sender) - elif terminate: + print("feedback: ", feedback, "\n") + if terminate or feedback == "exit": return + elif feedback: + self._send(feedback, sender) + # if feedback: + # self._send(feedback, sender) + # elif terminate: + # return # try to execute the code code, lang = extract_code(message) if lang == "unknown": From 4e7940eed7a439a5252b7b9964ac38755285a1cd Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Tue, 9 May 2023 02:09:32 +0000 Subject: [PATCH 13/32] save and execute code --- flaml/autogen/agent/coding_agent.py | 2 +- flaml/autogen/agent/human_proxy_agent.py | 6 +++++- test/autogen/test_agent.py | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/flaml/autogen/agent/coding_agent.py b/flaml/autogen/agent/coding_agent.py index afdf47afce..e6f3632db6 100644 --- a/flaml/autogen/agent/coding_agent.py +++ b/flaml/autogen/agent/coding_agent.py @@ -6,7 +6,7 @@ class PythonAgent(Agent): """(Experimental) Suggest code blocks.""" - DEFAULT_SYSTEM_MESSAGE = """You suggest python code (in a python coding block) for a user to execute for a given task. Finish the task smartly. Don't suggest shell command. Use 'print' function for the output when relevant. Check the execution result returned by the user. + DEFAULT_SYSTEM_MESSAGE = """You suggest python code (in a python coding block) for a user to execute for a given task. If you want the user to save the code in a file before executing it, put # filename: inside the code block as the first line. Finish the task smartly. Don't suggest shell command. Don't include multiple code blocks in one response. Use 'print' function for the output when relevant. Check the execution result returned by the user. If the result indicates there is an error, fix the error and output the code again. Reply "TERMINATE" in the end when the task is done. """ diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index bdcced6cb2..ea9ad48b27 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -60,7 +60,11 @@ def receive(self, message, sender): file_name = code[len("python ") :] exitcode, logs = execute_code(filename=file_name, work_dir=self._work_dir) elif lang == "python": - exitcode, logs = execute_code(code, work_dir=self._work_dir) + if code.startswith("# filename: "): + filename = code[11 : code.find("\n")].strip() + else: + filename = None + exitcode, logs = execute_code(code, work_dir=self._work_dir, filename=filename) else: # TODO: could this happen? exitcode = 1 diff --git a/test/autogen/test_agent.py b/test/autogen/test_agent.py index c716764e39..a1a51ab35a 100644 --- a/test/autogen/test_agent.py +++ b/test/autogen/test_agent.py @@ -30,6 +30,10 @@ def test_coding_agent(human_input_mode="NEVER"): # agent.receive("""Let $a_1,a_2,a_3,\\dots$ be an arithmetic sequence. If $\\frac{a_4}{a_2} = 3$, what is $\\frac{a_5}{a_3}$? Solve the problem smartly.""", user) # agent.reset() # agent.receive("""The product of the first and the third terms of an arithmetic sequence is $5$. If all terms of the sequence are positive integers, what is the fourth term? Solve the problem smartly.""", user) + agent.receive( + """Create and execute a script to plot a rocket""", + user, + ) agent.reset() agent.receive( """Create a temp.py file with the following content: From 1294c81146e5898dd9d6b3fcdb68346120c6d018 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Mon, 8 May 2023 22:40:37 -0400 Subject: [PATCH 14/32] add max_turn_num --- flaml/autogen/agent/human_proxy_agent.py | 34 +++++++++++++++++++++--- test/autogen/test_human_proxy_agent.py | 2 +- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index bdcced6cb2..2ff5e9c09e 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -7,9 +7,17 @@ class HumanProxyAgent(Agent): DEFAULT_SYSTEM_MESSAGE = """You are human agent. You can execute_code or give feedback to the sender. """ + MAX_TURN_NUM = 100 # maximum number of turns in one conversation session (subject to future change) def __init__( - self, name, system_message="", work_dir=None, human_input_mode="ALWAYS", is_termination_msg=None, **config + self, + name, + system_message="", + work_dir=None, + human_input_mode="ALWAYS", + max_turn_num=None, + is_termination_msg=None, + **config, ): """ Args: @@ -21,9 +29,15 @@ def __init__( When "ALWAYS", the agent will ask for human input every time a message is received. When "TERMINATE", the agent will ask for human input only when a termination message is received. When "NEVER", the agent will never ask for human input. + max_turn_num (int): the maximum number of turns in one conversation session. + default: None (no limit provided, class attribute MAX_TURN_NUM will be used as the limit). + The limit only plays a role when human_input_mode is not "ALWAYS". is_termination_msg (function): a function that takes a message and returns a boolean value. This function is used to determine if a received message is a termination message. config (dict): other configurations. + + The conversation stops when a termination message is received or the number of turns larger than + the provided max_turn_num or the human input is "exit". """ super().__init__(name, system_message) self._work_dir = work_dir @@ -32,22 +46,34 @@ def __init__( is_termination_msg if is_termination_msg is not None else (lambda x: x == "TERMINATE") ) self._config = config + self._max_turn_num = max_turn_num if max_turn_num is not None else self.MAX_TURN_NUM + self._conversation_turn_counter = {} def receive(self, message, sender): """Receive a message from the sender agent. Every time a message is received, the human agent will give feedback. + The conversation stops when a termination message is received or the number of turns larger than + the provided max_turn_num or the human input is "exit". """ super().receive(message, sender) # to determine if the message is a termination message using a function terminate = self._is_termination_msg(message) feedback = ( - input("Please give feedback to the sender (press enter to skip): ") + input("Please give feedback to the sender (press enter to skip, type exit to stop the conversation): ") if self._human_input_mode == "ALWAYS" or terminate and self._human_input_mode == "TERMINATE" else "" ) - if feedback: + if feedback and feedback != "exit": self._send(feedback, sender) - elif terminate: + elif ( + terminate + or feedback == "exit" + or ( + self._human_input_mode != "ALWAYS" + and (len(self._conversations[sender.name]) + 1) / 2 >= self._max_turn_num + ) + ): + # note that len(self._conversations[sender.name])+1)/2 is the number of turns in the conversation return # try to execute the code code, lang = extract_code(message) diff --git a/test/autogen/test_human_proxy_agent.py b/test/autogen/test_human_proxy_agent.py index bb7c3ef52b..507311bd83 100644 --- a/test/autogen/test_human_proxy_agent.py +++ b/test/autogen/test_human_proxy_agent.py @@ -12,7 +12,7 @@ def test_human_agent(): conversations = {} oai.ChatCompletion.start_logging(conversations) agent = ChatAgent("chat_agent") - user = HumanProxyAgent("human_user") + user = HumanProxyAgent("human_user", human_input_mode="NEVER", max_turn_num=2) agent.receive( """Write python code to solve the equation x^3=125. You must write code in the following format. You must always print the result. Wait for me to return the result. From a32a4b3bb3142a89dc54299d60da95f47de1e679 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Mon, 8 May 2023 22:57:51 -0400 Subject: [PATCH 15/32] add max_turn_num in test_agent.py --- test/autogen/test_agent.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test/autogen/test_agent.py b/test/autogen/test_agent.py index a1a51ab35a..cfecf73b6b 100644 --- a/test/autogen/test_agent.py +++ b/test/autogen/test_agent.py @@ -6,7 +6,7 @@ def test_extract_code(): print(extract_code("```bash\npython temp.py\n```")) -def test_coding_agent(human_input_mode="NEVER"): +def test_coding_agent(human_input_mode="NEVER", max_turn_num=10): try: import openai except ImportError: @@ -18,7 +18,10 @@ def test_coding_agent(human_input_mode="NEVER"): oai.ChatCompletion.start_logging(conversations) agent = PythonAgent("coding_agent", request_timeout=600, seed=42) user = HumanProxyAgent( - "user", human_input_mode=human_input_mode, is_termination_msg=lambda x: x.rstrip().endswith("TERMINATE") + "user", + human_input_mode=human_input_mode, + max_turn_num=max_turn_num, + is_termination_msg=lambda x: x.rstrip().endswith("TERMINATE"), ) # agent.receive("""Find $a+b+c$, given that $x+y\\neq -1$ and \\begin{align*} # ax+by+c&=x+7,\\\\ @@ -49,7 +52,7 @@ def test_coding_agent(human_input_mode="NEVER"): oai.ChatCompletion.stop_logging() -def test_tsp(human_input_mode="NEVER"): +def test_tsp(human_input_mode="NEVER", max_turn_num=10): try: import openai except ImportError: @@ -65,7 +68,9 @@ def test_tsp(human_input_mode="NEVER"): oai.ChatCompletion.start_logging() agent = PythonAgent("coding_agent", temperature=0) - user = HumanProxyAgent("user", work_dir="test/autogen", human_input_mode=human_input_mode) + user = HumanProxyAgent( + "user", work_dir="test/autogen", human_input_mode=human_input_mode, max_turn_num=max_turn_num + ) with open("test/autogen/tsp_prompt.txt", "r") as f: prompt = f.read() # agent.receive(prompt.format(question=hard_questions[0]), user) @@ -86,4 +91,4 @@ def test_tsp(human_input_mode="NEVER"): # openai.api_key = "" # test_extract_code() test_coding_agent(human_input_mode="TERMINATE") - # test_tsp(human_input_mode="ALWAYS") + test_tsp(human_input_mode="NEVER", max_turn_num=3) From 87116ecc61a66e832087f103326d8c6f404918de Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Mon, 8 May 2023 23:01:53 -0400 Subject: [PATCH 16/32] reduce max_turn_num in the test --- test/autogen/test_agent.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/autogen/test_agent.py b/test/autogen/test_agent.py index cfecf73b6b..81e70805fb 100644 --- a/test/autogen/test_agent.py +++ b/test/autogen/test_agent.py @@ -91,4 +91,4 @@ def test_tsp(human_input_mode="NEVER", max_turn_num=10): # openai.api_key = "" # test_extract_code() test_coding_agent(human_input_mode="TERMINATE") - test_tsp(human_input_mode="NEVER", max_turn_num=3) + test_tsp(human_input_mode="NEVER", max_turn_num=2) From 3b3dd60730931c793e88e4b2aa870fa0192db3f5 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Tue, 9 May 2023 00:09:47 -0400 Subject: [PATCH 17/32] change max_turn_num to max_consecutive_auto_reply --- flaml/autogen/agent/human_proxy_agent.py | 33 ++++++++++++++---------- test/autogen/test_agent.py | 13 ++++++---- test/autogen/test_human_proxy_agent.py | 2 +- 3 files changed, 29 insertions(+), 19 deletions(-) diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index ebb360c68c..237330a442 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -1,5 +1,6 @@ from .agent import Agent from flaml.autogen.code_utils import extract_code, execute_code +from collections import defaultdict class HumanProxyAgent(Agent): @@ -7,7 +8,7 @@ class HumanProxyAgent(Agent): DEFAULT_SYSTEM_MESSAGE = """You are human agent. You can execute_code or give feedback to the sender. """ - MAX_TURN_NUM = 100 # maximum number of turns in one conversation session (subject to future change) + MAX_CONSECUTIVE_AUTO_REPLY = 100 # maximum number of consecutive auto replies (subject to future change) def __init__( self, @@ -15,7 +16,7 @@ def __init__( system_message="", work_dir=None, human_input_mode="ALWAYS", - max_turn_num=None, + max_consecutive_auto_reply=None, is_termination_msg=None, **config, ): @@ -29,15 +30,15 @@ def __init__( When "ALWAYS", the agent will ask for human input every time a message is received. When "TERMINATE", the agent will ask for human input only when a termination message is received. When "NEVER", the agent will never ask for human input. - max_turn_num (int): the maximum number of turns in one conversation session. - default: None (no limit provided, class attribute MAX_TURN_NUM will be used as the limit). + max_consecutive_auto_reply (int): the maximum number of consecutive auto replies. + default: None (no limit provided, class attribute MAX_CONSECUTIVE_AUTO_REPLY will be used as the limit in this case). The limit only plays a role when human_input_mode is not "ALWAYS". is_termination_msg (function): a function that takes a message and returns a boolean value. This function is used to determine if a received message is a termination message. config (dict): other configurations. - The conversation stops when a termination message is received or the number of turns larger than - the provided max_turn_num or the human input is "exit". + The conversation stops when the human input is "exit", or no human input is provided and a termination message is received, + or the number of consecutive auto reply is larger than the provided max_consecutive_auto_reply (when human_input_mode is not "ALWAYS"). """ super().__init__(name, system_message) self._work_dir = work_dir @@ -46,14 +47,17 @@ def __init__( is_termination_msg if is_termination_msg is not None else (lambda x: x == "TERMINATE") ) self._config = config - self._max_turn_num = max_turn_num if max_turn_num is not None else self.MAX_TURN_NUM - self._conversation_turn_counter = {} + self._max_consecutive_auto_reply = ( + max_consecutive_auto_reply if max_consecutive_auto_reply is not None else self.MAX_CONSECUTIVE_AUTO_REPLY + ) + self._consecutive_auto_reply_counter = defaultdict(int) def receive(self, message, sender): """Receive a message from the sender agent. Every time a message is received, the human agent will give feedback. - The conversation stops when a termination message is received or the number of turns larger than - the provided max_turn_num or the human input is "exit". + + The conversation stops when the human input is "exit", or no human input is provided and a termination message is received, + or the number of consecutive auto reply is larger than the provided max_consecutive_auto_reply (when human_input_mode is not "ALWAYS"). """ super().receive(message, sender) # to determine if the message is a termination message using a function @@ -63,6 +67,9 @@ def receive(self, message, sender): if self._human_input_mode == "ALWAYS" or terminate and self._human_input_mode == "TERMINATE" else "" ) + # reset the consecutive_auto_reply_counter + if self._human_input_mode != "ALWAYS" and feedback: + self._consecutive_auto_reply_counter[sender.name] = 0 if feedback and feedback != "exit": self._send(feedback, sender) elif ( @@ -70,11 +77,11 @@ def receive(self, message, sender): or feedback == "exit" or ( self._human_input_mode != "ALWAYS" - and (len(self._conversations[sender.name]) + 1) / 2 >= self._max_turn_num + and self._consecutive_auto_reply_counter[sender.name] >= self._max_consecutive_auto_reply ) ): - # note that len(self._conversations[sender.name])+1)/2 is the number of turns in the conversation return + self._consecutive_auto_reply_counter[sender.name] += 1 # try to execute the code code, lang = extract_code(message) if lang == "unknown": @@ -93,7 +100,7 @@ def receive(self, message, sender): exitcode, logs = execute_code(code, work_dir=self._work_dir, filename=filename) else: # TODO: could this happen? - exitcode = 1 + exitcode, logs = 1, "unknown language" raise NotImplementedError exitcode2str = "execution succeeded" if exitcode == 0 else "execution failed" self._send(f"exitcode: {exitcode} ({exitcode2str})\nCode output: {logs.decode('utf-8')}", sender) diff --git a/test/autogen/test_agent.py b/test/autogen/test_agent.py index 81e70805fb..9b3bec5b54 100644 --- a/test/autogen/test_agent.py +++ b/test/autogen/test_agent.py @@ -6,7 +6,7 @@ def test_extract_code(): print(extract_code("```bash\npython temp.py\n```")) -def test_coding_agent(human_input_mode="NEVER", max_turn_num=10): +def test_coding_agent(human_input_mode="NEVER", max_consecutive_auto_reply=10): try: import openai except ImportError: @@ -20,7 +20,7 @@ def test_coding_agent(human_input_mode="NEVER", max_turn_num=10): user = HumanProxyAgent( "user", human_input_mode=human_input_mode, - max_turn_num=max_turn_num, + max_consecutive_auto_reply=max_consecutive_auto_reply, is_termination_msg=lambda x: x.rstrip().endswith("TERMINATE"), ) # agent.receive("""Find $a+b+c$, given that $x+y\\neq -1$ and \\begin{align*} @@ -52,7 +52,7 @@ def test_coding_agent(human_input_mode="NEVER", max_turn_num=10): oai.ChatCompletion.stop_logging() -def test_tsp(human_input_mode="NEVER", max_turn_num=10): +def test_tsp(human_input_mode="NEVER", max_consecutive_auto_reply=10): try: import openai except ImportError: @@ -69,7 +69,10 @@ def test_tsp(human_input_mode="NEVER", max_turn_num=10): oai.ChatCompletion.start_logging() agent = PythonAgent("coding_agent", temperature=0) user = HumanProxyAgent( - "user", work_dir="test/autogen", human_input_mode=human_input_mode, max_turn_num=max_turn_num + "user", + work_dir="test/autogen", + human_input_mode=human_input_mode, + max_consecutive_auto_reply=max_consecutive_auto_reply, ) with open("test/autogen/tsp_prompt.txt", "r") as f: prompt = f.read() @@ -91,4 +94,4 @@ def test_tsp(human_input_mode="NEVER", max_turn_num=10): # openai.api_key = "" # test_extract_code() test_coding_agent(human_input_mode="TERMINATE") - test_tsp(human_input_mode="NEVER", max_turn_num=2) + test_tsp(human_input_mode="NEVER", max_consecutive_auto_reply=2) diff --git a/test/autogen/test_human_proxy_agent.py b/test/autogen/test_human_proxy_agent.py index 507311bd83..335552f8e6 100644 --- a/test/autogen/test_human_proxy_agent.py +++ b/test/autogen/test_human_proxy_agent.py @@ -12,7 +12,7 @@ def test_human_agent(): conversations = {} oai.ChatCompletion.start_logging(conversations) agent = ChatAgent("chat_agent") - user = HumanProxyAgent("human_user", human_input_mode="NEVER", max_turn_num=2) + user = HumanProxyAgent("human_user", human_input_mode="NEVER", max_consecutive_auto_reply=2) agent.receive( """Write python code to solve the equation x^3=125. You must write code in the following format. You must always print the result. Wait for me to return the result. From 417da9c2d35a44426709f7e82be7b1a447ac04a5 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Tue, 9 May 2023 14:27:33 -0400 Subject: [PATCH 18/32] update human proxy agent --- flaml/autogen/agent/human_proxy_agent.py | 91 +++++++++++++++--------- 1 file changed, 56 insertions(+), 35 deletions(-) diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index 237330a442..08e30d26b5 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -52,6 +52,24 @@ def __init__( ) self._consecutive_auto_reply_counter = defaultdict(int) + def _execute_code(self, code, lang): + """Execute the code and return the result.""" + if lang == "bash": + assert code.startswith("python "), code + file_name = code[len("python ") :] + exitcode, logs = execute_code(filename=file_name, work_dir=self._work_dir) + elif lang == "python": + if code.startswith("# filename: "): + filename = code[11 : code.find("\n")].strip() + else: + filename = None + exitcode, logs = execute_code(code, work_dir=self._work_dir, filename=filename) + else: + # TODO: could this happen? + exitcode, logs = 1, "unknown language" + # raise NotImplementedError + return exitcode, logs + def receive(self, message, sender): """Receive a message from the sender agent. Every time a message is received, the human agent will give feedback. @@ -60,47 +78,50 @@ def receive(self, message, sender): or the number of consecutive auto reply is larger than the provided max_consecutive_auto_reply (when human_input_mode is not "ALWAYS"). """ super().receive(message, sender) - # to determine if the message is a termination message using a function - terminate = self._is_termination_msg(message) - feedback = ( - input("Please give feedback to the sender (press enter to skip, type exit to stop the conversation): ") - if self._human_input_mode == "ALWAYS" or terminate and self._human_input_mode == "TERMINATE" - else "" - ) - # reset the consecutive_auto_reply_counter - if self._human_input_mode != "ALWAYS" and feedback: - self._consecutive_auto_reply_counter[sender.name] = 0 - if feedback and feedback != "exit": - self._send(feedback, sender) - elif ( - terminate - or feedback == "exit" - or ( - self._human_input_mode != "ALWAYS" - and self._consecutive_auto_reply_counter[sender.name] >= self._max_consecutive_auto_reply + # default reply is empty (i.e., no reply, in this case we will try to generate auto reply) + reply = "" + if self._human_input_mode == "ALWAYS": + # TODO: if skip and use auto reply, should we also display the auto reply? + code, lang = extract_code(message) + msg2display = ( + f"Code block detected: \n{code}\nUse auto-reply to execute the code and return the result." + if lang != "unknown" + else "" + ) + reply = input( + "*" * 40 + + f"\n{msg2display} \nProvide feedback to the sender. Press enter to skip and use auto-reply, or type 'exit' to end the conversation: " ) + elif self._human_input_mode == "TERMINATE": + if self._consecutive_auto_reply_counter[ + sender.name + ] >= self._max_consecutive_auto_reply or self._is_termination_msg(message): + reply = input( + "Please give feedback to the sender. (Press enter or type 'exit' to stop the conversation): " + ) + reply = reply if reply else "exit" + elif ( + self._human_input_mode == "NEVER" + and self._consecutive_auto_reply_counter[sender.name] >= self._max_consecutive_auto_reply ): + reply = "exit" + + if reply == "exit": + return + elif reply: + # reset the consecutive_auto_reply_counter + self._consecutive_auto_reply_counter[sender.name] = 0 + self._send(reply, sender) return + self._consecutive_auto_reply_counter[sender.name] += 1 - # try to execute the code - code, lang = extract_code(message) + if self._human_input_mode != "ALWAYS": + code, lang = extract_code(message) if lang == "unknown": # no code block is found, lang should be "unknown" - self._send(feedback, sender) + self._send(reply, sender) else: - if lang == "bash": - assert code.startswith("python "), code - file_name = code[len("python ") :] - exitcode, logs = execute_code(filename=file_name, work_dir=self._work_dir) - elif lang == "python": - if code.startswith("# filename: "): - filename = code[11 : code.find("\n")].strip() - else: - filename = None - exitcode, logs = execute_code(code, work_dir=self._work_dir, filename=filename) - else: - # TODO: could this happen? - exitcode, logs = 1, "unknown language" - raise NotImplementedError + # try to execute the code + exitcode, logs = self._execute_code(code, lang) exitcode2str = "execution succeeded" if exitcode == 0 else "execution failed" self._send(f"exitcode: {exitcode} ({exitcode2str})\nCode output: {logs.decode('utf-8')}", sender) From 3e3dab5d9fc7ed73e45218f0c4e3865327a29c93 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Tue, 9 May 2023 14:32:41 -0400 Subject: [PATCH 19/32] remove execution agent and dated docstr --- flaml/autogen/agent/agent.py | 4 ++-- flaml/autogen/agent/execution_agent.py | 24 ------------------------ 2 files changed, 2 insertions(+), 26 deletions(-) delete mode 100644 flaml/autogen/agent/execution_agent.py diff --git a/flaml/autogen/agent/agent.py b/flaml/autogen/agent/agent.py index 117a04551f..2acdbb198a 100644 --- a/flaml/autogen/agent/agent.py +++ b/flaml/autogen/agent/agent.py @@ -3,8 +3,8 @@ class Agent: """(Experimental) An abstract class for AI agent. - An agent can communicate with other agents, human and perform actions. - Different agents can differ in how and who they communicate with, and what actions they can perform. For example, an autonomous agent can communicate with human and other agents, and perform actions by creating agents and sending messages to other agents. A planning agent can communicate with other agents to make a plan and keep track of tasks. An execution agent can only communicate with other agents, and perform actions such as executing a command or code. + An agent can communicate with other agents and perform actions. + Different agents can differ in what actions they perform in the receive method. """ def __init__(self, name, system_message=""): diff --git a/flaml/autogen/agent/execution_agent.py b/flaml/autogen/agent/execution_agent.py deleted file mode 100644 index 4cb9f92deb..0000000000 --- a/flaml/autogen/agent/execution_agent.py +++ /dev/null @@ -1,24 +0,0 @@ -from .agent import Agent -from flaml.autogen.code_utils import execute_code, extract_code - - -class ExecutionAgent(Agent): - """(Experimental) Perform actions based on instructions from other agents. - An execution agent can only communicate with other agents, and perform actions such as executing a command or code. - """ - - def __init__(self, name, system_message="", work_dir=None): - super().__init__(name, system_message) - self._word_dir = work_dir - - def receive(self, message, sender): - super().receive(message, sender) - # extract code - code, lang = extract_code(message) - if lang == "bash": - assert code.startswith("python ") - file_name = code[len("python ") :] - exitcode, logs = execute_code(filename=file_name, work_dir=self._word_dir) - else: - exitcode, logs = execute_code(code, work_dir=self._word_dir) - self._send(f"exitcode: {exitcode}\n{logs.decode('utf-8')}", sender) From 404cf4a0c64d7d21c5998c430ab809d9dc6483f4 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Wed, 10 May 2023 11:31:31 -0400 Subject: [PATCH 20/32] clean doc --- flaml/autogen/agent/agent.py | 2 +- flaml/autogen/agent/coding_agent.py | 23 ------------------- flaml/autogen/agent/human_proxy_agent.py | 28 ++++++++++-------------- test/autogen/test_agent.py | 10 --------- 4 files changed, 12 insertions(+), 51 deletions(-) diff --git a/flaml/autogen/agent/agent.py b/flaml/autogen/agent/agent.py index 2acdbb198a..be65bea353 100644 --- a/flaml/autogen/agent/agent.py +++ b/flaml/autogen/agent/agent.py @@ -4,7 +4,7 @@ class Agent: """(Experimental) An abstract class for AI agent. An agent can communicate with other agents and perform actions. - Different agents can differ in what actions they perform in the receive method. + Different agents can differ in what actions they perform in the `receive` method. """ def __init__(self, name, system_message=""): diff --git a/flaml/autogen/agent/coding_agent.py b/flaml/autogen/agent/coding_agent.py index e6f3632db6..f4a3d230ae 100644 --- a/flaml/autogen/agent/coding_agent.py +++ b/flaml/autogen/agent/coding_agent.py @@ -14,8 +14,6 @@ class PythonAgent(Agent): DEFAULT_CONFIG = { "model": DEFAULT_MODEL, } - # EXECUTION_AGENT_PREFIX = "execution_agent4" - # SUCCESS_EXIT_CODE = "exitcode: 0\n" def __init__(self, name, system_message=DEFAULT_SYSTEM_MESSAGE, work_dir=None, **config): """ @@ -36,29 +34,8 @@ def receive(self, message, sender): self._sender_dict[sender.name] = sender self._conversations[sender.name] = [{"content": self._system_message, "role": "system"}] super().receive(message, sender) - # if sender.name.startswith(self.EXECUTION_AGENT_PREFIX) and message.startswith(self.SUCCESS_EXIT_CODE): - # # the code is correct, respond to the original sender - # name = sender.name[len(self.EXECUTION_AGENT_PREFIX) :] - # original_sender = self._sender_dict[name] - # output = message[len(self.SUCCESS_EXIT_CODE) :] - # if output: - # self._send(f"{output}", original_sender) - # else: - # self._send("Done. No output.", original_sender) - # return responses = oai.ChatCompletion.create(messages=self._conversations[sender.name], **self._config) - # cost = oai.ChatCompletion.cost(responses) response = oai.ChatCompletion.extract_text(responses)[0] - # if sender.name.startswith(self.EXECUTION_AGENT_PREFIX): - # execution_agent = sender - # else: - # # create an execution agent - # execution_agent = ExecutionAgent(f"{self.EXECUTION_AGENT_PREFIX}{sender.name}", work_dir=self._work_dir) - # # initialize the conversation - # self._conversations[execution_agent.name] = self._conversations[sender.name].copy() - # self._sender_dict[execution_agent.name] = execution_agent - # # send the response to the execution agent - # self._send(response, execution_agent) self._send(response, sender) def reset(self): diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index 08e30d26b5..fb4bd60ed5 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -6,15 +6,12 @@ class HumanProxyAgent(Agent): """(Experimental) A proxy agent for human, that can execute code and provide feedback to the other agents.""" - DEFAULT_SYSTEM_MESSAGE = """You are human agent. You can execute_code or give feedback to the sender. - """ MAX_CONSECUTIVE_AUTO_REPLY = 100 # maximum number of consecutive auto replies (subject to future change) def __init__( self, name, system_message="", - work_dir=None, human_input_mode="ALWAYS", max_consecutive_auto_reply=None, is_termination_msg=None, @@ -24,12 +21,15 @@ def __init__( Args: name (str): name of the agent system_message (str): system message to be sent to the agent - work_dir (str): working directory for the agent to execute code human_input_mode (bool): whether to ask for human inputs every time a message is received. Possible values are "ALWAYS", "TERMINATE", "NEVER". - When "ALWAYS", the agent will ask for human input every time a message is received. - When "TERMINATE", the agent will ask for human input only when a termination message is received. - When "NEVER", the agent will never ask for human input. + (1) When "ALWAYS", the agent prompts for human input every time a message is received. + Under this mode, the conversation stops when the human input is "exit". + (2) When "TERMINATE", the agent only prompts for human input only when a termination message is received or + the number of auto reply reaches the max_consecutive_auto_reply. + When human is prompted for input, the conversation stops if there is no input or the input is "exit". + (3) When "NEVER", the agent will never prompt for human input. Under this mode, the conversation stops + when the number of auto reply reaches the max_consecutive_auto_reply. max_consecutive_auto_reply (int): the maximum number of consecutive auto replies. default: None (no limit provided, class attribute MAX_CONSECUTIVE_AUTO_REPLY will be used as the limit in this case). The limit only plays a role when human_input_mode is not "ALWAYS". @@ -37,11 +37,8 @@ def __init__( This function is used to determine if a received message is a termination message. config (dict): other configurations. - The conversation stops when the human input is "exit", or no human input is provided and a termination message is received, - or the number of consecutive auto reply is larger than the provided max_consecutive_auto_reply (when human_input_mode is not "ALWAYS"). """ super().__init__(name, system_message) - self._work_dir = work_dir self._human_input_mode = human_input_mode self._is_termination_msg = ( is_termination_msg if is_termination_msg is not None else (lambda x: x == "TERMINATE") @@ -72,25 +69,22 @@ def _execute_code(self, code, lang): def receive(self, message, sender): """Receive a message from the sender agent. - Every time a message is received, the human agent will give feedback. - - The conversation stops when the human input is "exit", or no human input is provided and a termination message is received, - or the number of consecutive auto reply is larger than the provided max_consecutive_auto_reply (when human_input_mode is not "ALWAYS"). + Once a message is received, this function sends a reply to the sender or simply stop. + The reply can be generated automatically or entered manually by a human. """ super().receive(message, sender) # default reply is empty (i.e., no reply, in this case we will try to generate auto reply) reply = "" if self._human_input_mode == "ALWAYS": - # TODO: if skip and use auto reply, should we also display the auto reply? code, lang = extract_code(message) msg2display = ( - f"Code block detected: \n{code}\nUse auto-reply to execute the code and return the result." + f"\nCode block detected: \n{code}\nUse auto-reply to execute the code and return the result.\n" if lang != "unknown" else "" ) reply = input( "*" * 40 - + f"\n{msg2display} \nProvide feedback to the sender. Press enter to skip and use auto-reply, or type 'exit' to end the conversation: " + + f"{msg2display}Provide feedback to the sender. Press enter to skip and use auto-reply, or type 'exit' to end the conversation: " ) elif self._human_input_mode == "TERMINATE": if self._consecutive_auto_reply_counter[ diff --git a/test/autogen/test_agent.py b/test/autogen/test_agent.py index 9b3bec5b54..c3b33e3299 100644 --- a/test/autogen/test_agent.py +++ b/test/autogen/test_agent.py @@ -23,16 +23,6 @@ def test_coding_agent(human_input_mode="NEVER", max_consecutive_auto_reply=10): max_consecutive_auto_reply=max_consecutive_auto_reply, is_termination_msg=lambda x: x.rstrip().endswith("TERMINATE"), ) - # agent.receive("""Find $a+b+c$, given that $x+y\\neq -1$ and \\begin{align*} - # ax+by+c&=x+7,\\\\ - # a+bx+cy&=2x+6y,\\\\ - # ay+b+cx&=4x+y. - # \end{align*} - # Solve the problem smartly.""", user) - # agent.reset() - # agent.receive("""Let $a_1,a_2,a_3,\\dots$ be an arithmetic sequence. If $\\frac{a_4}{a_2} = 3$, what is $\\frac{a_5}{a_3}$? Solve the problem smartly.""", user) - # agent.reset() - # agent.receive("""The product of the first and the third terms of an arithmetic sequence is $5$. If all terms of the sequence are positive integers, what is the fourth term? Solve the problem smartly.""", user) agent.receive( """Create and execute a script to plot a rocket""", user, From fe191812f333a20124b742437a6c7afabe0cac3c Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Wed, 10 May 2023 12:13:26 -0400 Subject: [PATCH 21/32] add back work_dir --- flaml/autogen/agent/agent.py | 1 + flaml/autogen/agent/human_proxy_agent.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/flaml/autogen/agent/agent.py b/flaml/autogen/agent/agent.py index be65bea353..ec4165515f 100644 --- a/flaml/autogen/agent/agent.py +++ b/flaml/autogen/agent/agent.py @@ -5,6 +5,7 @@ class Agent: """(Experimental) An abstract class for AI agent. An agent can communicate with other agents and perform actions. Different agents can differ in what actions they perform in the `receive` method. + """ def __init__(self, name, system_message=""): diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index fb4bd60ed5..e38f94ef66 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -12,6 +12,7 @@ def __init__( self, name, system_message="", + work_dir=None, human_input_mode="ALWAYS", max_consecutive_auto_reply=None, is_termination_msg=None, @@ -21,6 +22,7 @@ def __init__( Args: name (str): name of the agent system_message (str): system message to be sent to the agent + work_dir (str): working directory for the agent human_input_mode (bool): whether to ask for human inputs every time a message is received. Possible values are "ALWAYS", "TERMINATE", "NEVER". (1) When "ALWAYS", the agent prompts for human input every time a message is received. @@ -39,6 +41,7 @@ def __init__( """ super().__init__(name, system_message) + self._work_dir = work_dir self._human_input_mode = human_input_mode self._is_termination_msg = ( is_termination_msg if is_termination_msg is not None else (lambda x: x == "TERMINATE") From 6d5bbd7438977453a09649fdc6708dc9a58848f4 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Wed, 10 May 2023 12:48:45 -0400 Subject: [PATCH 22/32] add is_termination_msg when mode is NEVER --- flaml/autogen/agent/human_proxy_agent.py | 19 ++++++++----------- test/autogen/test_agent.py | 5 ++++- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index e38f94ef66..454233adc0 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -31,7 +31,7 @@ def __init__( the number of auto reply reaches the max_consecutive_auto_reply. When human is prompted for input, the conversation stops if there is no input or the input is "exit". (3) When "NEVER", the agent will never prompt for human input. Under this mode, the conversation stops - when the number of auto reply reaches the max_consecutive_auto_reply. + when the number of auto reply reaches the max_consecutive_auto_reply or when is_termination_msg is True max_consecutive_auto_reply (int): the maximum number of consecutive auto replies. default: None (no limit provided, class attribute MAX_CONSECUTIVE_AUTO_REPLY will be used as the limit in this case). The limit only plays a role when human_input_mode is not "ALWAYS". @@ -89,20 +89,17 @@ def receive(self, message, sender): "*" * 40 + f"{msg2display}Provide feedback to the sender. Press enter to skip and use auto-reply, or type 'exit' to end the conversation: " ) - elif self._human_input_mode == "TERMINATE": - if self._consecutive_auto_reply_counter[ - sender.name - ] >= self._max_consecutive_auto_reply or self._is_termination_msg(message): + elif self._consecutive_auto_reply_counter[ + sender.name + ] >= self._max_consecutive_auto_reply or self._is_termination_msg(message): + if self._human_input_mode == "TERMINATE": reply = input( "Please give feedback to the sender. (Press enter or type 'exit' to stop the conversation): " ) reply = reply if reply else "exit" - elif ( - self._human_input_mode == "NEVER" - and self._consecutive_auto_reply_counter[sender.name] >= self._max_consecutive_auto_reply - ): - reply = "exit" - + else: + # this corresponds to the case when self._human_input_mode == "NEVER" + reply = "exit" if reply == "exit": return elif reply: diff --git a/test/autogen/test_agent.py b/test/autogen/test_agent.py index c3b33e3299..b794e36784 100644 --- a/test/autogen/test_agent.py +++ b/test/autogen/test_agent.py @@ -84,4 +84,7 @@ def test_tsp(human_input_mode="NEVER", max_consecutive_auto_reply=10): # openai.api_key = "" # test_extract_code() test_coding_agent(human_input_mode="TERMINATE") - test_tsp(human_input_mode="NEVER", max_consecutive_auto_reply=2) + # when GPT-4, i.e., the DEFAULT_MODEL, is used, conversation in the following test + # should terminate in 2-3 rounds of interactions (because is_termination_msg should be true after 2-3 rounds) + # although the max_consecutive_auto_reply is set to 10. + test_tsp(human_input_mode="NEVER", max_consecutive_auto_reply=10) From d3a72cd3839a02f6cb834e543305c8511a370e2a Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Wed, 10 May 2023 16:36:49 -0400 Subject: [PATCH 23/32] revise stop condition --- flaml/autogen/agent/human_proxy_agent.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index 454233adc0..7422cce818 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -29,9 +29,11 @@ def __init__( Under this mode, the conversation stops when the human input is "exit". (2) When "TERMINATE", the agent only prompts for human input only when a termination message is received or the number of auto reply reaches the max_consecutive_auto_reply. - When human is prompted for input, the conversation stops if there is no input or the input is "exit". (3) When "NEVER", the agent will never prompt for human input. Under this mode, the conversation stops - when the number of auto reply reaches the max_consecutive_auto_reply or when is_termination_msg is True + when the number of auto reply reaches the max_consecutive_auto_reply. + In any case, the agent also stop the conversation in the following two cases: + (a) if is_termination_msg is True and there is no human input; + (b) if the human input is "exit". max_consecutive_auto_reply (int): the maximum number of consecutive auto replies. default: None (no limit provided, class attribute MAX_CONSECUTIVE_AUTO_REPLY will be used as the limit in this case). The limit only plays a role when human_input_mode is not "ALWAYS". @@ -100,7 +102,7 @@ def receive(self, message, sender): else: # this corresponds to the case when self._human_input_mode == "NEVER" reply = "exit" - if reply == "exit": + if reply == "exit" or (self._is_termination_msg(message) and not reply): return elif reply: # reset the consecutive_auto_reply_counter From 66da272cd3d97beb796fd97dc006188b34aefd98 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Thu, 11 May 2023 14:44:20 -0400 Subject: [PATCH 24/32] remove work_dir in coding agent --- flaml/autogen/agent/coding_agent.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flaml/autogen/agent/coding_agent.py b/flaml/autogen/agent/coding_agent.py index f4a3d230ae..f7c16f8253 100644 --- a/flaml/autogen/agent/coding_agent.py +++ b/flaml/autogen/agent/coding_agent.py @@ -15,16 +15,14 @@ class PythonAgent(Agent): "model": DEFAULT_MODEL, } - def __init__(self, name, system_message=DEFAULT_SYSTEM_MESSAGE, work_dir=None, **config): + def __init__(self, name, system_message=DEFAULT_SYSTEM_MESSAGE, **config): """ Args: name (str): agent name system_message (str): system message to be sent to the agent - work_dir (str): working directory for the agent to execute code config (dict): other configurations. """ super().__init__(name, system_message) - self._work_dir = work_dir self._config = self.DEFAULT_CONFIG.copy() self._config.update(config) self._sender_dict = {} From 5faade36976974eec629c8700055c0f13bf42b8f Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Thu, 11 May 2023 14:49:15 -0400 Subject: [PATCH 25/32] human_proxy_agent docstr --- flaml/autogen/agent/human_proxy_agent.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index 7422cce818..f23ffd55c3 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -26,14 +26,12 @@ def __init__( human_input_mode (bool): whether to ask for human inputs every time a message is received. Possible values are "ALWAYS", "TERMINATE", "NEVER". (1) When "ALWAYS", the agent prompts for human input every time a message is received. - Under this mode, the conversation stops when the human input is "exit". + Under this mode, the conversation stops when the human input is "exit", + or when is_termination_msg is True and there is no human input. (2) When "TERMINATE", the agent only prompts for human input only when a termination message is received or the number of auto reply reaches the max_consecutive_auto_reply. (3) When "NEVER", the agent will never prompt for human input. Under this mode, the conversation stops - when the number of auto reply reaches the max_consecutive_auto_reply. - In any case, the agent also stop the conversation in the following two cases: - (a) if is_termination_msg is True and there is no human input; - (b) if the human input is "exit". + when the number of auto reply reaches the max_consecutive_auto_reply or or when is_termination_msg is True. max_consecutive_auto_reply (int): the maximum number of consecutive auto replies. default: None (no limit provided, class attribute MAX_CONSECUTIVE_AUTO_REPLY will be used as the limit in this case). The limit only plays a role when human_input_mode is not "ALWAYS". From 1a72ba0202c19d03dd76ed578a1c7f679e4d43af Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Fri, 12 May 2023 01:21:39 +0000 Subject: [PATCH 26/32] add meta agent --- flaml/autogen/agent/chat_agent.py | 9 +++++--- test/autogen/test_meta_agent.py | 38 +++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 test/autogen/test_meta_agent.py diff --git a/flaml/autogen/agent/chat_agent.py b/flaml/autogen/agent/chat_agent.py index a6d487a711..71414dea96 100644 --- a/flaml/autogen/agent/chat_agent.py +++ b/flaml/autogen/agent/chat_agent.py @@ -1,6 +1,7 @@ from .agent import Agent -from flaml.autogen.code_utils import DEFAULT_MODEL +from flaml.autogen.code_utils import DEFAULT_MODEL, FAST_MODEL from flaml import oai +from collections import defaultdict class ChatAgent(Agent): @@ -10,10 +11,10 @@ class ChatAgent(Agent): """ DEFAULT_CONFIG = { - "model": DEFAULT_MODEL, + "model": FAST_MODEL, } - def __init__(self, name, system_message=DEFAULT_SYSTEM_MESSAGE, work_dir=None, **config): + def __init__(self, name, system_message=DEFAULT_SYSTEM_MESSAGE, work_dir=None, meta_prompt=None, **config): """ Args: name (str): agent name @@ -26,8 +27,10 @@ def __init__(self, name, system_message=DEFAULT_SYSTEM_MESSAGE, work_dir=None, * self._config = self.DEFAULT_CONFIG.copy() self._config.update(config) self._sender_dict = {} + self._meta_prompt = [meta_prompt] if meta_prompt else [] def receive(self, message, sender): + message = self._meta_prompt.pop() + f"User message is: {message}" if self._meta_prompt else message super().receive(message, sender) responses = oai.ChatCompletion.create(messages=self._conversations[sender.name], **self._config) # cost = oai.ChatCompletion.cost(responses) diff --git a/test/autogen/test_meta_agent.py b/test/autogen/test_meta_agent.py new file mode 100644 index 0000000000..639447e73e --- /dev/null +++ b/test/autogen/test_meta_agent.py @@ -0,0 +1,38 @@ +from flaml.autogen.agent.chat_agent import ChatAgent +from flaml.autogen.agent.coding_agent import PythonAgent + +from flaml.autogen.agent.human_proxy_agent import HumanProxyAgent +from flaml.autogen.math_utils import eval_math_responses, get_answer +from flaml import oai +from flaml.autogen.agent.meta_agent import MetaAgent + + +def test_meta_prompt(): + """ + Based on: + https://python.langchain.com/en/latest/use_cases/autonomous_agents/meta_prompt.html + """ + + tasks = [ + "Provide a systematic argument for why we should always eat pasta with olives.", + "Provide a systematic argument for why we should always eat noodle with garlic.", + "What should I eat for dinner", + "What should I eat for lunch", + ] + oai.ChatCompletion.start_logging() + agent = MetaAgent(ChatAgent, name="chat_agent", meta_agent_name="meta_agent") + user = HumanProxyAgent( + "human user", work_dir="test/autogen", human_input_mode="ALWAYS", max_consecutive_auto_reply=10 + ) + for i, task in enumerate(tasks): + agent.receive(task, user) + print(f"====={i+1}-th task finished!=====") + agent.reflect() + + +if __name__ == "__main__": + import openai + + # openai.api_key_path = "test/openai/key.txt" + openai.api_key_path = "test/openai/key_gpt3.txt" + test_meta_prompt() From 93d078885047344dfadce85ac77216881b410207 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Fri, 12 May 2023 01:24:32 +0000 Subject: [PATCH 27/32] add meta agent --- flaml/autogen/agent/meta_agent.py | 140 ++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 flaml/autogen/agent/meta_agent.py diff --git a/flaml/autogen/agent/meta_agent.py b/flaml/autogen/agent/meta_agent.py new file mode 100644 index 0000000000..faa8184305 --- /dev/null +++ b/flaml/autogen/agent/meta_agent.py @@ -0,0 +1,140 @@ +from collections import defaultdict +from flaml.autogen.agent.agent import Agent +from flaml.autogen.agent.chat_agent import ChatAgent +from flaml.autogen.agent.coding_agent import PythonAgent +from langchain import OpenAI, LLMChain, PromptTemplate +from flaml import oai +from flaml.autogen.code_utils import DEFAULT_MODEL, FAST_MODEL + + +class MetaAgent: + """(Experimental) A meta agent that can wrap other agents and perform actions based on the messages received.""" + + DEFAULT_CONFIG = { + "model": FAST_MODEL, + } + + DEFAULT_SYSTEM_MESSAGE = """ + Assistant has just had the below interactions with a User. Assistant followed their "Instructions" closely. + Your job is to critique the Assistant's performance and then revise the Instructions so that Assistant would + quickly and correctly respond in the future. + + #### + + {chat_history} + + #### + + Please reflect on these interactions. + + You should first critique Assistant's performance. What could Assistant have done better? + What should the Assistant remember about this user? Are there things this user always wants? + Indicate this with "Critique: ...". + + You should next revise the Instructions so that Assistant would quickly and correctly respond in the future. + Assistant's goal is to satisfy the user in as few interactions as possible. Assistant will only see the new Instructions, not the interaction history, so anything important must be summarized in the Instructions. Don't forget any important details in the current Instructions! Indicate the new Instructions by "Instructions: ...". + """ + + def __init__( + self, agent_class, name, system_message="", meta_agent_name="meta-agent", meta_agent_system_message="" + ): + """ + Args: + agent_class (Agent): the class of the agent to be wrapped + name (str): name of the agent to be wrapped + system_message (str): system message to be sent to the agent to be wrapped + meta_agent_name (str): name of the meta agent + meta_agent_system_message (str): system message to be sent to the meta agent + """ + self._memory = [] + # a dictionary of conversations, default value is list + self._conversations = defaultdict(list) + self._name = meta_agent_name + self._config = self.DEFAULT_CONFIG + self._system_prompt_glossary = { + ChatAgent.__name__: ChatAgent.DEFAULT_SYSTEM_MESSAGE, + PythonAgent.__name__: PythonAgent.DEFAULT_SYSTEM_MESSAGE, + } + self._system_message = ( + meta_agent_system_message + if meta_agent_system_message + else self._system_prompt_glossary[agent_class.__name__] + ) + self._agent_class = agent_class + + self._agent = agent_class(name) + print("Agent class", self._agent_class) + self._meta_template = """ + Assistant has just had the below interactions with a User. Assistant followed their "Instructions" closely. + Your job is to critique the Assistant's performance and then revise the Instructions so that Assistant would + quickly and correctly respond in the future. + + #### + + {chat_history} + + #### + + Please reflect on these interactions. + + You should first critique Assistant's performance. What could Assistant have done better? + What should the Assistant remember about this user? Are there things this user always wants? + Indicate this with "Critique: ...". + + You should next revise the Instructions so that Assistant would quickly and correctly respond in the future. + Assistant's goal is to satisfy the user in as few interactions as possible. Assistant will only see the new Instructions, not the interaction history, so anything important must be summarized in the Instructions. Don't forget any important details in the current Instructions! Indicate the new Instructions by "Instructions: ...". + """ + + def _receive(self, message, sender): + """Receive a message from another agent.""" + self._agent.receive(message, sender) + + def _init_reflect(self): + PromptTemplate(input_variables=["chat_history"], template=self._system_message) + + def _get_chat_history(self): + """Get the chat history of the agent.""" + chat_history = "" + for conversation in self._agent._conversations.values(): + for message in conversation: + if message["role"] == "user": + chat_history += "User: " + message["content"] + "\n" + else: + chat_history += "Assistant: " + message["content"] + "\n" + return chat_history + + def _new_task_bookkeeping(self): + """Bookkeeping for a new task.""" + self._agent._conversations = defaultdict(list) + self._agent._memory = [] + self._agent._sender_dict = {} + + def reflect(self): + """Reflect on the conversations with the agents.""" + chat_history = self._get_chat_history() + # PromptTemplate( + # input_variables=["chat_history"], + # template=self._meta_template + # ) + meta_prompt = self._meta_template.format(chat_history=chat_history) + + responses = oai.ChatCompletion.create(messages=[{"content": meta_prompt, "role": "user"}], **self._config) + response = oai.ChatCompletion.extract_text(responses)[0] + print(f"{self._name}", response) + + # new_instructions = self._get_new_instructions() + # self._init_reflect() + # self._agent._send(chat_history, self) + # self._agent._send(new_instructions, self) + # self._system_message = new_instructions + self._agent = self._agent_class(self._name, meta_prompt=response) + # TODO: maybe we should also consider adding the instruction as the init prompt + + def receive(self, message, sender, is_new_task=False): + """Receive a message from another agent. + This method is called by the sender. + It needs to be overriden by the subclass to perform followup actions. + """ + print(f'{self._name} received "{message}" from {sender._name}') + self._receive(message, sender) + # perform actions based on the message From 6b4eca4a7962ba94cc362d5ecc0395e754878af9 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Fri, 12 May 2023 14:14:36 -0400 Subject: [PATCH 28/32] auto_reply --- flaml/autogen/agent/human_proxy_agent.py | 25 +++++++++++++++--------- test/autogen/test_agent.py | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index f23ffd55c3..2d0d307fdb 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -70,6 +70,19 @@ def _execute_code(self, code, lang): # raise NotImplementedError return exitcode, logs + def auto_reply(self, message, sender, default_reply="", lang=None, code=None): + """Generate an auto reply.""" + if lang is None: + code, lang = extract_code(message) + if lang == "unknown": + # no code block is found, lang should be "unknown" + self._send(default_reply, sender) + else: + # try to execute the code + exitcode, logs = self._execute_code(code, lang) + exitcode2str = "execution succeeded" if exitcode == 0 else "execution failed" + self._send(f"exitcode: {exitcode} ({exitcode2str})\nCode output: {logs.decode('utf-8')}", sender) + def receive(self, message, sender): """Receive a message from the sender agent. Once a message is received, this function sends a reply to the sender or simply stop. @@ -109,13 +122,7 @@ def receive(self, message, sender): return self._consecutive_auto_reply_counter[sender.name] += 1 - if self._human_input_mode != "ALWAYS": - code, lang = extract_code(message) - if lang == "unknown": - # no code block is found, lang should be "unknown" - self._send(reply, sender) + if self._human_input_mode == "ALWAYS": + self.auto_reply(message, sender, default_reply=reply, lang=lang, code=code) else: - # try to execute the code - exitcode, logs = self._execute_code(code, lang) - exitcode2str = "execution succeeded" if exitcode == 0 else "execution failed" - self._send(f"exitcode: {exitcode} ({exitcode2str})\nCode output: {logs.decode('utf-8')}", sender) + self.auto_reply(message, sender, default_reply=reply) diff --git a/test/autogen/test_agent.py b/test/autogen/test_agent.py index b794e36784..e7832aaff4 100644 --- a/test/autogen/test_agent.py +++ b/test/autogen/test_agent.py @@ -24,7 +24,7 @@ def test_coding_agent(human_input_mode="NEVER", max_consecutive_auto_reply=10): is_termination_msg=lambda x: x.rstrip().endswith("TERMINATE"), ) agent.receive( - """Create and execute a script to plot a rocket""", + """Create and execute a script to plot a rocket without using matplotlib""", user, ) agent.reset() From 5c2eac26a7c71ab15ff7cf42c213832d4995686f Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Fri, 12 May 2023 22:15:57 -0400 Subject: [PATCH 29/32] update meta agent --- flaml/autogen/agent/meta_agent.py | 70 ++++++++++--------------------- test/autogen/test_meta_agent.py | 13 ++---- 2 files changed, 25 insertions(+), 58 deletions(-) diff --git a/flaml/autogen/agent/meta_agent.py b/flaml/autogen/agent/meta_agent.py index faa8184305..c002e16e58 100644 --- a/flaml/autogen/agent/meta_agent.py +++ b/flaml/autogen/agent/meta_agent.py @@ -2,12 +2,11 @@ from flaml.autogen.agent.agent import Agent from flaml.autogen.agent.chat_agent import ChatAgent from flaml.autogen.agent.coding_agent import PythonAgent -from langchain import OpenAI, LLMChain, PromptTemplate from flaml import oai from flaml.autogen.code_utils import DEFAULT_MODEL, FAST_MODEL -class MetaAgent: +class MetaAgent(Agent): """(Experimental) A meta agent that can wrap other agents and perform actions based on the messages received.""" DEFAULT_CONFIG = { @@ -51,20 +50,21 @@ def __init__( self._conversations = defaultdict(list) self._name = meta_agent_name self._config = self.DEFAULT_CONFIG - self._system_prompt_glossary = { - ChatAgent.__name__: ChatAgent.DEFAULT_SYSTEM_MESSAGE, - PythonAgent.__name__: PythonAgent.DEFAULT_SYSTEM_MESSAGE, - } - self._system_message = ( - meta_agent_system_message - if meta_agent_system_message - else self._system_prompt_glossary[agent_class.__name__] - ) + self._system_message = meta_agent_system_message if meta_agent_system_message else self.DEFAULT_SYSTEM_MESSAGE self._agent_class = agent_class - - self._agent = agent_class(name) - print("Agent class", self._agent_class) - self._meta_template = """ + if system_message: + self._agent = agent_class(name, system_message=system_message) + else: + self._agent = agent_class(name) + + # TODO: Maintain a dictionary of meta prompts for each agent class + # self._meta_prompt_glossary = { + # ChatAgent.__name__: ChatAgent.DEFAULT_SYSTEM_MESSAGE, + # PythonAgent.__name__: PythonAgent.DEFAULT_SYSTEM_MESSAGE, + # } + + # the following meta prompt is based on https://noahgoodman.substack.com/p/meta-prompt-a-simple-self-improving + self._meta_prompt_template = """ Assistant has just had the below interactions with a User. Assistant followed their "Instructions" closely. Your job is to critique the Assistant's performance and then revise the Instructions so that Assistant would quickly and correctly respond in the future. @@ -77,21 +77,19 @@ def __init__( Please reflect on these interactions. - You should first critique Assistant's performance. What could Assistant have done better? + You should first reflect on Assistant's performance. What could Assistant have done better? What should the Assistant remember about this user? Are there things this user always wants? - Indicate this with "Critique: ...". + Indicate this with "Reflection: ...". You should next revise the Instructions so that Assistant would quickly and correctly respond in the future. - Assistant's goal is to satisfy the user in as few interactions as possible. Assistant will only see the new Instructions, not the interaction history, so anything important must be summarized in the Instructions. Don't forget any important details in the current Instructions! Indicate the new Instructions by "Instructions: ...". + Assistant's goal is to satisfy the user in as few interactions as possible. Assistant will only see the new Instructions, not the interaction history, so anything important must be summarized in the Instructions. + Indicate the new Instructions by "Instructions: ...". """ def _receive(self, message, sender): """Receive a message from another agent.""" self._agent.receive(message, sender) - def _init_reflect(self): - PromptTemplate(input_variables=["chat_history"], template=self._system_message) - def _get_chat_history(self): """Get the chat history of the agent.""" chat_history = "" @@ -103,38 +101,12 @@ def _get_chat_history(self): chat_history += "Assistant: " + message["content"] + "\n" return chat_history - def _new_task_bookkeeping(self): - """Bookkeeping for a new task.""" - self._agent._conversations = defaultdict(list) - self._agent._memory = [] - self._agent._sender_dict = {} - def reflect(self): """Reflect on the conversations with the agents.""" chat_history = self._get_chat_history() - # PromptTemplate( - # input_variables=["chat_history"], - # template=self._meta_template - # ) - meta_prompt = self._meta_template.format(chat_history=chat_history) - + meta_prompt = self._meta_prompt_template.format(chat_history=chat_history) responses = oai.ChatCompletion.create(messages=[{"content": meta_prompt, "role": "user"}], **self._config) response = oai.ChatCompletion.extract_text(responses)[0] - print(f"{self._name}", response) - - # new_instructions = self._get_new_instructions() - # self._init_reflect() - # self._agent._send(chat_history, self) - # self._agent._send(new_instructions, self) - # self._system_message = new_instructions + print(f"Reflecting.....\n{self._name}", response) self._agent = self._agent_class(self._name, meta_prompt=response) # TODO: maybe we should also consider adding the instruction as the init prompt - - def receive(self, message, sender, is_new_task=False): - """Receive a message from another agent. - This method is called by the sender. - It needs to be overriden by the subclass to perform followup actions. - """ - print(f'{self._name} received "{message}" from {sender._name}') - self._receive(message, sender) - # perform actions based on the message diff --git a/test/autogen/test_meta_agent.py b/test/autogen/test_meta_agent.py index 639447e73e..8e8df84de9 100644 --- a/test/autogen/test_meta_agent.py +++ b/test/autogen/test_meta_agent.py @@ -8,26 +8,21 @@ def test_meta_prompt(): - """ - Based on: - https://python.langchain.com/en/latest/use_cases/autonomous_agents/meta_prompt.html - """ - tasks = [ "Provide a systematic argument for why we should always eat pasta with olives.", "Provide a systematic argument for why we should always eat noodle with garlic.", "What should I eat for dinner", "What should I eat for lunch", ] + # Reply: Your response should be in the form of a poem. Try again! oai.ChatCompletion.start_logging() agent = MetaAgent(ChatAgent, name="chat_agent", meta_agent_name="meta_agent") - user = HumanProxyAgent( - "human user", work_dir="test/autogen", human_input_mode="ALWAYS", max_consecutive_auto_reply=10 - ) + user = HumanProxyAgent("human user", work_dir="test/autogen", human_input_mode="ALWAYS") for i, task in enumerate(tasks): + print(f".........Starting the {i+1}-th task!.........") agent.receive(task, user) - print(f"====={i+1}-th task finished!=====") agent.reflect() + print(f".........{i+1}-th task finished!.........") if __name__ == "__main__": From 16fffbaaeb4cfd0360a75defc01dc24baac34d4c Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Fri, 12 May 2023 22:22:06 -0400 Subject: [PATCH 30/32] clean auto_reply --- flaml/autogen/agent/human_proxy_agent.py | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/flaml/autogen/agent/human_proxy_agent.py b/flaml/autogen/agent/human_proxy_agent.py index 2d0d307fdb..70cd285e4c 100644 --- a/flaml/autogen/agent/human_proxy_agent.py +++ b/flaml/autogen/agent/human_proxy_agent.py @@ -70,10 +70,9 @@ def _execute_code(self, code, lang): # raise NotImplementedError return exitcode, logs - def auto_reply(self, message, sender, default_reply="", lang=None, code=None): + def auto_reply(self, message, sender, default_reply=""): """Generate an auto reply.""" - if lang is None: - code, lang = extract_code(message) + code, lang = extract_code(message) if lang == "unknown": # no code block is found, lang should be "unknown" self._send(default_reply, sender) @@ -92,15 +91,8 @@ def receive(self, message, sender): # default reply is empty (i.e., no reply, in this case we will try to generate auto reply) reply = "" if self._human_input_mode == "ALWAYS": - code, lang = extract_code(message) - msg2display = ( - f"\nCode block detected: \n{code}\nUse auto-reply to execute the code and return the result.\n" - if lang != "unknown" - else "" - ) reply = input( - "*" * 40 - + f"{msg2display}Provide feedback to the sender. Press enter to skip and use auto-reply, or type 'exit' to end the conversation: " + "Provide feedback to the sender. Press enter to skip and use auto-reply, or type 'exit' to end the conversation: " ) elif self._consecutive_auto_reply_counter[ sender.name @@ -122,7 +114,4 @@ def receive(self, message, sender): return self._consecutive_auto_reply_counter[sender.name] += 1 - if self._human_input_mode == "ALWAYS": - self.auto_reply(message, sender, default_reply=reply, lang=lang, code=code) - else: - self.auto_reply(message, sender, default_reply=reply) + self.auto_reply(message, sender, default_reply=reply) From f4214bdded056dc6ac205684f5096a032b15517c Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Wed, 24 May 2023 15:27:13 -0400 Subject: [PATCH 31/32] update meta agent --- flaml/autogen/agent/meta_agent.py | 65 ++++++++++++++----------------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/flaml/autogen/agent/meta_agent.py b/flaml/autogen/agent/meta_agent.py index c002e16e58..bbd0825332 100644 --- a/flaml/autogen/agent/meta_agent.py +++ b/flaml/autogen/agent/meta_agent.py @@ -1,7 +1,6 @@ from collections import defaultdict +from typing import Optional from flaml.autogen.agent.agent import Agent -from flaml.autogen.agent.chat_agent import ChatAgent -from flaml.autogen.agent.coding_agent import PythonAgent from flaml import oai from flaml.autogen.code_utils import DEFAULT_MODEL, FAST_MODEL @@ -10,7 +9,7 @@ class MetaAgent(Agent): """(Experimental) A meta agent that can wrap other agents and perform actions based on the messages received.""" DEFAULT_CONFIG = { - "model": FAST_MODEL, + "model": DEFAULT_MODEL, } DEFAULT_SYSTEM_MESSAGE = """ @@ -35,35 +34,25 @@ class MetaAgent(Agent): """ def __init__( - self, agent_class, name, system_message="", meta_agent_name="meta-agent", meta_agent_system_message="" + self, + name, + system_message="", + agent: Optional[Agent] = None, + dev_agent: Optional[Agent] = None, ): """ Args: - agent_class (Agent): the class of the agent to be wrapped - name (str): name of the agent to be wrapped - system_message (str): system message to be sent to the agent to be wrapped - meta_agent_name (str): name of the meta agent - meta_agent_system_message (str): system message to be sent to the meta agent + name (str): name of the meta agent + system_message (str): system message to be sent to the meta agent + user_agent (optional): the agent to be wrapped + dev_agent (optional): the agent to be wrapped for development """ - self._memory = [] - # a dictionary of conversations, default value is list - self._conversations = defaultdict(list) - self._name = meta_agent_name - self._config = self.DEFAULT_CONFIG - self._system_message = meta_agent_system_message if meta_agent_system_message else self.DEFAULT_SYSTEM_MESSAGE - self._agent_class = agent_class - if system_message: - self._agent = agent_class(name, system_message=system_message) - else: - self._agent = agent_class(name) - - # TODO: Maintain a dictionary of meta prompts for each agent class - # self._meta_prompt_glossary = { - # ChatAgent.__name__: ChatAgent.DEFAULT_SYSTEM_MESSAGE, - # PythonAgent.__name__: PythonAgent.DEFAULT_SYSTEM_MESSAGE, - # } - - # the following meta prompt is based on https://noahgoodman.substack.com/p/meta-prompt-a-simple-self-improving + # use super() to call the parent class's __init__ method + super().__init__(name, system_message=system_message) + self._system_message = system_message if system_message else self.DEFAULT_SYSTEM_MESSAGE + # TODO: do we only need to have only user_agent or dev_agent? + self._agent = agent + self._dev_agent = dev_agent self._meta_prompt_template = """ Assistant has just had the below interactions with a User. Assistant followed their "Instructions" closely. Your job is to critique the Assistant's performance and then revise the Instructions so that Assistant would @@ -88,7 +77,10 @@ def __init__( def _receive(self, message, sender): """Receive a message from another agent.""" - self._agent.receive(message, sender) + if self._agent: + self._agent.receive(message, sender) + # if self._dev_agent: + # self._dev_agent.receive(message, sender) def _get_chat_history(self): """Get the chat history of the agent.""" @@ -102,11 +94,12 @@ def _get_chat_history(self): return chat_history def reflect(self): - """Reflect on the conversations with the agents.""" - chat_history = self._get_chat_history() - meta_prompt = self._meta_prompt_template.format(chat_history=chat_history) - responses = oai.ChatCompletion.create(messages=[{"content": meta_prompt, "role": "user"}], **self._config) - response = oai.ChatCompletion.extract_text(responses)[0] - print(f"Reflecting.....\n{self._name}", response) - self._agent = self._agent_class(self._name, meta_prompt=response) + self.receive("reflect", self._dev_agent) + # """Reflect on the conversations with the agents.""" + # chat_history = self._get_chat_history() + # meta_prompt = self._meta_prompt_template.format(chat_history=chat_history) + # responses = oai.ChatCompletion.create(messages=[{"content": meta_prompt, "role": "user"}], **self._config) + # response = oai.ChatCompletion.extract_text(responses)[0] + # print(f"Reflecting.....\n{self._name}", response) + # self._agent = self._agent_class(self._name, meta_prompt=response) # TODO: maybe we should also consider adding the instruction as the init prompt From 9bb46551dbe8b55d22c9422f3e8508974625c6c6 Mon Sep 17 00:00:00 2001 From: Qingyun Wu Date: Wed, 24 May 2023 15:28:57 -0400 Subject: [PATCH 32/32] update meta agent test --- test/autogen/test_meta_agent.py | 43 ++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/test/autogen/test_meta_agent.py b/test/autogen/test_meta_agent.py index 8e8df84de9..15b210af0b 100644 --- a/test/autogen/test_meta_agent.py +++ b/test/autogen/test_meta_agent.py @@ -1,33 +1,42 @@ -from flaml.autogen.agent.chat_agent import ChatAgent from flaml.autogen.agent.coding_agent import PythonAgent - -from flaml.autogen.agent.human_proxy_agent import HumanProxyAgent -from flaml.autogen.math_utils import eval_math_responses, get_answer -from flaml import oai +from flaml.autogen.agent.user_proxy_agent import UserProxyAgent from flaml.autogen.agent.meta_agent import MetaAgent def test_meta_prompt(): tasks = [ - "Provide a systematic argument for why we should always eat pasta with olives.", - "Provide a systematic argument for why we should always eat noodle with garlic.", - "What should I eat for dinner", - "What should I eat for lunch", + "Create and execute a script to plot a rocket without using matplotlib.", + "Create and execute a script to plot a helicopter without using matplotlib", ] - # Reply: Your response should be in the form of a poem. Try again! - oai.ChatCompletion.start_logging() - agent = MetaAgent(ChatAgent, name="chat_agent", meta_agent_name="meta_agent") - user = HumanProxyAgent("human user", work_dir="test/autogen", human_input_mode="ALWAYS") + + ## User mode: + python_agent = PythonAgent("python agent") + assistant = MetaAgent(name="meta agent", agent=python_agent) + user = UserProxyAgent("human user", work_dir="test/autogen", human_input_mode="ALWAYS") + for i, task in enumerate(tasks): + print(f".........Starting the {i+1}-th task!.........") + assistant.receive(task, user) + print(f".........{i+1}-th task finished!.........") + + ## Dev mode: + dev = UserProxyAgent("expert", work_dir="test/autogen", human_input_mode="ALWAYS") + assistant = MetaAgent(name="meta agent", agent=python_agent, dev_agent=dev) + user = UserProxyAgent("human user", work_dir="test/autogen", human_input_mode="ALWAYS") + for i, task in enumerate(tasks[0:2]): + assistant.receive(task, user) + assistant.reflect() + + ### Can also be used in the following way: for i, task in enumerate(tasks): print(f".........Starting the {i+1}-th task!.........") - agent.receive(task, user) - agent.reflect() + assistant.receive(task, dev) + assistant.reflect() print(f".........{i+1}-th task finished!.........") if __name__ == "__main__": import openai - # openai.api_key_path = "test/openai/key.txt" - openai.api_key_path = "test/openai/key_gpt3.txt" + openai.api_key_path = "test/openai/key.txt" + # openai.api_key_path = "test/openai/key_gpt3.txt" test_meta_prompt()