From 9aec636b02435ea857c181b165307cad973a0d89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=8E=E5=A4=A9?= <460342015@qq.com> Date: Wed, 27 Dec 2023 16:25:55 +0800 Subject: [PATCH] feat(agent): Multi agent sdk (#976) Co-authored-by: xtyuns Co-authored-by: Fangyin Cheng Co-authored-by: csunny Co-authored-by: qidanrui --- .gitignore | 2 +- dbgpt/agent/__init__.py | 12 +- dbgpt/agent/{commands => agents}/__init__.py | 0 dbgpt/agent/agents/agent.py | 188 +++++ dbgpt/agent/agents/agents_mange.py | 48 ++ dbgpt/agent/agents/base_agent.py | 768 ++++++++++++++++++ .../built_in => agents/expand}/__init__.py | 0 .../agents/expand/code_assistant_agent.py | 261 ++++++ .../expand/dashboard_assistant_agent.py | 121 +++ .../agents/expand/data_scientist_agent.py | 162 ++++ .../agents/expand/plugin_assistant_agent.py | 125 +++ .../expand/rag_assistant_agent.py} | 0 .../agents/expand/sql_assistant_agent.py | 111 +++ dbgpt/agent/{db => agents/llm}/__init__.py | 0 dbgpt/agent/agents/llm/llm.py | 35 + dbgpt/agent/agents/llm/llm_client.py | 194 +++++ dbgpt/agent/agents/plan_group_chat.py | 452 +++++++++++ dbgpt/agent/agents/planner_agent.py | 191 +++++ dbgpt/agent/agents/user_proxy_agent.py | 89 ++ dbgpt/agent/common/schema.py | 14 +- dbgpt/agent/memory/base.py | 235 ++++++ dbgpt/agent/memory/default_gpts_memory.py | 119 +++ dbgpt/agent/memory/gpts_memory.py | 95 +++ dbgpt/agent/memory/gpts_memory_storage.py | 443 ++++++++++ dbgpt/agent/{hub => plugin}/__init__.py | 0 dbgpt/agent/plugin/commands/__init__.py | 0 .../plugin/commands/built_in/__init__.py | 0 .../commands/built_in/audio_text.py | 2 +- .../commands/built_in/disply_type/__init__.py | 1 + .../built_in}/disply_type/show_chart_gen.py | 18 +- .../built_in}/disply_type/show_table_gen.py | 6 +- .../built_in}/disply_type/show_text_gen.py | 6 +- .../commands/built_in/image_gen.py | 4 +- dbgpt/agent/{ => plugin}/commands/command.py | 6 +- .../{ => plugin}/commands/command_mange.py | 141 +++- .../commands/exception_not_commands.py | 0 dbgpt/agent/{commands => plugin}/generator.py | 0 dbgpt/agent/plugin/plugin.py | 19 + dbgpt/agent/{ => plugin}/plugins_util.py | 10 +- dbgpt/app/base.py | 14 +- dbgpt/app/component_configs.py | 6 +- dbgpt/app/dbgpt_server.py | 24 +- .../initialization/db_model_initialization.py | 4 +- dbgpt/app/openapi/api_v1/api_v1.py | 1 + dbgpt/app/scene/chat_agent/chat.py | 6 +- .../chat_excel/excel_analyze/chat.py | 2 +- .../chat_excel/excel_learning/chat.py | 4 +- dbgpt/app/scene/chat_db/auto_execute/chat.py | 2 +- dbgpt/app/scene/chat_execution/chat.py | 4 +- dbgpt/component.py | 1 + dbgpt/core/interface/output_parser.py | 2 +- dbgpt/datasource/rdbms/base.py | 4 +- dbgpt/model/proxy/llms/chatgpt.py | 53 +- dbgpt/model/proxy/llms/tongyi.py | 1 + dbgpt/serve/agent/agents/__init__.py | 0 dbgpt/serve/agent/agents/controller.py | 338 ++++++++ dbgpt/serve/agent/agents/db_gpts_memory.py | 121 +++ dbgpt/serve/agent/agents/dbgpts.py | 93 +++ dbgpt/serve/agent/db/__init__.py | 6 + dbgpt/serve/agent/db/gpts_conversations_db.py | 95 +++ dbgpt/serve/agent/db/gpts_mange_db.py | 78 ++ dbgpt/serve/agent/db/gpts_messages_db.py | 160 ++++ dbgpt/serve/agent/db/gpts_plans_db.py | 156 ++++ dbgpt/{ => serve}/agent/db/my_plugin_db.py | 0 dbgpt/{ => serve}/agent/db/plugin_hub_db.py | 0 dbgpt/serve/agent/dbgpts/__init__.py | 0 dbgpt/serve/agent/hub/__init__.py | 0 dbgpt/{ => serve}/agent/hub/agent_hub.py | 5 +- .../{agent => serve/agent/hub}/controller.py | 11 +- dbgpt/{ => serve}/agent/model.py | 0 dbgpt/storage/metadata/meta_data.py | 0 dbgpt/util/code_utils.py | 468 +++++++++++ dbgpt/util/error_types.py | 16 + dbgpt/util/json_utils.py | 110 ++- dbgpt/util/string_utils.py | 9 + examples/agents/__init__.py | 0 .../auto_plan_agent_dialogue_example.py | 73 ++ .../agents/single_agent_dialogue_example.py | 48 ++ .../agent_auto_plan_dialogue_example.ipynb | 681 ++++++++++++++++ 79 files changed, 6356 insertions(+), 118 deletions(-) rename dbgpt/agent/{commands => agents}/__init__.py (100%) create mode 100644 dbgpt/agent/agents/agent.py create mode 100644 dbgpt/agent/agents/agents_mange.py create mode 100644 dbgpt/agent/agents/base_agent.py rename dbgpt/agent/{commands/built_in => agents/expand}/__init__.py (100%) create mode 100644 dbgpt/agent/agents/expand/code_assistant_agent.py create mode 100644 dbgpt/agent/agents/expand/dashboard_assistant_agent.py create mode 100644 dbgpt/agent/agents/expand/data_scientist_agent.py create mode 100644 dbgpt/agent/agents/expand/plugin_assistant_agent.py rename dbgpt/agent/{commands/disply_type/__init__.py => agents/expand/rag_assistant_agent.py} (100%) create mode 100644 dbgpt/agent/agents/expand/sql_assistant_agent.py rename dbgpt/agent/{db => agents/llm}/__init__.py (100%) create mode 100644 dbgpt/agent/agents/llm/llm.py create mode 100644 dbgpt/agent/agents/llm/llm_client.py create mode 100644 dbgpt/agent/agents/plan_group_chat.py create mode 100644 dbgpt/agent/agents/planner_agent.py create mode 100644 dbgpt/agent/agents/user_proxy_agent.py create mode 100644 dbgpt/agent/memory/base.py create mode 100644 dbgpt/agent/memory/default_gpts_memory.py create mode 100644 dbgpt/agent/memory/gpts_memory.py create mode 100644 dbgpt/agent/memory/gpts_memory_storage.py rename dbgpt/agent/{hub => plugin}/__init__.py (100%) create mode 100644 dbgpt/agent/plugin/commands/__init__.py create mode 100644 dbgpt/agent/plugin/commands/built_in/__init__.py rename dbgpt/agent/{ => plugin}/commands/built_in/audio_text.py (96%) create mode 100644 dbgpt/agent/plugin/commands/built_in/disply_type/__init__.py rename dbgpt/agent/{commands => plugin/commands/built_in}/disply_type/show_chart_gen.py (99%) rename dbgpt/agent/{commands => plugin/commands/built_in}/disply_type/show_table_gen.py (91%) rename dbgpt/agent/{commands => plugin/commands/built_in}/disply_type/show_text_gen.py (94%) rename dbgpt/agent/{ => plugin}/commands/built_in/image_gen.py (98%) rename dbgpt/agent/{ => plugin}/commands/command.py (98%) rename dbgpt/agent/{ => plugin}/commands/command_mange.py (77%) rename dbgpt/agent/{ => plugin}/commands/exception_not_commands.py (100%) rename dbgpt/agent/{commands => plugin}/generator.py (100%) create mode 100644 dbgpt/agent/plugin/plugin.py rename dbgpt/agent/{ => plugin}/plugins_util.py (100%) create mode 100644 dbgpt/serve/agent/agents/__init__.py create mode 100644 dbgpt/serve/agent/agents/controller.py create mode 100644 dbgpt/serve/agent/agents/db_gpts_memory.py create mode 100644 dbgpt/serve/agent/agents/dbgpts.py create mode 100644 dbgpt/serve/agent/db/__init__.py create mode 100644 dbgpt/serve/agent/db/gpts_conversations_db.py create mode 100644 dbgpt/serve/agent/db/gpts_mange_db.py create mode 100644 dbgpt/serve/agent/db/gpts_messages_db.py create mode 100644 dbgpt/serve/agent/db/gpts_plans_db.py rename dbgpt/{ => serve}/agent/db/my_plugin_db.py (100%) rename dbgpt/{ => serve}/agent/db/plugin_hub_db.py (100%) create mode 100644 dbgpt/serve/agent/dbgpts/__init__.py create mode 100644 dbgpt/serve/agent/hub/__init__.py rename dbgpt/{ => serve}/agent/hub/agent_hub.py (98%) rename dbgpt/{agent => serve/agent/hub}/controller.py (94%) rename dbgpt/{ => serve}/agent/model.py (100%) create mode 100644 dbgpt/storage/metadata/meta_data.py create mode 100644 dbgpt/util/code_utils.py create mode 100644 dbgpt/util/error_types.py create mode 100644 examples/agents/__init__.py create mode 100644 examples/agents/auto_plan_agent_dialogue_example.py create mode 100644 examples/agents/single_agent_dialogue_example.py create mode 100644 examples/notebook/agent_auto_plan_dialogue_example.ipynb diff --git a/.gitignore b/.gitignore index 50c82741b..15beea989 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ __pycache__/ *.so message/ - +dbgpt/util/extensions/ .env* .vscode .idea diff --git a/dbgpt/agent/__init__.py b/dbgpt/agent/__init__.py index 60f0489da..88c7ce02c 100644 --- a/dbgpt/agent/__init__.py +++ b/dbgpt/agent/__init__.py @@ -1,11 +1 @@ -from .db.my_plugin_db import MyPluginEntity, MyPluginDao -from .db.plugin_hub_db import PluginHubEntity, PluginHubDao - -from .commands.command import execute_command, get_command -from .commands.generator import PluginPromptGenerator -from .commands.disply_type.show_chart_gen import static_message_img_path - -from .common.schema import Status, PluginStorageType - -from .commands.command_mange import ApiCall -from .commands.command import execute_command +from .common.schema import PluginStorageType diff --git a/dbgpt/agent/commands/__init__.py b/dbgpt/agent/agents/__init__.py similarity index 100% rename from dbgpt/agent/commands/__init__.py rename to dbgpt/agent/agents/__init__.py diff --git a/dbgpt/agent/agents/agent.py b/dbgpt/agent/agents/agent.py new file mode 100644 index 000000000..ea31eb676 --- /dev/null +++ b/dbgpt/agent/agents/agent.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import dataclasses +from collections import defaultdict +from dataclasses import asdict, dataclass, fields +from typing import Any, Dict, List, Optional, Union + +from ..memory.gpts_memory import GptsMemory +from dbgpt.core import LLMClient +from dbgpt.core.interface.llm import ModelMetadata + + +class Agent: + """ + An interface for AI agent. + An agent can communicate with other agents and perform actions. + """ + + def __init__( + self, + name: str, + memory: GptsMemory, + describe: str, + ): + """ + Args: + name (str): name of the agent. + """ + self._name = name + self._describe = describe + + # the agent's collective memory + self._memory = memory + + @property + def name(self): + """Get the name of the agent.""" + return self._name + + @property + def memory(self): + return self._memory + + @property + def describe(self): + """Get the name of the agent.""" + return self._describe + + async def a_send( + self, + message: Union[Dict, str], + recipient: Agent, + reviewer: Agent, + request_reply: Optional[bool] = True, + is_recovery: Optional[bool] = False, + ): + """(Abstract async method) Send a message to another agent.""" + + async def a_receive( + self, + message: Optional[Dict], + sender: Agent, + reviewer: Agent, + request_reply: Optional[bool] = None, + silent: Optional[bool] = False, + is_recovery: Optional[bool] = False, + ): + """(Abstract async method) Receive a message from another agent.""" + + async def a_review(self, message: Union[Dict, str], censored: Agent): + """ + + Args: + message: + censored: + + Returns: + + """ + + def reset(self): + """(Abstract method) Reset the agent.""" + + async def a_generate_reply( + self, + message: Optional[Dict], + sender: Agent, + reviewer: Agent, + silent: Optional[bool] = False, + **kwargs, + ) -> Union[str, Dict, None]: + """(Abstract async method) Generate a reply based on the received messages. + + Args: + messages (Optional[Dict]): a dict of messages received from other agents. + sender: sender of an Agent instance. + Returns: + str or dict or None: the generated reply. If None, no reply is generated. + """ + + async def a_reasoning_reply( + self, messages: Optional[List[Dict]] + ) -> Union[str, Dict, None]: + """ + Based on the requirements of the current agent, reason about the current task goal through LLM + Args: + message: + + Returns: + str or dict or None: the generated reply. If None, no reply is generated. + """ + + async def a_action_reply( + self, + messages: Optional[str], + sender: Agent, + **kwargs, + ) -> Union[str, Dict, None]: + """ + Parse the inference results for the current target and execute the inference results using the current agent's executor + Args: + messages (list[dict]): a list of messages received. + sender: sender of an Agent instance. + **kwargs: + Returns: + str or dict or None: the agent action reply. If None, no reply is generated. + """ + + async def a_verify_reply( + self, + message: Optional[Dict], + sender: Agent, + reviewer: Agent, + **kwargs, + ) -> Union[str, Dict, None]: + """ + Verify whether the current execution results meet the target expectations + Args: + messages: + sender: + **kwargs: + + Returns: + + """ + + +@dataclass +class AgentResource: + type: str + name: str + introduce: str + + @staticmethod + def from_dict(d: Dict[str, Any]) -> Optional[AgentResource]: + if d is None: + return None + return AgentResource( + type=d.get("type"), + name=d.get("name"), + introduce=d.get("introduce"), + ) + + def to_dict(self) -> Dict[str, Any]: + return dataclasses.asdict(self) + + +@dataclass +class AgentContext: + conv_id: str + llm_provider: LLMClient + + gpts_name: Optional[str] = None + resource_db: Optional[AgentResource] = None + resource_knowledge: Optional[AgentResource] = None + resource_internet: Optional[AgentResource] = None + llm_models: Optional[List[Union[ModelMetadata, str]]] = None + model_priority: Optional[dict] = None + agents: Optional[List[str]] = None + + max_chat_round: Optional[int] = 100 + max_retry_round: Optional[int] = 10 + max_new_tokens: Optional[int] = 1024 + temperature: Optional[float] = 0.5 + allow_format_str_template: Optional[bool] = False + + def to_dict(self) -> Dict[str, Any]: + return dataclasses.asdict(self) diff --git a/dbgpt/agent/agents/agents_mange.py b/dbgpt/agent/agents/agents_mange.py new file mode 100644 index 000000000..eb20635ff --- /dev/null +++ b/dbgpt/agent/agents/agents_mange.py @@ -0,0 +1,48 @@ +from collections import defaultdict +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union + +from .agent import Agent +from .expand.code_assistant_agent import CodeAssistantAgent +from .expand.dashboard_assistant_agent import DashboardAssistantAgent +from .expand.data_scientist_agent import DataScientistAgent +from .expand.sql_assistant_agent import SQLAssistantAgent + + +def get_all_subclasses(cls): + all_subclasses = [] + direct_subclasses = cls.__subclasses__() + all_subclasses.extend(direct_subclasses) + + for subclass in direct_subclasses: + all_subclasses.extend(get_all_subclasses(subclass)) + return all_subclasses + + +class AgentsMange: + def __init__(self): + self._agents = defaultdict() + + def register_agent(self, cls): + self._agents[cls.NAME] = cls + + def get_by_name(self, name: str) -> Optional[Type[Agent]]: + if name not in self._agents: + raise ValueError(f"Agent:{name} not register!") + return self._agents[name] + + def get_describe_by_name(self, name: str) -> Optional[Type[Agent]]: + return self._agents[name].DEFAULT_DESCRIBE + + def all_agents(self): + result = {} + for name, cls in self._agents.items(): + result[name] = cls.DEFAULT_DESCRIBE + return result + + +agent_mange = AgentsMange() + +agent_mange.register_agent(CodeAssistantAgent) +agent_mange.register_agent(DashboardAssistantAgent) +agent_mange.register_agent(DataScientistAgent) +agent_mange.register_agent(SQLAssistantAgent) diff --git a/dbgpt/agent/agents/base_agent.py b/dbgpt/agent/agents/base_agent.py new file mode 100644 index 000000000..db0753d25 --- /dev/null +++ b/dbgpt/agent/agents/base_agent.py @@ -0,0 +1,768 @@ +import asyncio +import copy +import json +import logging +from typing import Any, Callable, Dict, List, Optional, Type, Union + +from dbgpt.agent.agents.llm.llm_client import AIWrapper +from dbgpt.core.awel import BaseOperator +from dbgpt.core.interface.message import ModelMessageRoleType +from dbgpt.util.error_types import LLMChatError + +from ..memory.base import GptsMessage +from ..memory.gpts_memory import GptsMemory +from .agent import Agent, AgentContext + +try: + from termcolor import colored +except ImportError: + + def colored(x, *args, **kwargs): + return x + + +logger = logging.getLogger(__name__) + + +class ConversableAgent(Agent): + DEFAULT_SYSTEM_MESSAGE = "You are a helpful AI Assistant." + MAX_CONSECUTIVE_AUTO_REPLY = ( + 100 # maximum number of consecutive auto replies (subject to future change) + ) + + def __init__( + self, + name: str, + describe: str = DEFAULT_SYSTEM_MESSAGE, + memory: GptsMemory = GptsMemory(), + agent_context: AgentContext = None, + system_message: Optional[str] = DEFAULT_SYSTEM_MESSAGE, + is_termination_msg: Optional[Callable[[Dict], bool]] = None, + max_consecutive_auto_reply: Optional[int] = None, + human_input_mode: Optional[str] = "TERMINATE", + default_auto_reply: Optional[Union[str, Dict, None]] = "", + ): + super().__init__(name, memory, describe) + + # a dictionary of conversations, default value is list + # self._oai_messages = defaultdict(list) + self._oai_system_message = [ + {"content": system_message, "role": ModelMessageRoleType.SYSTEM} + ] + self._rely_messages = [] + self._is_termination_msg = ( + is_termination_msg + if is_termination_msg is not None + else (lambda x: x.get("content") == "TERMINATE") + ) + + self.client = AIWrapper(llm_client=agent_context.llm_provider) + + self.human_input_mode = human_input_mode + self._max_consecutive_auto_reply = ( + max_consecutive_auto_reply + if max_consecutive_auto_reply is not None + else self.MAX_CONSECUTIVE_AUTO_REPLY + ) + self.consecutive_auto_reply_counter: int = 0 + + self._current_retry_counter: int = 0 + + ## By default, the memory of 4 rounds of dialogue is retained. + self.dialogue_memory_rounds = 5 + self._default_auto_reply = default_auto_reply + self._reply_func_list = [] + self._max_consecutive_auto_reply_dict = {} + + self.agent_context = agent_context + + def register_reply( + self, + trigger: Union[Type[Agent], str, Agent, Callable[[Agent], bool], List], + reply_func: Callable, + position: int = 0, + config: Optional[Any] = None, + reset_config: Optional[Callable] = None, + ): + if not isinstance(trigger, (type, str, Agent, Callable, list)): + raise ValueError( + "trigger must be a class, a string, an agent, a callable or a list." + ) + self._reply_func_list.insert( + position, + { + "trigger": trigger, + "reply_func": reply_func, + "config": copy.copy(config), + "init_config": config, + "reset_config": reset_config, + }, + ) + + @property + def system_message(self): + """Return the system message.""" + return self._oai_system_message[0]["content"] + + def update_system_message(self, system_message: str): + """Update the system message. + + Args: + system_message (str): system message for the ChatCompletion inference. + """ + self._oai_system_message[0]["content"] = system_message + + def update_max_consecutive_auto_reply( + self, value: int, sender: Optional[Agent] = None + ): + """Update the maximum number of consecutive auto replies. + + Args: + value (int): the maximum number of consecutive auto replies. + sender (Agent): when the sender is provided, only update the max_consecutive_auto_reply for that sender. + """ + if sender is None: + self._max_consecutive_auto_reply = value + for k in self._max_consecutive_auto_reply_dict: + self._max_consecutive_auto_reply_dict[k] = value + else: + self._max_consecutive_auto_reply_dict[sender] = value + + def max_consecutive_auto_reply(self, sender: Optional[Agent] = None) -> int: + """The maximum number of consecutive auto replies.""" + return ( + self._max_consecutive_auto_reply + if sender is None + else self._max_consecutive_auto_reply_dict[sender] + ) + + @property + # def chat_messages(self) -> Dict[Agent, List[Dict]]: + def chat_messages(self) -> Any: + """A dictionary of conversations from agent to list of messages.""" + all_gpts_messages = self.memory.message_memory.get_by_agent( + self.agent_context.conv_id, self.name + ) + return self._gpts_message_to_ai_message(all_gpts_messages) + + def last_message(self, agent: Optional[Agent] = None) -> Optional[Dict]: + """The last message exchanged with the agent. + + Args: + agent (Agent): The agent in the conversation. + If None and more than one agent's conversations are found, an error will be raised. + If None and only one conversation is found, the last message of the only conversation will be returned. + + Returns: + The last message exchanged with the agent. + """ + + if agent is None: + all_oai_messages = self.chat_messages() + n_conversations = len(all_oai_messages) + if n_conversations == 0: + return None + if n_conversations == 1: + for conversation in all_oai_messages.values(): + return conversation[-1] + raise ValueError( + "More than one conversation is found. Please specify the sender to get the last message." + ) + + agent_messages = self.memory.message_memory.get_between_agents( + self.agent_context.conv_id, self.name, agent.name + ) + if len(agent_messages) <= 0: + raise KeyError( + f"The agent '{agent.name}' is not present in any conversation. No history available for this agent." + ) + return self._gpts_message_to_ai_message(agent_messages)[-1] + + @staticmethod + def _message_to_dict(message: Union[Dict, str]): + """Convert a message to a dictionary. + + The message can be a string or a dictionary. The string will be put in the "content" field of the new dictionary. + """ + if isinstance(message, str): + return {"content": message} + elif isinstance(message, dict): + return message + else: + return dict(message) + + def append_rely_message(self, message: Union[Dict, str], role) -> None: + message = self._message_to_dict(message) + message["role"] = role + # create oai message to be appended to the oai conversation that can be passed to oai directly. + self._rely_messages.append(message) + + def reset_rely_message(self) -> None: + # create oai message to be appended to the oai conversation that can be passed to oai directly. + self._rely_messages = [] + + def append_message(self, message: Optional[Dict], role, sender: Agent) -> bool: + """ + Put the received message content into the collective message memory + Args: + conv_id: + message: + role: + sender: + + Returns: + + """ + oai_message = { + k: message[k] + for k in ( + "content", + "function_call", + "name", + "context", + "action_report", + "review_info", + "current_gogal", + "model_name", + ) + if k in message + } + if "content" not in oai_message: + if "function_call" in oai_message: + oai_message[ + "content" + ] = None # if only function_call is provided, content will be set to None. + else: + return False + oai_message["role"] = "function" if message.get("role") == "function" else role + if "function_call" in oai_message: + oai_message[ + "role" + ] = "assistant" # only messages with role 'assistant' can have a function call. + oai_message["function_call"] = dict(oai_message["function_call"]) + + gpts_message: GptsMessage = GptsMessage( + conv_id=self.agent_context.conv_id, + sender=sender.name, + receiver=self.name, + role=role, + rounds=self.consecutive_auto_reply_counter, + current_gogal=oai_message.get("current_gogal", None), + content=oai_message.get("content", None), + context=json.dumps(oai_message["context"]) + if "context" in oai_message + else None, + review_info=json.dumps(oai_message["review_info"]) + if "review_info" in oai_message + else None, + action_report=json.dumps(oai_message["action_report"]) + if "action_report" in oai_message + else None, + model_name=oai_message.get("model_name", None), + ) + + self.memory.message_memory.append(gpts_message) + return True + + async def a_send( + self, + message: Optional[Dict], + recipient: Agent, + reviewer: "Agent", + request_reply: Optional[bool] = True, + silent: Optional[bool] = False, + is_recovery: Optional[bool] = False, + ): + await recipient.a_receive( + message=message, + sender=self, + reviewer=reviewer, + request_reply=request_reply, + silent=silent, + is_recovery=is_recovery, + ) + + def _print_received_message(self, message: Union[Dict, str], sender: Agent): + # print the message received + print( + colored(sender.name, "yellow"), + "(to", + f"{self.name})-[{message.get('model_name', '')}]:\n", + flush=True, + ) + message = self._message_to_dict(message) + + if message.get("role") == "function": + func_print = ( + f"***** Response from calling function \"{message['name']}\" *****" + ) + print(colored(func_print, "green"), flush=True) + print(message["content"], flush=True) + print(colored("*" * len(func_print), "green"), flush=True) + else: + content = json.dumps(message.get("content")) + if content is not None: + if "context" in message: + content = AIWrapper.instantiate( + content, + message["context"], + self.agent_context.allow_format_str_template, + ) + print(content, flush=True) + if "function_call" in message: + function_call = dict(message["function_call"]) + func_print = f"***** Suggested function Call: {function_call.get('name', '(No function name found)')} *****" + print(colored(func_print, "green"), flush=True) + print( + "Arguments: \n", + function_call.get("arguments", "(No arguments found)"), + flush=True, + sep="", + ) + print(colored("*" * len(func_print), "green"), flush=True) + + review_info = message.get("review_info", None) + if review_info: + approve_print = f">>>>>>>>{sender.name} Review info: \n {'Pass' if review_info.get('approve') else 'Reject'}.{review_info.get('comments')}" + print(colored(approve_print, "green"), flush=True) + + action_report = message.get("action_report", None) + if action_report: + action_print = f">>>>>>>>{sender.name} Action report: \n{'execution succeeded' if action_report['is_exe_success'] else 'execution failed'},\n{action_report['content']}" + print(colored(action_print, "blue"), flush=True) + print("\n", "-" * 80, flush=True, sep="") + + def _process_received_message(self, message, sender, silent): + message = self._message_to_dict(message) + # When the agent receives a message, the role of the message is "user". (If 'role' exists and is 'function', it will remain unchanged.) + valid = self.append_message(message, None, sender) + if not valid: + raise ValueError( + "Received message can't be converted into a valid ChatCompletion message. Either content or function_call must be provided." + ) + if not silent: + self._print_received_message(message, sender) + + async def a_review(self, message: Union[Dict, str], censored: "Agent"): + return True, None + + def _process_action_reply(self, action_reply: Optional[Union[str, Dict, None]]): + if isinstance(action_reply, str): + return {"is_exe_success": True, "content": action_reply} + elif isinstance(action_reply, dict): + return action_reply + elif action_reply is None: + return None + else: + return dict(action_reply) + + def _gpts_message_to_ai_message( + self, gpts_messages: Optional[List[GptsMessage]] + ) -> List[Dict]: + oai_messages: List[Dict] = [] + ###Based on the current agent, all messages received are user, and all messages sent are assistant. + for item in gpts_messages: + role = "" + if item.role: + role = role + else: + if item.receiver == self.name: + role = ModelMessageRoleType.HUMAN + elif item.sender == self.name: + role = ModelMessageRoleType.AI + else: + continue + oai_messages.append( + { + "content": item.content, + "role": role, + "context": json.loads(item.context) + if item.context is not None + else None, + "review_info": json.loads(item.review_info) + if item.review_info is not None + else None, + "action_report": json.loads(item.action_report) + if item.action_report is not None + else None, + } + ) + return oai_messages + + def process_now_message(self, sender, current_gogal: Optional[str] = None): + ### Convert and tailor the information in collective memory into contextual memory available to the current Agent + current_gogal_messages = self._gpts_message_to_ai_message( + self.memory.message_memory.get_between_agents( + self.agent_context.conv_id, self.name, sender.name, current_gogal + ) + ) + + ### relay messages + cut_messages = [] + cut_messages.extend(self._rely_messages) + + if len(current_gogal_messages) < self.dialogue_memory_rounds: + cut_messages.extend(current_gogal_messages) + else: + ### TODO 基于token预算来分配历史信息 + cut_messages.extend(current_gogal_messages[:2]) + # end_round = self.dialogue_memory_rounds - 2 + cut_messages.extend(current_gogal_messages[-3:]) + return cut_messages + + async def a_system_fill_param(self): + self.update_system_message(self.DEFAULT_SYSTEM_MESSAGE) + + async def a_generate_reply( + self, + message: Optional[Dict], + sender: Agent, + reviewer: "Agent", + silent: Optional[bool] = False, + ): + ## 0.New message build + new_message = {} + new_message["context"] = message.get("context", None) + new_message["current_gogal"] = message.get("current_gogal", None) + + ## 1.LLM Reasonging + await self.a_system_fill_param() + await asyncio.sleep(5) ##TODO Rate limit reached for gpt-3.5-turbo + current_messages = self.process_now_message( + sender, message.get("current_gogal", None) + ) + if current_messages is None or len(current_messages) <= 0: + current_messages = [message] + ai_reply, model = await self.a_reasoning_reply(messages=current_messages) + new_message["content"] = ai_reply + new_message["model_name"] = model + ## 2.Review of reasoning results + approve = True + comments = None + if reviewer and ai_reply: + approve, comments = await reviewer.a_review(ai_reply, self) + new_message["review_info"] = {"approve": approve, "comments": comments} + ## 3.reasoning result action + if approve: + excute_reply = await self.a_action_reply( + message=ai_reply, + sender=sender, + reviewer=reviewer, + ) + new_message["action_report"] = self._process_action_reply(excute_reply) + ## 4.verify reply + return await self.a_verify_reply(new_message, sender, reviewer) + + async def a_receive( + self, + message: Optional[Dict], + sender: Agent, + reviewer: "Agent", + request_reply: Optional[bool] = True, + silent: Optional[bool] = False, + is_recovery: Optional[bool] = False, + ): + if not is_recovery: + self.consecutive_auto_reply_counter = ( + sender.consecutive_auto_reply_counter + 1 + ) + self._process_received_message(message, sender, silent) + + else: + logger.info("Process received retrying") + self.consecutive_auto_reply_counter = sender.consecutive_auto_reply_counter + if request_reply is False or request_reply is None: + logger.info("Messages that do not require a reply") + return + + verify_paas, reply = await self.a_generate_reply( + message=message, sender=sender, reviewer=reviewer, silent=silent + ) + + if verify_paas: + await self.a_send( + message=reply, recipient=sender, reviewer=reviewer, silent=silent + ) + else: + self._current_retry_counter += 1 + logger.info( + "The generated answer failed to verify, so send it to yourself for optimization." + ) + ### TODO 自优化最大轮次后，异常退出 + await sender.a_send( + message=reply, recipient=self, reviewer=reviewer, silent=silent + ) + + async def a_verify(self, message: Optional[Dict]): + return True, message + + async def _optimization_check(self, message: Optional[Dict]): + need_retry = False + fail_reason = "" + ## Check approval results + if "review_info" in message: + review_info = message.get("review_info") + if review_info and not review_info.get("approve"): + fail_reason = review_info.get("comments") + need_retry = True + ## Check execution results + if "action_report" in message and not need_retry: + action_report = message["action_report"] + if action_report: + if not action_report["is_exe_success"]: + fail_reason = action_report["content"] + need_retry = True + else: + if ( + not action_report["content"] + or len(action_report["content"].strip()) < 1 + ): + fail_reason = f'The code is executed successfully but the output:{action_report["content"]} is invalid or empty. Please reanalyze the target to generate valid code.' + need_retry = True + ## Verify the correctness of the results + if not need_retry: + verify_pass, verfiy_msg = await self.a_verify(message) + if not verify_pass: + need_retry = True + fail_reason = verfiy_msg + return need_retry, fail_reason + + async def a_verify_reply( + self, message: Optional[Dict], sender: "Agent", reviewer: "Agent", **kwargs + ) -> Union[str, Dict, None]: + need_retry, fail_reason = await self._optimization_check(message) + if need_retry: + ## Before optimization, wrong answers are stored in memory + await self.a_send( + message=message, + recipient=sender, + reviewer=reviewer, + request_reply=False, + ) + ## Send error messages to yourself for retrieval optimization and increase the number of retrievals + retry_message = {} + retry_message["context"] = message.get("context", None) + retry_message["current_gogal"] = message.get("current_gogal", None) + retry_message["model_name"] = message.get("model_name", None) + retry_message["content"] = fail_reason + ## Use the original sender to send the retry message to yourself + return False, retry_message + else: + ## The verification passes, the message is released, and the number of retries returns to 0. + self._current_retry_counter = 0 + return True, message + + async def a_retry_chat( + self, + recipient: "ConversableAgent", + agent_map: dict, + reviewer: "Agent" = None, + clear_history: Optional[bool] = True, + silent: Optional[bool] = False, + **context, + ): + last_message: GptsMessage = self.memory.message_memory.get_last_message( + self.agent_context.conv_id + ) + self.consecutive_auto_reply_counter = last_message.rounds + message = { + "content": last_message.content, + "context": json.loads(last_message.context) + if last_message.context + else None, + "current_gogal": last_message.current_gogal, + "review_info": json.loads(last_message.review_info) + if last_message.review_info + else None, + "action_report": json.loads(last_message.action_report) + if last_message.action_report + else None, + "model_name": last_message.model_name, + } + await self.a_send( + message, + recipient, + reviewer, + request_reply=True, + silent=silent, + is_recovery=True, + ) + + async def a_initiate_chat( + self, + recipient: "ConversableAgent", + reviewer: "Agent" = None, + clear_history: Optional[bool] = True, + silent: Optional[bool] = False, + **context, + ): + await self.a_send( + { + "content": self.generate_init_message(**context), + "current_gogal": self.generate_init_message(**context), + }, + recipient, + reviewer, + request_reply=True, + silent=silent, + ) + + def reset(self): + """Reset the agent.""" + self.clear_history() + self.reset_consecutive_auto_reply_counter() + + for reply_func_tuple in self._reply_func_list: + if reply_func_tuple["reset_config"] is not None: + reply_func_tuple["reset_config"](reply_func_tuple["config"]) + else: + reply_func_tuple["config"] = copy.copy(reply_func_tuple["init_config"]) + + def reset_consecutive_auto_reply_counter(self): + """Reset the consecutive_auto_reply_counter of the sender.""" + self.consecutive_auto_reply_counter = 0 + + def clear_history(self, agent: Optional[Agent] = None): + """Clear the chat history of the agent. + + Args: + agent: the agent with whom the chat history to clear. If None, clear the chat history with all agents. + """ + pass + # if agent is None: + # self._oai_messages.clear() + # else: + # self._oai_messages[agent].clear() + + def _get_model_priority(self): + llm_models_priority = self.agent_context.model_priority + if llm_models_priority: + if self.name in llm_models_priority: + model_priority = llm_models_priority[self.name] + else: + model_priority = llm_models_priority["default"] + return model_priority + else: + return None + + def _filter_health_models(self, need_uses: Optional[list]): + all_modes = self.agent_context.llm_models + can_uses = [] + for item in all_modes: + if item.model in need_uses: + can_uses.append(item) + return can_uses + + def _select_llm_model(self, old_model: str = None): + """ + LLM model selector, currently only supports manual selection, more strategies will be opened in the future + Returns: + """ + all_modes = self.agent_context.llm_models + model_priority = self._get_model_priority() + if model_priority and len(model_priority) > 0: + can_uses = self._filter_health_models(model_priority) + if len(can_uses) > 0: + return can_uses[0].model + + now_model = all_modes[0] + if old_model: + filtered_list = [item for item in all_modes if item.model != old_model] + if filtered_list and len(filtered_list) >= 1: + now_model = filtered_list[0] + return now_model.model + + async def a_reasoning_reply( + self, messages: Optional[List[Dict]] = None + ) -> Union[str, Dict, None]: + """(async) Reply based on the conversation history and the sender. + Args: + messages: a list of messages in the conversation history. + default_reply (str or dict): default reply. + sender: sender of an Agent instance. + exclude: a list of functions to exclude. + + Returns: + str or dict or None: reply. None if no reply is generated. + """ + last_model = None + last_err = None + retry_count = 0 + while retry_count < 3: + llm_model = self._select_llm_model(last_model) + try: + response = await self.client.create( + context=messages[-1].pop("context", None), + messages=self._oai_system_message + messages, + llm_model=llm_model, + max_new_tokens=self.agent_context.max_new_tokens, + temperature=self.agent_context.temperature, + ) + return response, llm_model + except LLMChatError as e: + logger.error(f"model:{llm_model} generate Failed!{str(e)}") + retry_count += 1 + last_model = llm_model + last_err = str(e) + await asyncio.sleep(10) ## TODO，Rate limit reached for gpt-3.5-turbo + + if last_err: + raise ValueError(last_err) + + async def a_action_reply( + self, + message: Optional[str] = None, + sender: Optional[Agent] = None, + reviewer: "Agent" = None, + exclude: Optional[List[Callable]] = None, + **kwargs, + ) -> Union[str, Dict, None]: + for reply_func_tuple in self._reply_func_list: + reply_func = reply_func_tuple["reply_func"] + if exclude and reply_func in exclude: + continue + if self._match_trigger(reply_func_tuple["trigger"], sender): + if asyncio.coroutines.iscoroutinefunction(reply_func): + final, reply = await reply_func( + self, + message=message, + sender=sender, + reviewer=reviewer, + config=reply_func_tuple["config"], + ) + else: + final, reply = reply_func( + self, + message=message, + sender=sender, + reviewer=reviewer, + config=reply_func_tuple["config"], + ) + if final: + return reply + return self._default_auto_reply + + def _match_trigger(self, trigger, sender): + """Check if the sender matches the trigger.""" + if trigger is None: + return sender is None + elif isinstance(trigger, str): + return trigger == sender.name + elif isinstance(trigger, type): + return isinstance(sender, trigger) + elif isinstance(trigger, Agent): + return trigger == sender + elif isinstance(trigger, Callable): + return trigger(sender) + elif isinstance(trigger, list): + return any(self._match_trigger(t, sender) for t in trigger) + else: + raise ValueError(f"Unsupported trigger type: {type(trigger)}") + + def generate_init_message(self, **context) -> Union[str, Dict]: + """Generate the initial message for the agent. + + Override this function to customize the initial message based on user's request. + If not overridden, "message" needs to be provided in the context. + """ + return context["message"] diff --git a/dbgpt/agent/commands/built_in/__init__.py b/dbgpt/agent/agents/expand/__init__.py similarity index 100% rename from dbgpt/agent/commands/built_in/__init__.py rename to dbgpt/agent/agents/expand/__init__.py diff --git a/dbgpt/agent/agents/expand/code_assistant_agent.py b/dbgpt/agent/agents/expand/code_assistant_agent.py new file mode 100644 index 000000000..89d60209f --- /dev/null +++ b/dbgpt/agent/agents/expand/code_assistant_agent.py @@ -0,0 +1,261 @@ +import json +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union + +from dbgpt.core.awel import BaseOperator +from dbgpt.util.code_utils import UNKNOWN, execute_code, extract_code, infer_lang +from dbgpt.util.string_utils import str_to_bool + +from ...memory.gpts_memory import GptsMemory +from ..agent import Agent, AgentContext +from ..base_agent import ConversableAgent +from dbgpt.core.interface.message import ModelMessageRoleType + +try: + from termcolor import colored +except ImportError: + + def colored(x, *args, **kwargs): + return x + + +class CodeAssistantAgent(ConversableAgent): + """(In preview) Assistant agent, designed to solve a task with LLM. + + AssistantAgent is a subclass of ConversableAgent configured with a default system message. + The default system message is designed to solve a task with LLM, + including suggesting python code blocks and debugging. + `human_input_mode` is default to "NEVER" + and `code_execution_config` is default to False. + This agent doesn't execute code by default, and expects the user to execute the code. + """ + + DEFAULT_SYSTEM_MESSAGE = """You are a helpful AI assistant. + Solve tasks using your coding and language skills. + In the following cases, suggest python code (in a python coding block) or shell script (in a sh coding block) for the user to execute. + 1. When you need to collect info, use the code to output the info you need, for example, browse or search the web, download/read a file, print the content of a webpage or a file, get the current date/time, check the operating system. After sufficient info is printed and the task is ready to be solved based on your language skill, you can solve the task by yourself. + 2. When you need to perform some task with code, use the code to perform the task and output the result. Finish the task smartly. + + *** IMPORTANT REMINDER *** + - The user cannot provide any other feedback or perform any other action beyond executing the code you suggest. The user can't modify your code. So do not suggest incomplete code which requires users to modify. Don't use a code block if it's not intended to be executed by the user.Don't ask users to copy and paste results. Instead, the "Print" function must be used for output when relevant. + - When using code, you must indicate the script type in the code block. Please don't include multiple code blocks in one response. + - If you want the user to save the code in a file before executing it, put # filename: inside the code block as the first line. + - If you receive user input that indicates an error in the code execution, fix the error and output the complete code again. It is recommended to use the complete code rather than partial code or code changes. If the error cannot be fixed, or the task is not resolved even after the code executes successfully, analyze the problem, revisit your assumptions, gather additional information you need from historical conversation records, and consider trying a different approach + - Unless necessary, give priority to solving problems with python code. If it involves downloading files or storing data locally, please use "Print" to output the full file path of the stored data and a brief introduction to the data. + - The output content of the "Print" function will be passed to other LLM agents. Please ensure that the information output by the "Print" function has been streamlined as much as possible and only retains key data information. + - The code is executed without user participation. It is forbidden to use methods that will block the process or need to be shut down, such as the plt.show() method of matplotlib.pyplot as plt. + """ + CHECK_RESULT_SYSTEM_MESSAGE = f""" + You are an expert in analyzing the results of task execution. + Your responsibility is to analyze the task goals and execution results provided by the user, and then make a judgment. You need to answer according to the following rules: + Rule 1: Analysis and judgment only focus on whether the execution result is related to the task goal and whether it is answering the target question, but does not pay attention to whether the result content is reasonable or the correctness of the scope boundary of the answer content. + Rule 2: If the target is a calculation type, there is no need to verify the correctness of the calculation of the values ​​in the execution result. + As long as the execution result meets the task goal according to the above rules, True will be returned, otherwise False will be returned. Only returns True or False. + """ + + NAME = "CodeEngineer" + DEFAULT_DESCRIBE = """According to the current planning steps, write python/shell code to solve the problem, such as: data crawling, data sorting and conversion, etc. Wrap the code in a code block of the specified script type. Users cannot modify your code. So don't suggest incomplete code that needs to be modified by others. + Don't include multiple code blocks in one response. Don't ask others to copy and paste the results + """ + + def __init__( + self, + agent_context: AgentContext, + memory: Optional[GptsMemory] = None, + describe: Optional[str] = DEFAULT_DESCRIBE, + is_termination_msg: Optional[Callable[[Dict], bool]] = None, + max_consecutive_auto_reply: Optional[int] = None, + human_input_mode: Optional[str] = "NEVER", + code_execution_config: Optional[Union[Dict, Literal[False]]] = None, + **kwargs, + ): + """ + Args: + name (str): agent name. + system_message (str): system message for the ChatCompletion inference. + Please override this attribute if you want to reprogram the agent. + llm_config (dict): llm inference configuration. + Please refer to [OpenAIWrapper.create](/docs/reference/oai/client#create) + for available options. + is_termination_msg (function): a function that takes a message in the form of a dictionary + and returns a boolean value indicating if this received message is a termination message. + The dict can contain the following keys: "content", "role", "name", "function_call". + max_consecutive_auto_reply (int): the maximum number of consecutive auto replies. + default to None (no limit provided, class attribute MAX_CONSECUTIVE_AUTO_REPLY will be used as the limit in this case). + The limit only plays a role when human_input_mode is not "ALWAYS". + **kwargs (dict): Please refer to other kwargs in + [ConversableAgent](conversable_agent#__init__). + """ + super().__init__( + name=self.NAME, + memory=memory, + describe=describe, + system_message=self.DEFAULT_SYSTEM_MESSAGE, + is_termination_msg=is_termination_msg, + max_consecutive_auto_reply=max_consecutive_auto_reply, + human_input_mode=human_input_mode, + agent_context=agent_context, + **kwargs, + ) + self._code_execution_config: Union[Dict, Literal[False]] = ( + {} if code_execution_config is None else code_execution_config + ) + ### register code funtion + self.register_reply(Agent, CodeAssistantAgent.generate_code_execution_reply) + + def _vis_code_idea(self, code, exit_success, log, language): + param = {} + param["exit_success"] = exit_success + param["language"] = language + param["code"] = code + param["log"] = log + + return f"```vis-code\n{json.dumps(param)}\n```" + + async def generate_code_execution_reply( + self, + message: Optional[str] = None, + sender: Optional[Agent] = None, + reviewer: Optional[Agent] = None, + config: Optional[Union[Dict, Literal[False]]] = None, + ): + """Generate a reply using code execution.""" + code_execution_config = ( + config if config is not None else self._code_execution_config + ) + if code_execution_config is False: + return False, None + + last_n_messages = code_execution_config.pop("last_n_messages", 1) + + # iterate through the last n messages reversly + # if code blocks are found, execute the code blocks and return the output + # if no code blocks are found, continue + + code_blocks = extract_code(message) + + if len(code_blocks) < 1: + self.send( + f"Failed to get valid answer,{message}", sender, None, silent=True + ) + elif len(code_blocks) > 1 and code_blocks[0][0] == UNKNOWN: + self.send( + f"Failed to get valid answer,{message}", self, reviewer, silent=True + ) + + # found code blocks, execute code and push "last_n_messages" back + exitcode, logs = self.execute_code_blocks(code_blocks) + code_execution_config["last_n_messages"] = last_n_messages + exit_success = True if exitcode == 0 else False + if exit_success: + return True, { + "is_exe_success": exit_success, + "content": f"{logs}", + "view": self._vis_code_idea( + code_blocks, exit_success, logs, code_blocks[0][0] + ), + } + else: + return True, { + "is_exe_success": exit_success, + "content": f"exitcode: {exitcode} (execution failed)\n {logs}", + "view": self._vis_code_idea( + code_blocks, exit_success, logs, code_blocks[0][0] + ), + } + + async def a_verify(self, message: Optional[Dict]): + self.update_system_message(self.CHECK_RESULT_SYSTEM_MESSAGE) + task_gogal = message.get("current_gogal", None) + action_report = message.get("action_report", None) + task_result = "" + if action_report: + task_result = action_report.get("content", "") + + check_reult, model = await self.a_reasoning_reply( + [ + { + "role": ModelMessageRoleType.HUMAN, + "content": f"""Please understand the following task objectives and results and give your judgment: + Task Gogal: {task_gogal} + Execution Result: {task_result} + Only True or False is returned. + """, + } + ] + ) + sucess = str_to_bool(check_reult) + fail_reason = None + if sucess == False: + fail_reason = "The execution result of the code you wrote is judged as not answering the task question. Please re-understand and complete the task." + return sucess, fail_reason + + @property + def use_docker(self) -> Union[bool, str, None]: + """Bool value of whether to use docker to execute the code, + or str value of the docker image name to use, or None when code execution is disabled. + """ + return ( + None + if self._code_execution_config is False + else self._code_execution_config.get("use_docker") + ) + + def run_code(self, code, **kwargs): + """Run the code and return the result. + + Override this function to modify the way to run the code. + Args: + code (str): the code to be executed. + **kwargs: other keyword arguments. + + Returns: + A tuple of (exitcode, logs, image). + exitcode (int): the exit code of the code execution. + logs (str): the logs of the code execution. + image (str or None): the docker image used for the code execution. + """ + return execute_code(code, **kwargs) + + def execute_code_blocks(self, code_blocks): + """Execute the code blocks and return the result.""" + logs_all = "" + for i, code_block in enumerate(code_blocks): + lang, code = code_block + if not lang: + lang = infer_lang(code) + print( + colored( + f"\n>>>>>>>> EXECUTING CODE BLOCK {i} (inferred language is {lang})...", + "red", + ), + flush=True, + ) + if lang in ["bash", "shell", "sh"]: + exitcode, logs, image = self.run_code( + code, lang=lang, **self._code_execution_config + ) + elif lang in ["python", "Python"]: + if code.startswith("# filename: "): + filename = code[11 : code.find("\n")].strip() + else: + filename = None + exitcode, logs, image = self.run_code( + code, + lang="python", + filename=filename, + **self._code_execution_config, + ) + else: + # In case the language is not supported, we return an error message. + exitcode, logs, image = ( + 1, + f"unknown language {lang}", + None, + ) + # raise NotImplementedError + if image is not None: + self._code_execution_config["use_docker"] = image + logs_all += "\n" + logs + if exitcode != 0: + return exitcode, logs_all + return exitcode, logs_all diff --git a/dbgpt/agent/agents/expand/dashboard_assistant_agent.py b/dbgpt/agent/agents/expand/dashboard_assistant_agent.py new file mode 100644 index 000000000..88c0b03a5 --- /dev/null +++ b/dbgpt/agent/agents/expand/dashboard_assistant_agent.py @@ -0,0 +1,121 @@ +import json +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union + +from dbgpt.agent.plugin.commands.command_mange import ApiCall +from dbgpt.util.json_utils import find_json_objects + +from ...memory.gpts_memory import GptsMemory +from ..agent import Agent, AgentContext +from ..base_agent import ConversableAgent + +try: + from termcolor import colored +except ImportError: + + def colored(x, *args, **kwargs): + return x + + +from dbgpt._private.config import Config +from dbgpt.core.awel import BaseOperator + +CFG = Config() + + +class DashboardAssistantAgent(ConversableAgent): + """(In preview) Assistant agent, designed to solve a task with LLM. + + AssistantAgent is a subclass of ConversableAgent configured with a default system message. + The default system message is designed to solve a task with LLM, + including suggesting python code blocks and debugging. + `human_input_mode` is default to "NEVER" + and `code_execution_config` is default to False. + This agent doesn't execute code by default, and expects the user to execute the code. + """ + + DEFAULT_SYSTEM_MESSAGE = """Please read the historical messages, collect the generated JSON data of the analysis sql results, and integrate it into the following JSON format to return: + [ + {{ + "display_type":"The chart rendering method selected for the task 1 sql", + "sql": "Analysis sql of the step task 1", + "thought":"thoughts summary to say to user" + }}, + {{ + "display_type":"The chart rendering method selected for the task 2 sql", + "sql": "Analysis sql of the step task 2", + "thought":"thoughts summary to say to user" + }} + ] + Make sure the response is correct json and can be parsed by Python json.loads. + """ + DEFAULT_DESCRIBE = "Integrate analytical data generated by data scientists into a required format for building sales reports." + NAME = "Reporter" + + def __init__( + self, + memory: GptsMemory, + agent_context: AgentContext, + describe: Optional[str] = DEFAULT_DESCRIBE, + is_termination_msg: Optional[Callable[[Dict], bool]] = None, + max_consecutive_auto_reply: Optional[int] = None, + human_input_mode: Optional[str] = "NEVER", + **kwargs, + ): + super().__init__( + name=self.NAME, + memory=memory, + describe=describe, + system_message=self.DEFAULT_SYSTEM_MESSAGE, + is_termination_msg=is_termination_msg, + max_consecutive_auto_reply=max_consecutive_auto_reply, + human_input_mode=human_input_mode, + agent_context=agent_context, + **kwargs, + ) + self.register_reply(Agent, DashboardAssistantAgent.generate_dashboard_reply) + self.agent_context = agent_context + self.db_connect = CFG.LOCAL_DB_MANAGE.get_connect( + self.agent_context.resource_db.get("name", None) + ) + + async def generate_dashboard_reply( + self, + message: Optional[str] = None, + sender: Optional[Agent] = None, + reviewer: Optional[Agent] = None, + config: Optional[Union[Dict, Literal[False]]] = None, + ): + """Generate a reply using code execution.""" + + json_objects = find_json_objects(message) + plan_objects = [] + fail_reason = ( + "Please recheck your answer，no usable plans generated in correct format，" + ) + json_count = len(json_objects) + rensponse_succ = True + view = None + content = None + if json_count != 1: + ### Answer failed, turn on automatic repair + fail_reason += f"There are currently {json_count} json contents" + rensponse_succ = False + else: + try: + chart_objs = json_objects[0] + content = json.dumps(chart_objs) + vis_client = ApiCall() + view = vis_client.display_dashboard_vis( + chart_objs, self.db_connect.run_to_df + ) + except Exception as e: + fail_reason += f"Return json structure error and cannot be converted to a sql-rendered chart，{str(e)}" + rensponse_succ = False + + if not rensponse_succ: + content = fail_reason + return True, { + "is_exe_success": rensponse_succ, + "content": content, + "view": view, + } diff --git a/dbgpt/agent/agents/expand/data_scientist_agent.py b/dbgpt/agent/agents/expand/data_scientist_agent.py new file mode 100644 index 000000000..3574009cf --- /dev/null +++ b/dbgpt/agent/agents/expand/data_scientist_agent.py @@ -0,0 +1,162 @@ +import json +import logging +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union + +from dbgpt._private.config import Config +from dbgpt.agent.plugin.commands.command_mange import ApiCall +from dbgpt.core.awel import BaseOperator +from dbgpt.util.json_utils import find_json_objects + +from ...memory.gpts_memory import GptsMemory +from ..agent import Agent, AgentContext +from ..base_agent import ConversableAgent + +try: + from termcolor import colored +except ImportError: + + def colored(x, *args, **kwargs): + return x + + +CFG = Config() +logger = logging.getLogger(__name__) + + +class DataScientistAgent(ConversableAgent): + """(In preview) Assistant agent, designed to solve a task with LLM. + + AssistantAgent is a subclass of ConversableAgent configured with a default system message. + The default system message is designed to solve a task with LLM, + including suggesting python code blocks and debugging. + `human_input_mode` is default to "NEVER" + and `code_execution_config` is default to False. + This agent doesn't execute code by default, and expects the user to execute the code. + """ + + DEFAULT_SYSTEM_MESSAGE = """You are a helpful AI assistant who is good at writing SQL for various databases. + Based on the given data structure information, use the correct {dialect} SQL to analyze and solve the task, subject to the following constraints. + Data structure information: + {data_structure} + constraint: + 1. Please choose the best one from the display methods given below for data display, and put the type name into the name parameter value that returns the required format. If you can't find the most suitable display method, use Table as the display method. , the available data display methods are as follows: {disply_type} + 2. Please check the sql you generated. It is forbidden to use column names that do not exist in the table, and it is forbidden to make up fields and tables that do not exist. + 3. Pay attention to the data association between tables and tables, and you can use multiple tables at the same time to generate a SQL. + Please think step by step and return it in the following json format + {{ + "display_type":"The chart rendering method currently selected by SQL", + "sql": "Analysis sql of the current step task", + "thought":"Summary of thoughts to the user" + }} + Make sure the response is correct json and can be parsed by Python json.loads. + """ + DEFAULT_DESCRIBE = """It is possible use the local database to generate analysis SQL to obtain data based on the table structure, and at the same time generate visual charts of the corresponding data. Note that only local databases can be queried.""" + NAME = "DataScientist" + + def __init__( + self, + memory: GptsMemory, + agent_context: AgentContext, + describe: Optional[str] = DEFAULT_DESCRIBE, + is_termination_msg: Optional[Callable[[Dict], bool]] = None, + max_consecutive_auto_reply: Optional[int] = None, + human_input_mode: Optional[str] = "NEVER", + **kwargs, + ): + super().__init__( + name=self.NAME, + memory=memory, + describe=describe, + system_message=self.DEFAULT_SYSTEM_MESSAGE, + is_termination_msg=is_termination_msg, + max_consecutive_auto_reply=max_consecutive_auto_reply, + human_input_mode=human_input_mode, + agent_context=agent_context, + **kwargs, + ) + + self.register_reply(Agent, DataScientistAgent.generate_analysis_chart_reply) + self.agent_context = agent_context + self.db_connect = CFG.LOCAL_DB_MANAGE.get_connect( + self.agent_context.resource_db.get("name", None) + ) + + async def a_system_fill_param(self): + params = { + "data_structure": self.db_connect.get_table_info(), + "disply_type": ApiCall.default_chart_type_promot(), + "dialect": self.db_connect.db_type, + } + self.update_system_message(self.DEFAULT_SYSTEM_MESSAGE.format(**params)) + + async def generate_analysis_chart_reply( + self, + message: Optional[str] = None, + sender: Optional[Agent] = None, + reviewer: Optional[Agent] = None, + config: Optional[Union[Dict, Literal[False]]] = None, + ): + """Generate a reply using code execution.""" + + json_objects = find_json_objects(message) + fail_reason = "The required json format answer was not generated." + json_count = len(json_objects) + rensponse_succ = True + view = None + content = None + if json_count != 1: + ### Answer failed, turn on automatic repair + rensponse_succ = False + else: + try: + content = json.dumps(json_objects[0]) + except Exception as e: + content = ( + f"There is a format problem with the json of the answer，{str(e)}" + ) + rensponse_succ = False + try: + vis_client = ApiCall() + view = vis_client.display_only_sql_vis( + json_objects[0], self.db_connect.run_to_df + ) + except Exception as e: + view = f"```vis-convert-error\n{content}\n```" + + return True, { + "is_exe_success": rensponse_succ, + "content": content, + "view": view, + } + + async def a_verify(self, message: Optional[Dict]): + action_reply = message.get("action_report", None) + if action_reply.get("is_exe_success", False) == False: + return ( + False, + f"Please check your answer, {action_reply.get('content', '')}.", + ) + action_reply_obj = json.loads(action_reply.get("content", "")) + sql = action_reply_obj.get("sql", None) + if not sql: + return ( + False, + "Please check your answer, the sql information that needs to be generated is not found.", + ) + try: + columns, values = self.db_connect.query_ex(sql) + if not values or len(values) <= 0: + return ( + False, + "Please check your answer, the generated SQL cannot query any data.", + ) + else: + logger.info( + f"reply check success! There are {len(values)} rows of data" + ) + return True, None + except Exception as e: + return ( + False, + f"SQL execution error, please re-read the historical information to fix this SQL. The error message is as follows:{str(e)}", + ) diff --git a/dbgpt/agent/agents/expand/plugin_assistant_agent.py b/dbgpt/agent/agents/expand/plugin_assistant_agent.py new file mode 100644 index 000000000..6e7845e96 --- /dev/null +++ b/dbgpt/agent/agents/expand/plugin_assistant_agent.py @@ -0,0 +1,125 @@ +import json +import logging +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union + +from dbgpt.core.awel import BaseOperator +from dbgpt.util.json_utils import find_json_objects + +from ...memory.gpts_memory import GptsMemory +from ..agent import Agent, AgentContext +from ..base_agent import ConversableAgent + +try: + from termcolor import colored +except ImportError: + + def colored(x, *args, **kwargs): + return x + + +logger = logging.getLogger(__name__) + + +class PluginAgent(ConversableAgent): + """(In preview) Assistant agent, designed to solve a task with LLM. + + AssistantAgent is a subclass of ConversableAgent configured with a default system message. + The default system message is designed to solve a task with LLM, + including suggesting python code blocks and debugging. + `human_input_mode` is default to "NEVER" + and `code_execution_config` is default to False. + This agent doesn't execute code by default, and expects the user to execute the code. + """ + + DEFAULT_SYSTEM_MESSAGE = """ + You are a useful artificial intelligence tool agent assistant. + You have been assigned the following list of tools, please select the most appropriate tool to complete the task based on the current user's goals: + {tool_list} + + *** IMPORTANT REMINDER *** + Please read the parameter definition of the tool carefully and extract the specific parameters required to execute the tool from the user gogal. + Please output the selected tool name and specific parameter information in json in the following required format, refer to the following example: + user: Search for the latest hot financial news + assisant: {{ + "tool_name":"The chart rendering method currently selected by SQL", + "args": "{{ + "query": "latest hot financial news", + }}", + "thought":"I will use the google-search tool to search for the latest hot financial news." + }} + + Please think step by step and return it in the following json format + {{ + "tool_name":"The chart rendering method currently selected by SQL", + "args": "{{ + "arg name1": "arg value1", + "arg name2": "arg value2", + }}", + "thought":"Summary of thoughts to the user" + }} + Make sure the response is correct json and can be parsed by Python json.loads. + """ + DEFAULT_DESCRIBE = """You can use the following tools to complete the task objectives, tool information: {tool-infos}""" + NAME = "ToolScientist" + + def __init__( + self, + memory: GptsMemory, + agent_context: AgentContext, + describe: Optional[str] = DEFAULT_DESCRIBE, + is_termination_msg: Optional[Callable[[Dict], bool]] = None, + max_consecutive_auto_reply: Optional[int] = None, + human_input_mode: Optional[str] = "NEVER", + **kwargs, + ): + super().__init__( + name=self.NAME, + memory=memory, + describe=describe, + system_message=self.DEFAULT_SYSTEM_MESSAGE, + is_termination_msg=is_termination_msg, + max_consecutive_auto_reply=max_consecutive_auto_reply, + human_input_mode=human_input_mode, + agent_context=agent_context, + **kwargs, + ) + + self.register_reply(Agent, PluginAgent.tool_call) + self.agent_context = agent_context + + async def a_system_fill_param(self): + params = { + "tool_infos": self.db_connect.get_table_info(), + "dialect": self.db_connect.db_type, + } + self.update_system_message(self.DEFAULT_SYSTEM_MESSAGE.format(**params)) + + async def tool_call( + self, + message: Optional[str] = None, + sender: Optional[Agent] = None, + reviewer: Optional[Agent] = None, + config: Optional[Union[Dict, Literal[False]]] = None, + ): + """Generate a reply using code execution.""" + + json_objects = find_json_objects(message) + fail_reason = "The required json format answer was not generated." + json_count = len(json_objects) + rensponse_succ = True + view = None + content = None + if json_count != 1: + ### Answer failed, turn on automatic repair + rensponse_succ = False + else: + try: + view = "" + except Exception as e: + view = f"```vis-convert-error\n{content}\n```" + + return True, { + "is_exe_success": rensponse_succ, + "content": content, + "view": view, + } diff --git a/dbgpt/agent/commands/disply_type/__init__.py b/dbgpt/agent/agents/expand/rag_assistant_agent.py similarity index 100% rename from dbgpt/agent/commands/disply_type/__init__.py rename to dbgpt/agent/agents/expand/rag_assistant_agent.py diff --git a/dbgpt/agent/agents/expand/sql_assistant_agent.py b/dbgpt/agent/agents/expand/sql_assistant_agent.py new file mode 100644 index 000000000..0900a151c --- /dev/null +++ b/dbgpt/agent/agents/expand/sql_assistant_agent.py @@ -0,0 +1,111 @@ +from typing import Callable, Dict, List, Literal, Optional, Union + +from dbgpt.agent.agents.base_agent import ConversableAgent +from dbgpt.core.awel import BaseOperator + +from dbgpt.agent.plugin.commands.command_mange import ApiCall +from ...memory.gpts_memory import GptsMemory +from ..agent import Agent, AgentContext + +try: + from termcolor import colored +except ImportError: + + def colored(x, *args, **kwargs): + return x + + +from dbgpt._private.config import Config + +CFG = Config() + + +class SQLAssistantAgent(ConversableAgent): + """(In preview) Assistant agent, designed to solve a task with LLM. + + AssistantAgent is a subclass of ConversableAgent configured with a default system message. + The default system message is designed to solve a task with LLM, + including suggesting python code blocks and debugging. + `human_input_mode` is default to "NEVER" + and `code_execution_config` is default to False. + This agent doesn't execute code by default, and expects the user to execute the code. + """ + + DEFAULT_SYSTEM_MESSAGE = """You are a SQL expert and answer user questions by writing SQL using the following data structures. + Use the following data structure to write the best mysql SQL for the user's problem. + Data Structure information: + {data_structure} + + - Please ensure that the SQL is correct and high-performance. + - Please be careful not to use tables or fields that are not mentioned. + - Make sure to only return SQL. + """ + + DEFAULT_DESCRIBE = """You can analyze data with a known structure through SQL and generate a single analysis chart for a given target. Please note that you do not have the ability to obtain and process data and can only perform data analysis based on a given structure. If the task goal cannot or does not need to be solved by SQL analysis, please do not use""" + + NAME = "SqlEngineer" + + def __init__( + self, + memory: GptsMemory, + agent_context: AgentContext, + describe: Optional[str] = DEFAULT_DESCRIBE, + is_termination_msg: Optional[Callable[[Dict], bool]] = None, + max_consecutive_auto_reply: Optional[int] = None, + human_input_mode: Optional[str] = "NEVER", + **kwargs, + ): + super().__init__( + name=self.NAME, + memory=memory, + describe=describe, + system_message=self.DEFAULT_SYSTEM_MESSAGE, + is_termination_msg=is_termination_msg, + max_consecutive_auto_reply=max_consecutive_auto_reply, + human_input_mode=human_input_mode, + agent_context=agent_context, + **kwargs, + ) + self.register_reply(Agent, SQLAssistantAgent.generate_analysis_chart_reply) + self.agent_context = agent_context + self.db_connect = CFG.LOCAL_DB_MANAGE.get_connect( + self.agent_context.resource_db.get("name", None) + ) + + async def a_system_fill_param(self): + params = { + "data_structure": self.db_connect.get_table_info(), + "disply_type": ApiCall.default_chart_type_promot(), + "dialect": self.db_connect.db_type, + } + self.update_system_message(self.DEFAULT_SYSTEM_MESSAGE.format(**params)) + + async def generate_analysis_chart_reply( + self, + message: Optional[str] = None, + sender: Optional[Agent] = None, + reviewer: Optional[Agent] = None, + config: Optional[Union[Dict, Literal[False]]] = None, + ): + """Generate a reply using code execution.""" + + # iterate through the last n messages reversly + # if code blocks are found, execute the code blocks and return the output + # if no code blocks are found, continue + + self.api_call = ApiCall(display_registry=[]) + if self.api_call.check_have_plugin_call(message): + exit_success = True + try: + chart_vis = self.api_call.display_sql_llmvis( + message, self.db_connect.run_to_df + ) + except Exception as e: + err_info = f"{str(e)}" + exit_success = False + output = chart_vis if exit_success else err_info + else: + exit_success = False + output = message + + return True, {"is_exe_success": exit_success, "content": f"{output}"} diff --git a/dbgpt/agent/db/__init__.py b/dbgpt/agent/agents/llm/__init__.py similarity index 100% rename from dbgpt/agent/db/__init__.py rename to dbgpt/agent/agents/llm/__init__.py diff --git a/dbgpt/agent/agents/llm/llm.py b/dbgpt/agent/agents/llm/llm.py new file mode 100644 index 000000000..23684de97 --- /dev/null +++ b/dbgpt/agent/agents/llm/llm.py @@ -0,0 +1,35 @@ +from typing import Dict, Optional, Union + +from dbgpt.core.interface.llm import ModelRequest + + +def _build_model_request( + input_value: Union[Dict, str], model: Optional[str] = None +) -> ModelRequest: + """Build model request from input value. + + Args: + input_value(str or dict): input value + model(Optional[str]): model name + + Returns: + ModelRequest: model request, pass to llm client + """ + if isinstance(input_value, str): + return ModelRequest._build(model, input_value) + elif isinstance(input_value, dict): + parm = { + "model": input_value.get("model"), + "messages": input_value.get("messages"), + "temperature": input_value.get("temperature", None), + "max_new_tokens": input_value.get("max_new_tokens", None), + "stop": input_value.get("stop", None), + "stop_token_ids": input_value.get("stop_token_ids", None), + "context_len": input_value.get("context_len", None), + "echo": input_value.get("echo", None), + "span_id": input_value.get("span_id", None), + } + + return ModelRequest(**parm) + else: + raise ValueError("Build model request input Error!") diff --git a/dbgpt/agent/agents/llm/llm_client.py b/dbgpt/agent/agents/llm/llm_client.py new file mode 100644 index 000000000..47f478804 --- /dev/null +++ b/dbgpt/agent/agents/llm/llm_client.py @@ -0,0 +1,194 @@ +import json +import logging +import traceback +from typing import Callable, Dict, Optional, Union + +from dbgpt.core import LLMClient +from dbgpt.core.interface.output_parser import BaseOutputParser +from dbgpt.util.error_types import LLMChatError +from dbgpt.util.tracer import root_tracer + +from ..llm.llm import _build_model_request + +logger = logging.getLogger(__name__) + + +class AIWrapper: + cache_path_root: str = ".cache" + extra_kwargs = { + "cache_seed", + "filter_func", + "allow_format_str_template", + "context", + "llm_model", + } + + def __init__( + self, llm_client: LLMClient, output_parser: Optional[BaseOutputParser] = None + ): + self.llm_echo = False + self.model_cache_enable = False + self._llm_client = llm_client + self._output_parser = output_parser or BaseOutputParser(is_stream_out=False) + + @classmethod + def instantiate( + cls, + template: Optional[Union[str, Callable]] = None, + context: Optional[Dict] = None, + allow_format_str_template: Optional[bool] = False, + ): + if not context or template is None: + return template + if isinstance(template, str): + return template.format(**context) if allow_format_str_template else template + return template(context) + + def _construct_create_params(self, create_config: Dict, extra_kwargs: Dict) -> Dict: + """Prime the create_config with additional_kwargs.""" + # Validate the config + prompt = create_config.get("prompt") + messages = create_config.get("messages") + if (prompt is None) == (messages is None): + raise ValueError( + "Either prompt or messages should be in create config but not both." + ) + + context = extra_kwargs.get("context") + if context is None: + # No need to instantiate if no context is provided. + return create_config + # Instantiate the prompt or messages + allow_format_str_template = extra_kwargs.get("allow_format_str_template", False) + # Make a copy of the config + params = create_config.copy() + if prompt is not None: + # Instantiate the prompt + params["prompt"] = self.instantiate( + prompt, context, allow_format_str_template + ) + elif context: + # Instantiate the messages + params["messages"] = [ + { + **m, + "content": self.instantiate( + m["content"], context, allow_format_str_template + ), + } + if m.get("content") + else m + for m in messages + ] + return params + + def _separate_create_config(self, config): + """Separate the config into create_config and extra_kwargs.""" + create_config = {k: v for k, v in config.items() if k not in self.extra_kwargs} + extra_kwargs = {k: v for k, v in config.items() if k in self.extra_kwargs} + return create_config, extra_kwargs + + def _get_key(self, config): + """Get a unique identifier of a configuration. + + Args: + config (dict or list): A configuration. + + Returns: + tuple: A unique identifier which can be used as a key for a dict. + """ + NON_CACHE_KEY = ["api_key", "base_url", "api_type", "api_version"] + copied = False + for key in NON_CACHE_KEY: + if key in config: + config, copied = config.copy() if not copied else config, True + config.pop(key) + return json.dumps(config, sort_keys=True) + + async def create(self, **config): + # merge the input config with the i-th config in the config list + full_config = {**config} + # separate the config into create_config and extra_kwargs + create_config, extra_kwargs = self._separate_create_config(full_config) + + # construct the create params + params = self._construct_create_params(create_config, extra_kwargs) + # get the cache_seed, filter_func and context + cache_seed = extra_kwargs.get("cache_seed", 66) + filter_func = extra_kwargs.get("filter_func") + context = extra_kwargs.get("context") + llm_model = extra_kwargs.get("llm_model") + if context: + use_cache = context.get("use_cache", True) + if not use_cache: + cache_seed = None + # # Try to load the response from cache + # if cache_seed is not None: + # with diskcache.Cache(f"{self.cache_path_root}/{cache_seed}") as cache: + # # Try to get the response from cache + # key = self._get_key(params) + # response = cache.get(key, None) + # if response is not None: + # # check the filter + # pass_filter = filter_func is None or filter_func(context=context, response=response) + # if pass_filter : + # # Return the response if it passes the filter + # # TODO: add response.cost + # return response + try: + response = await self._completions_create(llm_model, params) + except LLMChatError as e: + logger.debug(f"{llm_model} generate failed!{str(e)}") + raise e + else: + pass_filter = filter_func is None or filter_func( + context=context, response=response + ) + if pass_filter: + # Return the response if it passes the filter + return response + + def _get_span_metadata(self, payload: Dict) -> Dict: + metadata = {k: v for k, v in payload.items()} + + metadata["messages"] = list( + map(lambda m: m if isinstance(m, dict) else m.dict(), metadata["messages"]) + ) + return metadata + + def _llm_messages_convert(self, params): + gpts_messages = params["messages"] + ### TODO + + return gpts_messages + + async def _completions_create(self, llm_model, params): + payload = { + "model": llm_model, + "prompt": params.get("prompt"), + "messages": self._llm_messages_convert(params), + "temperature": float(params.get("temperature")), + "max_new_tokens": int(params.get("max_new_tokens")), + "echo": self.llm_echo, + } + logger.info(f"Request: \n{payload}") + span = root_tracer.start_span( + "Agent.llm_client.no_streaming_call", + metadata=self._get_span_metadata(payload), + ) + payload["span_id"] = span.span_id + payload["model_cache_enable"] = self.model_cache_enable + try: + model_request = _build_model_request(payload) + model_output = await self._llm_client.generate(model_request) + parsed_output = self._output_parser.parse_model_nostream_resp( + model_output, "###" + ) + return parsed_output + except Exception as e: + logger.error( + f"Call LLMClient error, {str(e)}, detail: {traceback.format_exc()}" + ) + raise LLMChatError(original_exception=e) from e + finally: + span.end() diff --git a/dbgpt/agent/agents/plan_group_chat.py b/dbgpt/agent/agents/plan_group_chat.py new file mode 100644 index 000000000..4b55a4aaf --- /dev/null +++ b/dbgpt/agent/agents/plan_group_chat.py @@ -0,0 +1,452 @@ +import json +import logging +import random +import re +import sys +from dataclasses import dataclass +from typing import Dict, List, Optional, Union + +from dbgpt.core.awel import BaseOperator +from dbgpt.util.string_utils import str_to_bool + +from ..common.schema import Status +from ..memory.gpts_memory import GptsMemory, GptsMessage, GptsPlan +from .agent import Agent, AgentContext +from .base_agent import ConversableAgent +from dbgpt.core.interface.message import ModelMessageRoleType + +logger = logging.getLogger(__name__) + + +@dataclass +class PlanChat: + """(In preview) A group chat class that contains the following data fields: + - agents: a list of participating agents. + - messages: a list of messages in the group chat. + - max_round: the maximum number of rounds. + - admin_name: the name of the admin agent if there is one. Default is "Admin". + KeyBoardInterrupt will make the admin agent take over. + - func_call_filter: whether to enforce function call filter. Default is True. + When set to True and when a message is a function call suggestion, + the next speaker will be chosen from an agent which contains the corresponding function name + in its `function_map`. + - speaker_selection_method: the method for selecting the next speaker. Default is "auto". + Could be any of the following (case insensitive), will raise ValueError if not recognized: + - "auto": the next speaker is selected automatically by LLM. + - "manual": the next speaker is selected manually by user input. + - "random": the next speaker is selected randomly. + - "round_robin": the next speaker is selected in a round robin fashion, i.e., iterating in the same order as provided in `agents`. + - allow_repeat_speaker: whether to allow the same speaker to speak consecutively. Default is True. + """ + + agents: List[Agent] + messages: List[Dict] + max_round: int = 50 + admin_name: str = "Admin" + func_call_filter: bool = True + speaker_selection_method: str = "auto" + allow_repeat_speaker: bool = True + + _VALID_SPEAKER_SELECTION_METHODS = ["auto", "manual", "random", "round_robin"] + + @property + def agent_names(self) -> List[str]: + """Return the names of the agents in the group chat.""" + return [agent.name for agent in self.agents] + + def reset(self): + """Reset the group chat.""" + self.messages.clear() + + def agent_by_name(self, name: str) -> Agent: + """Returns the agent with a given name.""" + return self.agents[self.agent_names.index(name)] + + # def select_speaker_msg(self, agents: List[Agent], task_context: str, models: Optional[List[dict]]): + # f"""Return the message for selecting the next speaker.""" + # return f"""You are in a role play game. Read and understand the following tasks and assign the appropriate role to complete them. + # Task content: {task_context} + # You can fill the following roles: {[agent.name for agent in agents]}, + # Please answer only the role name, such as: {agents[0].name}""" + + def select_speaker_msg(self, agents: List[Agent]): + """Return the message for selecting the next speaker.""" + return f"""You are in a role play game. The following roles are available: + {self._participant_roles(agents)}. + Read the following conversation. + Then select the next role from {[agent.name for agent in agents]} to play. The role can be selected repeatedly.Only return the role.""" + + async def a_select_speaker( + self, + last_speaker: Agent, + selector: ConversableAgent, + now_plan_context: str, + pre_allocated: str = None, + ): + """Select the next speaker.""" + if ( + self.speaker_selection_method.lower() + not in self._VALID_SPEAKER_SELECTION_METHODS + ): + raise ValueError( + f"GroupChat speaker_selection_method is set to '{self.speaker_selection_method}'. " + f"It should be one of {self._VALID_SPEAKER_SELECTION_METHODS} (case insensitive). " + ) + + agents = self.agents + n_agents = len(agents) + # Warn if GroupChat is underpopulated + + if ( + n_agents <= 2 + and self.speaker_selection_method.lower() != "round_robin" + and self.allow_repeat_speaker + ): + logger.warning( + f"GroupChat is underpopulated with {n_agents} agents. " + "It is recommended to set speaker_selection_method to 'round_robin' or allow_repeat_speaker to False." + "Or, use direct communication instead." + ) + + # remove the last speaker from the list to avoid selecting the same speaker if allow_repeat_speaker is False + agents = ( + agents + if self.allow_repeat_speaker + else [agent for agent in agents if agent != last_speaker] + ) + + # if self.speaker_selection_method.lower() == "manual": + # selected_agent = self.manual_select_speaker(agents) + # if selected_agent: + # return selected_agent + # elif self.speaker_selection_method.lower() == "round_robin": + # return self.next_agent(last_speaker, agents) + # elif self.speaker_selection_method.lower() == "random": + # return random.choice(agents) + + if pre_allocated: + # Preselect speakers + logger.info(f"Preselect speakers:{pre_allocated}") + name = pre_allocated + model = None + else: + # auto speaker selection + selector.update_system_message(self.select_speaker_msg(agents)) + final, name, model = await selector.a_generate_oai_reply( + self.messages + + [ + { + "role": ModelMessageRoleType.HUMAN, + "content": f"""Read and understand the following task content and assign the appropriate role to complete the task. + Task content: {now_plan_context} + select the role from: {[agent.name for agent in agents]}, + Please only return the role, such as: {agents[0].name}""", + } + ] + ) + if not final: + # the LLM client is None, thus no reply is generated. Use round robin instead. + return self.next_agent(last_speaker, agents), model + + # If exactly one agent is mentioned, use it. Otherwise, leave the OAI response unmodified + mentions = self._mentioned_agents(name, agents) + if len(mentions) == 1: + name = next(iter(mentions)) + else: + logger.warning( + f"GroupChat select_speaker failed to resolve the next speaker's name. This is because the speaker selection OAI call returned:\n{name}" + ) + + # Return the result + try: + return self.agent_by_name(name), model + except Exception as e: + logger.warning(f"auto select speaker failed!{str(e)}") + return self.next_agent(last_speaker, agents), model + + def _mentioned_agents(self, message_content: str, agents: List[Agent]) -> Dict: + """ + Finds and counts agent mentions in the string message_content, taking word boundaries into account. + + Returns: A dictionary mapping agent names to mention counts (to be included, at least one mention must occur) + """ + mentions = dict() + for agent in agents: + regex = ( + r"(?<=\W)" + re.escape(agent.name) + r"(?=\W)" + ) # Finds agent mentions, taking word boundaries into account + count = len( + re.findall(regex, " " + message_content + " ") + ) # Pad the message to help with matching + if count > 0: + mentions[agent.name] = count + return mentions + + def _participant_roles(self, agents: List[Agent] = None) -> str: + # Default to all agents registered + if agents is None: + agents = self.agents + + roles = [] + for agent in agents: + if agent.system_message.strip() == "": + logger.warning( + f"The agent '{agent.name}' has an empty system_message, and may not work well with GroupChat." + ) + roles.append(f"{agent.name}: {agent.describe}") + return "\n".join(roles) + + def agent_by_name(self, name: str) -> Agent: + """Returns the agent with a given name.""" + return self.agents[self.agent_names.index(name)] + + def next_agent(self, agent: Agent, agents: List[Agent]) -> Agent: + """Return the next agent in the list.""" + if agents == self.agents: + return agents[(self.agent_names.index(agent.name) + 1) % len(agents)] + else: + offset = self.agent_names.index(agent.name) + 1 + for i in range(len(self.agents)): + if self.agents[(offset + i) % len(self.agents)] in agents: + return self.agents[(offset + i) % len(self.agents)] + + +class PlanChatManager(ConversableAgent): + """(In preview) A chat manager agent that can manage a group chat of multiple agents.""" + + NAME = "plan_manager" + + def __init__( + self, + plan_chat: PlanChat, + planner: Agent, + memory: GptsMemory, + agent_context: "AgentContext", + # unlimited consecutive auto reply by default + max_consecutive_auto_reply: Optional[int] = sys.maxsize, + human_input_mode: Optional[str] = "NEVER", + describe: Optional[str] = "Plan chat manager.", + **kwargs, + ): + super().__init__( + name=self.NAME, + describe=describe, + memory=memory, + max_consecutive_auto_reply=max_consecutive_auto_reply, + human_input_mode=human_input_mode, + agent_context=agent_context, + **kwargs, + ) + # Order of register_reply is important. + + # Allow async chat if initiated using a_initiate_chat + self.register_reply( + Agent, + PlanChatManager.a_run_chat, + config=plan_chat, + reset_config=PlanChat.reset, + ) + self.plan_chat = plan_chat + self.planner = planner + + async def a_reasoning_reply( + self, messages: Optional[List[Dict]] = None + ) -> Union[str, Dict, None]: + if messages is None or len(messages) <= 0: + message = None + return None, None + else: + message = messages[-1] + self.plan_chat.messages.append(message) + return message["content"], None + + async def a_process_rely_message( + self, conv_id: str, now_plan: GptsPlan, speaker: ConversableAgent + ): + rely_prompt = "" + speaker.reset_rely_message() + if now_plan.rely and len(now_plan.rely) > 0: + rely_tasks_list = now_plan.rely.split(",") + rely_tasks = self.memory.plans_memory.get_by_conv_id_and_num( + conv_id, rely_tasks_list + ) + if rely_tasks: + rely_prompt = "Read the result data of the dependent steps in the above historical message to complete the current goal:" + for rely_task in rely_tasks: + speaker.append_rely_message( + {"content": rely_task.sub_task_content}, + ModelMessageRoleType.HUMAN, + ) + speaker.append_rely_message( + {"content": rely_task.result}, ModelMessageRoleType.AI + ) + return rely_prompt + + async def a_verify_reply( + self, message: Optional[Dict], sender: "Agent", reviewer: "Agent", **kwargs + ) -> Union[str, Dict, None]: + return True, message + + async def a_run_chat( + self, + message: Optional[str] = None, + sender: Optional[Agent] = None, + reviewer: Agent = None, + config: Optional[PlanChat] = None, + ): + """Run a group chat asynchronously.""" + + speaker = sender + groupchat = config + + final_message = None + + for i in range(groupchat.max_round): + plans = self.memory.plans_memory.get_by_conv_id(self.agent_context.conv_id) + if not plans or len(plans) <= 0: + ###Have no plan, generate a new plan TODO init plan use planmanger + await self.a_send( + {"content": message, "current_gogal": message}, + self.planner, + reviewer, + request_reply=False, + ) + verify_pass, reply = await self.planner.a_generate_reply( + {"content": message, "current_gogal": message}, self, reviewer + ) + + await self.planner.a_send( + message=reply, + recipient=self, + reviewer=reviewer, + request_reply=False, + ) + if not verify_pass: + final_message = reply + if i > 10: + break + else: + todo_plans = [ + plan + for plan in plans + if plan.state in [Status.TODO.value, Status.RETRYING.value] + ] + if not todo_plans or len(todo_plans) <= 0: + ### The plan has been fully executed and a success message is sent to the user. + # complete + complete_message = {"content": f"TERMINATE", "is_exe_success": True} + return True, complete_message + else: + now_plan: GptsPlan = todo_plans[0] + + # There is no need to broadcast the message to other agents, it will be automatically obtained from the collective memory according to the dependency relationship. + try: + if Status.RETRYING.value == now_plan.state: + if now_plan.retry_times <= now_plan.max_retry_times: + current_goal_message = { + "content": now_plan.result, + "current_gogal": now_plan.sub_task_content, + "context": { + "plan_task": now_plan.sub_task_content, + "plan_task_num": now_plan.sub_task_num, + }, + } + else: + self.memory.plans_memory.update_task( + self.agent_context.conv_id, + now_plan.sub_task_num, + Status.FAILED.value, + now_plan.retry_times + 1, + speaker.name, + "", + plan_result, + ) + faild_report = { + "content": f"ReTask [{now_plan.sub_task_content}] was retried more than the maximum number of times and still failed.{now_plan.result}", + "is_exe_success": False, + } + return True, faild_report + else: + current_goal_message = { + "content": now_plan.sub_task_content, + "current_gogal": now_plan.sub_task_content, + "context": { + "plan_task": now_plan.sub_task_content, + "plan_task_num": now_plan.sub_task_num, + }, + } + + # select the next speaker + speaker, model = await groupchat.a_select_speaker( + speaker, + self, + now_plan.sub_task_content, + now_plan.sub_task_agent, + ) + # Tell the speaker the dependent history information + rely_prompt = await self.a_process_rely_message( + conv_id=self.agent_context.conv_id, + now_plan=now_plan, + speaker=speaker, + ) + + current_goal_message["content"] = ( + rely_prompt + current_goal_message["content"] + ) + + is_recovery = False + if message == current_goal_message["content"]: + is_recovery = True + await self.a_send( + message=current_goal_message, + recipient=speaker, + reviewer=reviewer, + request_reply=False, + is_recovery=is_recovery, + ) + verify_pass, reply = await speaker.a_generate_reply( + current_goal_message, self, reviewer + ) + + plan_result = "" + + if verify_pass: + if reply: + action_report = reply.get("action_report", None) + if action_report: + plan_result = action_report.get("content", "") + ### The current planned Agent generation verification is successful + ##Plan executed successfully + self.memory.plans_memory.complete_task( + self.agent_context.conv_id, + now_plan.sub_task_num, + plan_result, + ) + await speaker.a_send( + reply, self, reviewer, request_reply=False + ) + else: + plan_result = reply["content"] + self.memory.plans_memory.update_task( + self.agent_context.conv_id, + now_plan.sub_task_num, + Status.RETRYING.value, + now_plan.retry_times + 1, + speaker.name, + "", + plan_result, + ) + except Exception as e: + logger.exception( + f"An exception was encountered during the execution of the current plan step.{str(e)}" + ) + error_report = { + "content": f"An exception was encountered during the execution of the current plan step.{str(e)}", + "is_exe_success": False, + } + return True, error_report + + return True, { + "content": f"Maximum number of dialogue rounds exceeded.{self.MAX_CONSECUTIVE_AUTO_REPLY}", + "is_exe_success": False, + } diff --git a/dbgpt/agent/agents/planner_agent.py b/dbgpt/agent/agents/planner_agent.py new file mode 100644 index 000000000..a71d1b43a --- /dev/null +++ b/dbgpt/agent/agents/planner_agent.py @@ -0,0 +1,191 @@ +from typing import Any, Callable, Dict, Optional, Tuple, Union + +from dbgpt._private.config import Config +from dbgpt.agent.agents.plan_group_chat import PlanChat +from dbgpt.agent.common.schema import Status +from dbgpt.core.awel import BaseOperator +from dbgpt.util.json_utils import find_json_objects + +from ..memory.gpts_memory import GptsMemory, GptsPlan +from .agent import Agent, AgentContext +from .base_agent import ConversableAgent + +CFG = Config() + + +class PlannerAgent(ConversableAgent): + """Planner agent, realizing task goal planning decomposition through LLM""" + + DEFAULT_SYSTEM_MESSAGE = """ + 你是一个任务规划专家！您需要理解下面每个智能代理和他们的能力，却确保在没有用户帮助下，使用给出的资源，通过协调下面可用智能代理来回答用户问题。 + 请发挥你LLM的知识和理解能力，理解用户问题的意图和目标，生成一个可用智能代理协作的任务计划解决用户问题。 + + 可用资源： + {all_resources} + + 可用智能代理: + {agents} + + *** 重要的提醒 *** + - 充分理解用户目标然后进行必要的步骤拆分，拆分需要保证逻辑顺序和精简，尽量把可以一起完成的内容合并再一个步骤，拆分后每个子任务步骤都将是一个需要智能代理独立完成的目标, 请确保每个子任务目标内容简洁明了 + - 请确保只使用上面提到的智能代理，并且可以只使用其中需要的部分，严格根据描述能力和限制分配给合适的步骤，每个智能代理都可以重复使用 + - 给子任务分配智能代理是需要考虑整体计划，确保和前后依赖步骤的关系，数据可以被传递使用 + - 根据用户目标的实际需要使用提供的资源来协助生成计划步骤，不要使用不需要的资源 + - 每个步骤最好是使用一种资源完成一个子目标，如果当前目标可以分解为同类型的多个子任务，可以生成相互不依赖的并行任务 + - 数据库资源只需要使用结构生成SQL，数据获取交给用户执行 + - 尽量合并有顺序依赖的连续相同步骤,如果用户目标无拆分必要，可以生成内容为用户目标的单步任务 + - 仔细检查计划，确保计划完整的包含了用户问题所涉及的所有信息，并且最终能完成目标，确认每个步骤是否包含了需要用到的资源信息,如URL、资源名等. + 具体任务计划的生成可参考如下例子: + user:help me build a sales report summarizing our key metrics and trends + assisant:[ + {{ + "serial_number": "1", + "agent": "DataScientist", + "content": "Retrieve total sales, average sales, and number of transactions grouped by "product_category"'.", + "rely": "" + }}, + {{ + "serial_number": "2", + "agent": "DataScientist", + "content": "Retrieve monthly sales and transaction number trends.", + "rely": "" + }}, + {{ + "serial_number": "3", + "agent": "DataScientist", + "content": "Count the number of transactions with "pay_status" as "paid" among all transactions to retrieve the sales conversion rate.", + "rely": "" + }}, + {{ + "serial_number": "4", + "agent": "Reporter", + "content": "Integrate analytical data into the format required to build sales reports.", + "rely": "1,2,3" + }} + ] + + 请一步步思考，并以如下json格式返回你的行动计划内容: + [{{ + "serial_number":"0", + "agent": "用来完成当前步骤的智能代理", + "content": "当前步骤的任务内容，确保可以被智能代理执行", + "rely":"当前任务执行依赖的其他任务serial_number, 如:1,2,3, 无依赖为空" + }}] + 确保回答的json可以被Python代码的json.loads函数加载解析. + """ + + REPAIR_SYSTEM_MESSAGE = """ + 您是规划专家!现在你需要利用你的专业知识，仔细检查已生成的计划,进行重新评估和分析，确保计划的每个步骤都是清晰完整的，可以被智能代理理解的，解决当前计划中遇到的问题！并按要求返回新的计划内容。 + """ + NAME = "Planner" + + def __init__( + self, + memory: GptsMemory, + plan_chat: PlanChat, + agent_context: AgentContext, + is_termination_msg: Optional[Callable[[Dict], bool]] = None, + max_consecutive_auto_reply: Optional[int] = None, + human_input_mode: Optional[str] = "NEVER", + **kwargs, + ): + super().__init__( + name=self.NAME, + memory=memory, + system_message=self.DEFAULT_SYSTEM_MESSAGE, + is_termination_msg=is_termination_msg, + max_consecutive_auto_reply=max_consecutive_auto_reply, + human_input_mode=human_input_mode, + agent_context=agent_context, + **kwargs, + ) + self.plan_chat = plan_chat + ### register planning funtion + self.register_reply(Agent, PlannerAgent._a_planning) + + def build_param(self, agent_context: AgentContext): + resources = [] + if agent_context.resource_db is not None: + db_connect = CFG.LOCAL_DB_MANAGE.get_connect( + agent_context.resource_db.get("name") + ) + + resources.append( + f"{agent_context.resource_db.get('type')}:{agent_context.resource_db.get('name')}\n{db_connect.get_table_info()}" + ) + if agent_context.resource_knowledge is not None: + resources.append( + f"{agent_context.resource_knowledge.get('type')}:{agent_context.resource_knowledge.get('name')}\n{agent_context.resource_knowledge.get('introduce')}" + ) + if agent_context.resource_internet is not None: + resources.append( + f"{agent_context.resource_internet.get('type')}:{agent_context.resource_internet.get('name')}\n{agent_context.resource_internet.get('introduce')}" + ) + return { + "all_resources": "\n".join([f"- {item}" for item in resources]), + "agents": "\n".join( + [f"- {item.name}:{item.describe}" for item in self.plan_chat.agents] + ), + } + + async def a_system_fill_param(self): + params = self.build_param(self.agent_context) + self.update_system_message(self.DEFAULT_SYSTEM_MESSAGE.format(**params)) + + async def _a_planning( + self, + message: Optional[str] = None, + sender: Optional[Agent] = None, + reviewer: Optional[Agent] = None, + config: Optional[Any] = None, + ) -> Tuple[bool, Union[str, Dict, None]]: + json_objects = find_json_objects(message) + plan_objects = [] + fail_reason = ( + "Please recheck your answer，no usable plans generated in correct format，" + ) + json_count = len(json_objects) + rensponse_succ = True + if json_count != 1: + ### Answer failed, turn on automatic repair + fail_reason += f"There are currently {json_count} json contents" + rensponse_succ = False + else: + try: + for item in json_objects[0]: + plan = GptsPlan( + conv_id=self.agent_context.conv_id, + sub_task_num=item.get("serial_number"), + sub_task_content=item.get("content"), + ) + plan.resource_name = item.get("resource") + plan.max_retry_times = self.agent_context.max_retry_round + plan.sub_task_agent = item.get("agent") + plan.sub_task_title = item.get("content") + plan.rely = item.get("rely") + plan.retry_times = 0 + plan.status = Status.TODO.value + plan_objects.append(plan) + except Exception as e: + fail_reason += f"Return json structure error and cannot be converted to a usable plan，{str(e)}" + rensponse_succ = False + + if rensponse_succ: + if len(plan_objects) > 0: + ### Delete the old plan every time before saving it + self.memory.plans_memory.remove_by_conv_id(self.agent_context.conv_id) + self.memory.plans_memory.batch_save(plan_objects) + + content = "\n".join( + [ + "{},{}".format(index + 1, item.get("content")) + for index, item in enumerate(json_objects[0]) + ] + ) + else: + content = fail_reason + return True, { + "is_exe_success": rensponse_succ, + "content": content, + "view": content, + } diff --git a/dbgpt/agent/agents/user_proxy_agent.py b/dbgpt/agent/agents/user_proxy_agent.py new file mode 100644 index 000000000..5fb577fae --- /dev/null +++ b/dbgpt/agent/agents/user_proxy_agent.py @@ -0,0 +1,89 @@ +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union + +from ..memory.gpts_memory import GptsMemory +from .agent import Agent, AgentContext +from .base_agent import ConversableAgent + +try: + from termcolor import colored +except ImportError: + + def colored(x, *args, **kwargs): + return x + + +class UserProxyAgent(ConversableAgent): + """(In preview) A proxy agent for the user, that can execute code and provide feedback to the other agents.""" + + NAME = "User" + DEFAULT_DESCRIBE = ( + "A human admin. Interact with the planner to discuss the plan. Plan execution needs to be approved by this admin.", + ) + + def __init__( + self, + memory: GptsMemory, + agent_context: AgentContext, + is_termination_msg: Optional[Callable[[Dict], bool]] = None, + max_consecutive_auto_reply: Optional[int] = None, + human_input_mode: Optional[str] = "ALWAYS", + default_auto_reply: Optional[Union[str, Dict, None]] = "", + ): + super().__init__( + name=self.NAME, + memory=memory, + describe=self.DEFAULT_DESCRIBE, + system_message=self.DEFAULT_DESCRIBE, + is_termination_msg=is_termination_msg, + max_consecutive_auto_reply=max_consecutive_auto_reply, + human_input_mode=human_input_mode, + agent_context=agent_context, + ) + self.register_reply(Agent, UserProxyAgent.check_termination_and_human_reply) + + def get_human_input(self, prompt: str) -> str: + """Get human input. + + Override this method to customize the way to get human input. + + Args: + prompt (str): prompt for the human input. + + Returns: + str: human input. + """ + reply = input(prompt) + return reply + + async def a_reasoning_reply( + self, messages: Optional[List[Dict]] = None + ) -> Union[str, Dict, None]: + if messages is None or len(messages) <= 0: + message = None + return None, None + else: + message = messages[-1] + self.plan_chat.messages.append(message) + return message["content"], None + + async def a_receive( + self, + message: Optional[Dict], + sender: Agent, + reviewer: Agent, + request_reply: Optional[bool] = True, + silent: Optional[bool] = False, + is_recovery: Optional[bool] = False, + ): + self.consecutive_auto_reply_counter = sender.consecutive_auto_reply_counter + 1 + self._process_received_message(message, sender, silent) + + async def check_termination_and_human_reply( + self, + message: Optional[str] = None, + sender: Optional[Agent] = None, + reviewer: Agent = None, + config: Optional[Union[Dict, Literal[False]]] = None, + ) -> Tuple[bool, Union[str, Dict, None]]: + """Check if the conversation should be terminated, and if human reply is provided.""" + return True, None diff --git a/dbgpt/agent/common/schema.py b/dbgpt/agent/common/schema.py index 87ba196b0..fc16dfd06 100644 --- a/dbgpt/agent/common/schema.py +++ b/dbgpt/agent/common/schema.py @@ -6,13 +6,15 @@ class PluginStorageType(Enum): Oss = "oss" +class ApiTagType(Enum): + API_VIEW = "dbgpt_view" + API_CALL = "dbgpt_call" + + class Status(Enum): TODO = "todo" RUNNING = "running" + WAITING = "waiting" + RETRYING = "retrying" FAILED = "failed" - COMPLETED = "completed" - - -class ApiTagType(Enum): - API_VIEW = "dbgpt_view" - API_CALL = "dbgpt_call" + COMPLETE = "complete" diff --git a/dbgpt/agent/memory/base.py b/dbgpt/agent/memory/base.py new file mode 100644 index 000000000..c72c47c76 --- /dev/null +++ b/dbgpt/agent/memory/base.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +import dataclasses +from abc import ABC, abstractmethod +from dataclasses import asdict, dataclass, fields +from datetime import datetime +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union + +from dbgpt.agent.common.schema import Status + + +@dataclass +class GptsPlan: + """Gpts plan""" + + conv_id: str + sub_task_num: int + sub_task_content: Optional[str] + sub_task_title: Optional[str] = None + sub_task_agent: Optional[str] = None + resource_name: Optional[str] = None + rely: Optional[str] = None + agent_model: Optional[str] = None + retry_times: Optional[int] = 0 + max_retry_times: Optional[int] = 5 + state: Optional[str] = Status.TODO.value + result: Optional[str] = None + + @staticmethod + def from_dict(d: Dict[str, Any]) -> GptsPlan: + return GptsPlan( + conv_id=d.get("conv_id"), + sub_task_num=d["sub_task_num"], + sub_task_content=d["sub_task_content"], + sub_task_agent=d["sub_task_agent"], + resource_name=d["resource_name"], + rely=d["rely"], + agent_model=d["agent_model"], + retry_times=d["retry_times"], + max_retry_times=d["max_retry_times"], + state=d["state"], + result=d["result"], + ) + + def to_dict(self) -> Dict[str, Any]: + return dataclasses.asdict(self) + + +@dataclass +class GptsMessage: + """Gpts plan""" + + conv_id: str + sender: str + + receiver: str + role: str + content: str + rounds: Optional[int] + current_gogal: str = None + context: Optional[str] = None + review_info: Optional[str] = None + action_report: Optional[str] = None + model_name: Optional[str] = None + created_at: datetime = datetime.utcnow + updated_at: datetime = datetime.utcnow + + @staticmethod + def from_dict(d: Dict[str, Any]) -> GptsMessage: + return GptsMessage( + conv_id=d["conv_id"], + sender=d["sender"], + receiver=d["receiver"], + role=d["role"], + content=d["content"], + rounds=d["rounds"], + model_name=d["model_name"], + current_gogal=d["current_gogal"], + context=d["context"], + review_info=d["review_info"], + action_report=d["action_report"], + created_at=d["created_at"], + updated_at=d["updated_at"], + ) + + def to_dict(self) -> Dict[str, Any]: + return dataclasses.asdict(self) + + +class GptsPlansMemory(ABC): + def batch_save(self, plans: list[GptsPlan]): + """ + batch save gpts plan + Args: + plans: panner generate plans info + Returns: + None + """ + pass + + def get_by_conv_id(self, conv_id: str) -> List[GptsPlan]: + """ + get plans by conv_id + Args: + conv_id: conversation id + Returns: + List of planning steps + """ + + def get_by_conv_id_and_num( + self, conv_id: str, task_nums: List[int] + ) -> List[GptsPlan]: + """ + get + Args: + conv_id: conversation id + task_nums: List of sequence numbers of plans in the same conversation + + Returns: + List of planning steps + + """ + + def get_todo_plans(self, conv_id: str) -> List[GptsPlan]: + """ + Get unfinished planning steps + Args: + conv_id: conversation id + + Returns: + List of planning steps + """ + + def complete_task(self, conv_id: str, task_num: int, result: str): + """ + Complete designated planning step + Args: + conv_id: conversation id + task_num: Planning step num + result: Plan step results + + Returns: + None + """ + + def update_task( + self, + conv_id: str, + task_num: int, + state: str, + retry_times: int, + agent: str = None, + model: str = None, + result: str = None, + ): + """ + Update planning step information + Args: + conv_id: conversation id + task_num: Planning step num + state: the status to update to + retry_times: Latest number of retries + agent: Agent's name + + Returns: + + """ + + def remove_by_conv_id(self, conv_id: str): + """ + Delete planning + Args: + conv_id: + + Returns: + + """ + + +class GptsMessageMemory(ABC): + def append(self, message: GptsMessage): + """ + Add a message + Args: + message: + + Returns: + + """ + + def get_by_agent(self, conv_id: str, agent: str) -> Optional[List[GptsMessage]]: + """ + Query information related to an agent + Args: + agent:agent's name + + Returns: + messages + """ + + def get_between_agents( + self, + conv_id: str, + agent1: str, + agent2: str, + current_gogal: Optional[str] = None, + ) -> Optional[List[GptsMessage]]: + """ + Query information related to an agent + Args: + agent:agent's name + + Returns: + messages + """ + + def get_by_conv_id(self, conv_id: str) -> Optional[List[GptsMessage]]: + """ + Query messages by conv id + Args: + conv_id: + + Returns: + + """ + + def get_last_message(self, conv_id: str) -> Optional[GptsMessage]: + """ + Query last message + Args: + conv_id: + + Returns: + + """ diff --git a/dbgpt/agent/memory/default_gpts_memory.py b/dbgpt/agent/memory/default_gpts_memory.py new file mode 100644 index 000000000..448826412 --- /dev/null +++ b/dbgpt/agent/memory/default_gpts_memory.py @@ -0,0 +1,119 @@ +from dataclasses import fields +from typing import List, Optional + +import pandas as pd + +from dbgpt.agent.common.schema import Status + +from .base import GptsMessage, GptsMessageMemory, GptsPlan, GptsPlansMemory + + +class DefaultGptsPlansMemory(GptsPlansMemory): + def __init__(self): + self.df = pd.DataFrame(columns=[field.name for field in fields(GptsPlan)]) + + def batch_save(self, plans: list[GptsPlan]): + new_rows = pd.DataFrame([item.to_dict() for item in plans]) + self.df = pd.concat([self.df, new_rows], ignore_index=True) + + def get_by_conv_id(self, conv_id: str) -> List[GptsPlan]: + result = self.df.query(f"conv_id==@conv_id") + plans = [] + for row in result.itertuples(index=False, name=None): + row_dict = dict(zip(self.df.columns, row)) + plans.append(GptsPlan.from_dict(row_dict)) + return plans + + def get_by_conv_id_and_num( + self, conv_id: str, task_nums: List[int] + ) -> List[GptsPlan]: + result = self.df.query(f"conv_id==@conv_id and sub_task_num in @task_nums") + plans = [] + for row in result.itertuples(index=False, name=None): + row_dict = dict(zip(self.df.columns, row)) + plans.append(GptsPlan.from_dict(row_dict)) + return plans + + def get_todo_plans(self, conv_id: str) -> List[GptsPlan]: + todo_states = [Status.TODO.value, Status.RETRYING.value] + result = self.df.query(f"conv_id==@conv_id and state in @todo_states") + plans = [] + for row in result.itertuples(index=False, name=None): + row_dict = dict(zip(self.df.columns, row)) + plans.append(GptsPlan.from_dict(row_dict)) + return plans + + def complete_task(self, conv_id: str, task_num: int, result: str): + condition = (self.df["conv_id"] == conv_id) & ( + self.df["sub_task_num"] == task_num + ) + self.df.loc[condition, "state"] = Status.COMPLETE.value + self.df.loc[condition, "result"] = result + + def update_task( + self, + conv_id: str, + task_num: int, + state: str, + retry_times: int, + agent: str = None, + model=None, + result: str = None, + ): + condition = (self.df["conv_id"] == conv_id) & ( + self.df["sub_task_num"] == task_num + ) + self.df.loc[condition, "state"] = state + self.df.loc[condition, "retry_times"] = retry_times + self.df.loc[condition, "result"] = result + + if agent: + self.df.loc[condition, "sub_task_agent"] = agent + + if model: + self.df.loc[condition, "agent_model"] = model + + def remove_by_conv_id(self, conv_id: str): + self.df.drop(self.df[self.df["conv_id"] == conv_id].index, inplace=True) + + +class DefaultGptsMessageMemory(GptsMessageMemory): + def __init__(self): + self.df = pd.DataFrame(columns=[field.name for field in fields(GptsMessage)]) + + def append(self, message: GptsMessage): + self.df.loc[len(self.df)] = message.to_dict() + + def get_by_agent(self, conv_id: str, agent: str) -> Optional[List[GptsMessage]]: + result = self.df.query( + f"conv_id==@conv_id and (sender==@agent or receiver==@agent)" + ) + messages = [] + for row in result.itertuples(index=False, name=None): + row_dict = dict(zip(self.df.columns, row)) + messages.append(GptsMessage.from_dict(row_dict)) + return messages + + def get_between_agents( + self, + conv_id: str, + agent1: str, + agent2: str, + current_gogal: Optional[str] = None, + ) -> Optional[List[GptsMessage]]: + result = self.df.query( + f"conv_id==@conv_id and ((sender==@agent1 and receiver==@agent2) or (sender==@agent2 and receiver==@agent1)) and current_gogal==@current_gogal" + ) + messages = [] + for row in result.itertuples(index=False, name=None): + row_dict = dict(zip(self.df.columns, row)) + messages.append(GptsMessage.from_dict(row_dict)) + return messages + + def get_by_conv_id(self, conv_id: str) -> Optional[List[GptsMessage]]: + result = self.df.query(f"conv_id==@conv_id") + messages = [] + for row in result.itertuples(index=False, name=None): + row_dict = dict(zip(self.df.columns, row)) + messages.append(GptsMessage.from_dict(row_dict)) + return messages diff --git a/dbgpt/agent/memory/gpts_memory.py b/dbgpt/agent/memory/gpts_memory.py new file mode 100644 index 000000000..b86be2c28 --- /dev/null +++ b/dbgpt/agent/memory/gpts_memory.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import json +from collections import defaultdict +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union + +from dbgpt.util.json_utils import EnhancedJSONEncoder + +from .base import GptsMessage, GptsMessageMemory, GptsPlan, GptsPlansMemory +from .default_gpts_memory import DefaultGptsMessageMemory, DefaultGptsPlansMemory + + +class GptsMemory: + def __init__( + self, + plans_memory: Optional[GptsPlansMemory] = None, + message_memory: Optional[GptsMessageMemory] = None, + ): + self._plans_memory: GptsPlansMemory = ( + plans_memory if plans_memory is not None else DefaultGptsPlansMemory() + ) + self._message_memory: GptsMessageMemory = ( + message_memory if message_memory is not None else DefaultGptsMessageMemory() + ) + + @property + def plans_memory(self): + return self._plans_memory + + @property + def message_memory(self): + return self._message_memory + + async def one_plan_chat_competions(self, conv_id: str): + plans = self.plans_memory.get_by_conv_id(conv_id=conv_id) + messages = self.message_memory.get_by_conv_id(conv_id=conv_id) + + messages_group = defaultdict(list) + for item in messages: + messages_group[item.current_gogal].append(item) + + plans_info_map = defaultdict() + for plan in plans: + plans_info_map[plan.sub_task_content] = { + "name": plan.sub_task_content, + "num": plan.sub_task_num, + "status": plan.state, + "agent": plan.sub_task_agent, + "markdown": self._messages_to_agents_vis( + messages_group.get(plan.sub_task_content) + ), + } + + normal_messages = [] + if messages_group: + for key, value in messages_group.items(): + if key not in plans_info_map: + normal_messages.extend(value) + return f"{self._messages_to_agents_vis(normal_messages)}\n{self._messages_to_plan_vis(list(plans_info_map.values()))}" + + @staticmethod + def _messages_to_agents_vis(messages: List[GptsMessage]): + if messages is None or len(messages) <= 0: + return "" + messages_view = [] + for message in messages: + action_report_str = message.action_report + view_info = message.content + if action_report_str and len(action_report_str) > 0: + action_report = json.loads(action_report_str) + if action_report: + view = action_report.get("view", None) + view_info = view if view else action_report.get("content", "") + + messages_view.append( + { + "sender": message.sender, + "receiver": message.receiver, + "model": message.model_name, + "markdown": view_info, + } + ) + messages_content = json.dumps( + messages_view, ensure_ascii=False, cls=EnhancedJSONEncoder + ) + return f"```agent-messages\n{messages_content}\n```" + + @staticmethod + def _messages_to_plan_vis(messages: List[Dict]): + if messages is None or len(messages) <= 0: + return "" + messages_content = json.dumps( + messages, ensure_ascii=False, cls=EnhancedJSONEncoder + ) + return f"```agent-plans\n{messages_content}\n```" diff --git a/dbgpt/agent/memory/gpts_memory_storage.py b/dbgpt/agent/memory/gpts_memory_storage.py new file mode 100644 index 000000000..031b129d6 --- /dev/null +++ b/dbgpt/agent/memory/gpts_memory_storage.py @@ -0,0 +1,443 @@ +import dataclasses +from dataclasses import asdict, dataclass, fields +import json +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, List, Optional +from dbgpt.core.interface.storage import ( + InMemoryStorage, + QuerySpec, + ResourceIdentifier, + StorageInterface, + StorageItem, +) +from dbgpt.agent.common.schema import Status +from datetime import datetime + +from .base import GptsMessageMemory, GptsPlansMemory, GptsPlan, GptsMessage + + +@dataclass +class GptsPlanIdentifier(ResourceIdentifier): + identifier_split: str = dataclasses.field(default="___$$$$___", init=False) + conv_id: str + sub_task_num: Optional[str] + + def __post_init__(self): + if self.conv_id is None or self.sub_task_num is None: + raise ValueError("conv_id and sub_task_num cannot be None") + + if any( + self.identifier_split in key + for key in [ + self.conv_id, + self.sub_task_num, + ] + if key is not None + ): + raise ValueError( + f"identifier_split {self.identifier_split} is not allowed in conv_id, sub_task_num" + ) + + @property + def str_identifier(self) -> str: + return self.identifier_split.join( + key + for key in [ + self.conv_id, + self.sub_task_num, + ] + if key is not None + ) + + def to_dict(self) -> Dict: + return { + "conv_id": self.conv_id, + "sub_task_num": self.sub_task_num, + } + + +@dataclass +class GptsPlanStorage(StorageItem): + """Gpts plan""" + + conv_id: str + sub_task_num: int + sub_task_content: Optional[str] + sub_task_title: Optional[str] = None + sub_task_agent: Optional[str] = None + resource_name: Optional[str] = None + rely: Optional[str] = None + agent_model: Optional[str] = None + retry_times: Optional[int] = 0 + max_retry_times: Optional[int] = 5 + state: Optional[str] = Status.TODO.value + result: Optional[str] = None + + _identifier: GptsPlanIdentifier = dataclasses.field(init=False) + + @staticmethod + def from_dict(d: Dict[str, Any]): + return GptsPlanStorage( + conv_id=d.get("conv_id"), + sub_task_num=d["sub_task_num"], + sub_task_content=d["sub_task_content"], + sub_task_agent=d["sub_task_agent"], + resource_name=d["resource_name"], + rely=d["rely"], + agent_model=d["agent_model"], + retry_times=d["retry_times"], + max_retry_times=d["max_retry_times"], + state=d["state"], + result=d["result"], + ) + + def to_dict(self) -> Dict[str, Any]: + return dataclasses.asdict(self) + + def _check(self): + if self.conv_id is None: + raise ValueError("conv_id cannot be None") + if self.sub_task_num is None: + raise ValueError("sub_task_num cannot be None") + if self.sub_task_content is None: + raise ValueError("sub_task_content cannot be None") + if self.state is None: + raise ValueError("state cannot be None") + + @property + def identifier(self) -> GptsPlanIdentifier: + return self._identifier + + def merge(self, other: "StorageItem") -> None: + """Merge the other item into the current item. + + Args: + other (StorageItem): The other item to merge + """ + if not isinstance(other, GptsPlanStorage): + raise ValueError( + f"Cannot merge {type(other)} into {type(self)} because they are not the same type." + ) + self.from_object(other) + + +@dataclass +class GptsMessageIdentifier(ResourceIdentifier): + identifier_split: str = dataclasses.field(default="___$$$$___", init=False) + conv_id: str + sender: Optional[str] + receiver: Optional[str] + rounds: Optional[int] + + def __post_init__(self): + if ( + self.conv_id is None + or self.sender is None + or self.receiver is None + or self.rounds is None + ): + raise ValueError("conv_id and sub_task_num cannot be None") + + if any( + self.identifier_split in key + for key in [ + self.conv_id, + self.sender, + self.receiver, + self.rounds, + ] + if key is not None + ): + raise ValueError( + f"identifier_split {self.identifier_split} is not allowed in conv_id, sender, receiver, rounds" + ) + + @property + def str_identifier(self) -> str: + return self.identifier_split.join( + key + for key in [ + self.conv_id, + self.sender, + self.receiver, + self.rounds, + ] + if key is not None + ) + + def to_dict(self) -> Dict: + return { + "conv_id": self.conv_id, + "sender": self.sender, + "receiver": self.receiver, + "rounds": self.rounds, + } + + +@dataclass +class GptsMessageStorage(StorageItem): + """Gpts Message""" + + conv_id: str + sender: str + + receiver: str + role: str + content: str + rounds: Optional[int] + current_gogal: str = None + context: Optional[str] = None + review_info: Optional[str] = None + action_report: Optional[str] = None + model_name: Optional[str] = None + created_at: datetime = datetime.utcnow + updated_at: datetime = datetime.utcnow + + _identifier: GptsMessageIdentifier = dataclasses.field(init=False) + + @staticmethod + def from_dict(d: Dict[str, Any]): + return GptsMessageStorage( + conv_id=d["conv_id"], + sender=d["sender"], + receiver=d["receiver"], + role=d["role"], + content=d["content"], + rounds=d["rounds"], + model_name=d["model_name"], + current_gogal=d["current_gogal"], + context=d["context"], + review_info=d["review_info"], + action_report=d["action_report"], + created_at=d["created_at"], + updated_at=d["updated_at"], + ) + + def to_dict(self) -> Dict[str, Any]: + return dataclasses.asdict(self) + + def _check(self): + if self.conv_id is None: + raise ValueError("conv_id cannot be None") + if self.sub_task_num is None: + raise ValueError("sub_task_num cannot be None") + if self.sub_task_content is None: + raise ValueError("sub_task_content cannot be None") + if self.state is None: + raise ValueError("state cannot be None") + + def to_gpts_message(self) -> GptsMessage: + """Convert the storage to a GptsMessage.""" + input_variables = ( + None + if not self.input_variables + else self.input_variables.strip().split(",") + ) + return GptsMessage( + conv_id=self.conv_id, + sender=self.sender, + receiver=self.receiver, + role=self.role, + content=self.content, + rounds=self.rounds, + current_gogal=self.current_gogal, + context=self.context, + review_info=self.review_info, + action_report=self.action_report, + model_name=self.model_name, + created_at=self.created_at, + updated_at=self.updated_at, + ) + + @staticmethod + def from_gpts_message(gpts_message: GptsMessage) -> "StoragePromptTemplate": + """Convert a GptsMessage to a storage e.""" + return GptsMessageStorage( + conv_id=gpts_message.conv_id, + sender=gpts_message.sender, + receiver=gpts_message.receiver, + role=gpts_message.role, + content=gpts_message.content, + rounds=gpts_message.rounds, + current_gogal=gpts_message.current_gogal, + context=gpts_message.context, + review_info=gpts_message.review_info, + action_report=gpts_message.action_report, + model_name=gpts_message.model_name, + created_at=gpts_message.created_at, + updated_at=gpts_message.updated_at, + ) + + @property + def identifier(self) -> GptsMessageIdentifier: + return self._identifier + + def merge(self, other: "StorageItem") -> None: + """Merge the other item into the current item. + + Args: + other (StorageItem): The other item to merge + """ + if not isinstance(other, GptsMessageStorage): + raise ValueError( + f"Cannot merge {type(other)} into {type(self)} because they are not the same type." + ) + self.from_object(other) + + +class GptsMessageManager(GptsMessageMemory): + """The manager class for GptsMessage. + + Simple wrapper for the storage interface. + + """ + + def __init__(self, storage: Optional[StorageInterface[GptsMessage, Any]] = None): + if storage is None: + storage = InMemoryStorage() + self._storage = storage + + @property + def storage(self) -> StorageInterface[GptsMessage, Any]: + """The storage interface for prompt templates.""" + return self._storage + + def append(self, message: GptsMessage): + self.storage.save(GptsMessageStorage.from_gpts_message(message)) + + def get_by_agent(self, conv_id: str, agent: str) -> Optional[List[GptsMessage]]: + query_spec = QuerySpec( + conditions={ + "conv_id": conv_id, + "sys_code": sys_code, + **kwargs, + } + ) + queries: List[GptsMessageStorage] = self.storage.query( + query_spec, GptsMessageStorage + ) + if not queries: + return [] + if prefer_prompt_language: + prefer_prompt_language = prefer_prompt_language.lower() + temp_queries = [ + query + for query in queries + if query.prompt_language + and query.prompt_language.lower() == prefer_prompt_language + ] + if temp_queries: + queries = temp_queries + if prefer_model: + prefer_model = prefer_model.lower() + temp_queries = [ + query + for query in queries + if query.model and query.model.lower() == prefer_model + ] + if temp_queries: + queries = temp_queries + return queries + + def get_between_agents( + self, + conv_id: str, + agent1: str, + agent2: str, + current_gogal: Optional[str] = None, + ) -> Optional[List[GptsMessage]]: + return super().get_between_agents(conv_id, agent1, agent2, current_gogal) + + def get_by_conv_id(self, conv_id: str) -> Optional[List[GptsMessage]]: + return super().get_by_conv_id(conv_id) + + def get_last_message(self, conv_id: str) -> Optional[GptsMessage]: + return super().get_last_message(conv_id) + + def prefer_query( + self, + prompt_name: str, + sys_code: Optional[str] = None, + prefer_prompt_language: Optional[str] = None, + prefer_model: Optional[str] = None, + **kwargs, + ) -> List[GptsMessage]: + """Query prompt templates from storage with prefer params. + + Sometimes, we want to query prompt templates with prefer params(e.g. some language or some model). + This method will query prompt templates with prefer params first, if not found, will query all prompt templates. + + Examples: + + Query a prompt template. + .. code-block:: python + + prompt_template_list = prompt_manager.prefer_query("hello") + + Query with sys_code and username. + + .. code-block:: python + + prompt_template_list = prompt_manager.prefer_query( + "hello", sys_code="sys_code", user_name="user_name" + ) + + Query with prefer prompt language. + + .. code-block:: python + + # First query with prompt name "hello" exactly. + # Second filter with prompt language "zh-cn", if not found, will return all prompt templates. + prompt_template_list = prompt_manager.prefer_query( + "hello", prefer_prompt_language="zh-cn" + ) + + Query with prefer model. + + .. code-block:: python + + # First query with prompt name "hello" exactly. + # Second filter with model "vicuna-13b-v1.5", if not found, will return all prompt templates. + prompt_template_list = prompt_manager.prefer_query( + "hello", prefer_model="vicuna-13b-v1.5" + ) + + Args: + prompt_name (str): The name of the prompt template. + sys_code (Optional[str], optional): The system code of the prompt template. Defaults to None. + prefer_prompt_language (Optional[str], optional): The language of the prompt template. Defaults to None. + prefer_model (Optional[str], optional): The model of the prompt template. Defaults to None. + kwargs (Dict): Other query params(If some key and value not None, wo we query it exactly). + """ + query_spec = QuerySpec( + conditions={ + "prompt_name": prompt_name, + "sys_code": sys_code, + **kwargs, + } + ) + queries: List[StoragePromptTemplate] = self.storage.query( + query_spec, StoragePromptTemplate + ) + if not queries: + return [] + if prefer_prompt_language: + prefer_prompt_language = prefer_prompt_language.lower() + temp_queries = [ + query + for query in queries + if query.prompt_language + and query.prompt_language.lower() == prefer_prompt_language + ] + if temp_queries: + queries = temp_queries + if prefer_model: + prefer_model = prefer_model.lower() + temp_queries = [ + query + for query in queries + if query.model and query.model.lower() == prefer_model + ] + if temp_queries: + queries = temp_queries + return queries diff --git a/dbgpt/agent/hub/__init__.py b/dbgpt/agent/plugin/__init__.py similarity index 100% rename from dbgpt/agent/hub/__init__.py rename to dbgpt/agent/plugin/__init__.py diff --git a/dbgpt/agent/plugin/commands/__init__.py b/dbgpt/agent/plugin/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbgpt/agent/plugin/commands/built_in/__init__.py b/dbgpt/agent/plugin/commands/built_in/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbgpt/agent/commands/built_in/audio_text.py b/dbgpt/agent/plugin/commands/built_in/audio_text.py similarity index 96% rename from dbgpt/agent/commands/built_in/audio_text.py rename to dbgpt/agent/plugin/commands/built_in/audio_text.py index b8320a95b..49fbaceed 100644 --- a/dbgpt/agent/commands/built_in/audio_text.py +++ b/dbgpt/agent/plugin/commands/built_in/audio_text.py @@ -3,8 +3,8 @@ import requests -from dbgpt.agent.commands.command_mange import command from dbgpt._private.config import Config +from ..command_mange import command CFG = Config() diff --git a/dbgpt/agent/plugin/commands/built_in/disply_type/__init__.py b/dbgpt/agent/plugin/commands/built_in/disply_type/__init__.py new file mode 100644 index 000000000..a13495925 --- /dev/null +++ b/dbgpt/agent/plugin/commands/built_in/disply_type/__init__.py @@ -0,0 +1 @@ +from .show_chart_gen import static_message_img_path diff --git a/dbgpt/agent/commands/disply_type/show_chart_gen.py b/dbgpt/agent/plugin/commands/built_in/disply_type/show_chart_gen.py similarity index 99% rename from dbgpt/agent/commands/disply_type/show_chart_gen.py rename to dbgpt/agent/plugin/commands/built_in/disply_type/show_chart_gen.py index d785e6c75..bfd62fe07 100644 --- a/dbgpt/agent/commands/disply_type/show_chart_gen.py +++ b/dbgpt/agent/plugin/commands/built_in/disply_type/show_chart_gen.py @@ -1,20 +1,22 @@ -from pandas import DataFrame - -from dbgpt.agent.commands.command_mange import command -import pandas as pd -import uuid import os +import uuid + import matplotlib +import pandas as pd import seaborn as sns +from pandas import DataFrame + +from ...command_mange import command matplotlib.use("Agg") +import logging + import matplotlib.pyplot as plt import matplotlib.ticker as mtick from matplotlib.font_manager import FontManager -from dbgpt.util.string_utils import is_scientific_notation -from dbgpt.configs.model_config import PILOT_PATH -import logging +from dbgpt.configs.model_config import PILOT_PATH +from dbgpt.util.string_utils import is_scientific_notation logger = logging.getLogger(__name__) diff --git a/dbgpt/agent/commands/disply_type/show_table_gen.py b/dbgpt/agent/plugin/commands/built_in/disply_type/show_table_gen.py similarity index 91% rename from dbgpt/agent/commands/disply_type/show_table_gen.py rename to dbgpt/agent/plugin/commands/built_in/disply_type/show_table_gen.py index 6b8479058..d0457bb40 100644 --- a/dbgpt/agent/commands/disply_type/show_table_gen.py +++ b/dbgpt/agent/plugin/commands/built_in/disply_type/show_table_gen.py @@ -1,8 +1,8 @@ -from pandas import DataFrame +import logging -from dbgpt.agent.commands.command_mange import command +from pandas import DataFrame -import logging +from ...command_mange import command logger = logging.getLogger(__name__) diff --git a/dbgpt/agent/commands/disply_type/show_text_gen.py b/dbgpt/agent/plugin/commands/built_in/disply_type/show_text_gen.py similarity index 94% rename from dbgpt/agent/commands/disply_type/show_text_gen.py rename to dbgpt/agent/plugin/commands/built_in/disply_type/show_text_gen.py index 8136637fc..37caf6fe5 100644 --- a/dbgpt/agent/commands/disply_type/show_text_gen.py +++ b/dbgpt/agent/plugin/commands/built_in/disply_type/show_text_gen.py @@ -1,8 +1,8 @@ -from pandas import DataFrame +import logging -from dbgpt.agent.commands.command_mange import command +from pandas import DataFrame -import logging +from ...command_mange import command logger = logging.getLogger(__name__) diff --git a/dbgpt/agent/commands/built_in/image_gen.py b/dbgpt/agent/plugin/commands/built_in/image_gen.py similarity index 98% rename from dbgpt/agent/commands/built_in/image_gen.py rename to dbgpt/agent/plugin/commands/built_in/image_gen.py index 3a313439f..7893ac1bb 100644 --- a/dbgpt/agent/commands/built_in/image_gen.py +++ b/dbgpt/agent/plugin/commands/built_in/image_gen.py @@ -1,14 +1,14 @@ """ Image Generation Module for AutoGPT.""" import io +import logging import uuid from base64 import b64decode -import logging import requests from PIL import Image -from dbgpt.agent.commands.command_mange import command from dbgpt._private.config import Config +from ..command_mange import command logger = logging.getLogger(__name__) CFG = Config() diff --git a/dbgpt/agent/commands/command.py b/dbgpt/agent/plugin/commands/command.py similarity index 98% rename from dbgpt/agent/commands/command.py rename to dbgpt/agent/plugin/commands/command.py index e307a3bf9..bdcfebaa4 100644 --- a/dbgpt/agent/commands/command.py +++ b/dbgpt/agent/plugin/commands/command.py @@ -4,11 +4,11 @@ import json from typing import Dict -from .exception_not_commands import NotCommands -from .generator import PluginPromptGenerator - from dbgpt._private.config import Config +from .exception_not_commands import NotCommands +from dbgpt.agent.plugin.generator import PluginPromptGenerator + def _resolve_pathlike_command_args(command_args): if "directory" in command_args and command_args["directory"] in {"", "/"}: diff --git a/dbgpt/agent/commands/command_mange.py b/dbgpt/agent/plugin/commands/command_mange.py similarity index 77% rename from dbgpt/agent/commands/command_mange.py rename to dbgpt/agent/plugin/commands/command_mange.py index 9002a07ca..c5b0a4ee0 100644 --- a/dbgpt/agent/commands/command_mange.py +++ b/dbgpt/agent/plugin/commands/command_mange.py @@ -4,17 +4,19 @@ import json import logging import xml.etree.ElementTree as ET - -from dbgpt.util.json_utils import serialize from datetime import datetime -from typing import Any, Callable, Optional, List +from typing import Any, Callable, List, Optional + from dbgpt._private.pydantic import BaseModel + +from .command import execute_command from dbgpt.agent.common.schema import Status -from dbgpt.agent.commands.command import execute_command -from dbgpt.util.string_utils import extract_content_open_ending, extract_content +from dbgpt.util.json_utils import serialize +from dbgpt.util.string_utils import extract_content, extract_content_open_ending # Unique identifier for auto-gpt commands AUTO_GPT_COMMAND_IDENTIFIER = "auto_gpt_command" +logger = logging.getLogger(__name__) class Command: @@ -404,7 +406,7 @@ def run(self, llm_text): value.api_result = execute_command( value.name, value.args, self.plugin_generator ) - value.status = Status.COMPLETED.value + value.status = Status.COMPLETE.value except Exception as e: value.status = Status.FAILED.value value.err_msg = str(e) @@ -436,7 +438,7 @@ def run_display_sql(self, llm_text, sql_run_func): "response_table", **param ) - value.status = Status.COMPLETED.value + value.status = Status.COMPLETE.value except Exception as e: value.status = Status.FAILED.value value.err_msg = str(e) @@ -474,12 +476,13 @@ def display_sql_llmvis(self, llm_text, sql_run_func): date_unit="s", ) ) - value.status = Status.COMPLETED.value + value.status = Status.COMPLETE.value else: value.status = Status.FAILED.value value.err_msg = "No executable sql！" except Exception as e: + logging.error(f"data prepare exception！{str(e)}") value.status = Status.FAILED.value value.err_msg = str(e) value.end_time = datetime.now().timestamp() * 1000 @@ -488,3 +491,125 @@ def display_sql_llmvis(self, llm_text, sql_run_func): raise ValueError("Api parsing exception," + str(e)) return self.api_view_context(llm_text, True) + + def display_only_sql_vis(self, chart: dict, sql_2_df_func): + err_msg = None + try: + sql = chart.get("sql", None) + param = {} + df = sql_2_df_func(sql) + if not sql or len(sql) <= 0: + return None + + param["sql"] = sql + param["type"] = chart.get("display_type", "response_table") + param["title"] = chart.get("title", "") + param["describe"] = chart.get("thought", "") + + param["data"] = json.loads( + df.to_json(orient="records", date_format="iso", date_unit="s") + ) + view_json_str = json.dumps(param, default=serialize, ensure_ascii=False) + except Exception as e: + logger.error("parse_view_response error!" + str(e)) + err_param = {} + err_param["sql"] = f"{sql}" + err_param["type"] = "response_table" + # err_param["err_msg"] = str(e) + err_param["data"] = [] + err_msg = str(e) + view_json_str = json.dumps(err_param, default=serialize, ensure_ascii=False) + + # api_call_element.text = view_json_str + result = f"```vis-chart\n{view_json_str}\n```" + if err_msg: + return f"""ERROR!{err_msg} \n {result}""" + else: + return result + + def display_dashboard_vis( + self, charts: List[dict], sql_2_df_func, title: str = None + ): + err_msg = None + view_json_str = None + + chart_items = [] + try: + if not charts or len(charts) <= 0: + return f"""Have no chart data!""" + for chart in charts: + param = {} + sql = chart.get("sql", "") + param["sql"] = sql + param["type"] = chart.get("display_type", "response_table") + param["title"] = chart.get("title", "") + param["describe"] = chart.get("thought", "") + try: + df = sql_2_df_func(sql) + param["data"] = json.loads( + df.to_json(orient="records", date_format="iso", date_unit="s") + ) + except Exception as e: + param["data"] = [] + param["err_msg"] = str(e) + chart_items.append( + f"```vis-chart-item\n{json.dumps(param, default=serialize, ensure_ascii=False)}\n```" + ) + + dashboard_param = { + "markdown": "\n".join(chart_items), + "chart_count": len(chart_items), + "title": title, + } + view_json_str = json.dumps( + dashboard_param, default=serialize, ensure_ascii=False + ) + + except Exception as e: + logger.error("parse_view_response error!" + str(e)) + return f"```error\nReport rendering exception！{str(e)}\n```" + + result = f"```vis-dashboard\n{view_json_str}\n```" + if err_msg: + return ( + f"""\\n ERROR!{err_msg} \n {result}""" + ) + else: + return result + + @staticmethod + def default_chart_type_promot() -> str: + """this function is moved from excel_analyze/chat.py,and used by subclass. + Returns: + + """ + antv_charts = [ + {"response_line_chart": "used to display comparative trend analysis data"}, + { + "response_pie_chart": "suitable for scenarios such as proportion and distribution statistics" + }, + { + "response_table": "suitable for display with many display columns or non-numeric columns" + }, + # {"response_data_text":" the default display method, suitable for single-line or simple content display"}, + { + "response_scatter_plot": "Suitable for exploring relationships between variables, detecting outliers, etc." + }, + { + "response_bubble_chart": "Suitable for relationships between multiple variables, highlighting outliers or special situations, etc." + }, + { + "response_donut_chart": "Suitable for hierarchical structure representation, category proportion display and highlighting key categories, etc." + }, + { + "response_area_chart": "Suitable for visualization of time series data, comparison of multiple groups of data, analysis of data change trends, etc." + }, + { + "response_heatmap": "Suitable for visual analysis of time series data, large-scale data sets, distribution of classified data, etc." + }, + ] + return "\n".join( + f"{key}:{value}" + for dict_item in antv_charts + for key, value in dict_item.items() + ) diff --git a/dbgpt/agent/commands/exception_not_commands.py b/dbgpt/agent/plugin/commands/exception_not_commands.py similarity index 100% rename from dbgpt/agent/commands/exception_not_commands.py rename to dbgpt/agent/plugin/commands/exception_not_commands.py diff --git a/dbgpt/agent/commands/generator.py b/dbgpt/agent/plugin/generator.py similarity index 100% rename from dbgpt/agent/commands/generator.py rename to dbgpt/agent/plugin/generator.py diff --git a/dbgpt/agent/plugin/plugin.py b/dbgpt/agent/plugin/plugin.py new file mode 100644 index 000000000..bc8bafe89 --- /dev/null +++ b/dbgpt/agent/plugin/plugin.py @@ -0,0 +1,19 @@ +import logging +from .generator import PluginPromptGenerator +from typing import List + +logger = logging.getLogger(__name__) + + +class PluginLoader: + def load_plugins( + self, generator: PluginPromptGenerator, my_plugins: List[str] + ) -> PluginPromptGenerator: + logger.info(f"load_select_plugin:{my_plugins}") + # load select plugin + for plugin in self.plugins: + if plugin._name in my_plugins: + if not plugin.can_handle_post_prompt(): + continue + generator = plugin.post_prompt(generator) + return generator diff --git a/dbgpt/agent/plugins_util.py b/dbgpt/agent/plugin/plugins_util.py similarity index 100% rename from dbgpt/agent/plugins_util.py rename to dbgpt/agent/plugin/plugins_util.py index d65eaa36d..e2620e562 100644 --- a/dbgpt/agent/plugins_util.py +++ b/dbgpt/agent/plugin/plugins_util.py @@ -1,17 +1,17 @@ """加载组件""" +import datetime +import glob import json +import logging import os -import glob -import zipfile -import git import threading -import datetime -import logging +import zipfile from pathlib import Path from typing import List from zipimport import zipimporter +import git import requests from auto_gpt_plugin_template import AutoGPTPluginTemplate diff --git a/dbgpt/app/base.py b/dbgpt/app/base.py index e834e92da..6a2acd210 100644 --- a/dbgpt/app/base.py +++ b/dbgpt/app/base.py @@ -32,7 +32,7 @@ def async_db_summary(system_app: SystemApp): def server_init(param: "WebServerParameters", system_app: SystemApp): - from dbgpt.agent.commands.command_mange import CommandRegistry + from dbgpt.agent.plugin.commands.command_mange import CommandRegistry # logger.info(f"args: {args}") @@ -47,8 +47,8 @@ def server_init(param: "WebServerParameters", system_app: SystemApp): # Loader plugins and commands command_categories = [ - "dbgpt.agent.commands.built_in.audio_text", - "dbgpt.agent.commands.built_in.image_gen", + "dbgpt.agent.plugin.commands.built_in.audio_text", + "dbgpt.agent.plugin.commands.built_in.image_gen", ] # exclude commands command_categories = [ @@ -61,9 +61,9 @@ def server_init(param: "WebServerParameters", system_app: SystemApp): cfg.command_registry = command_registry command_disply_commands = [ - "dbgpt.agent.commands.disply_type.show_chart_gen", - "dbgpt.agent.commands.disply_type.show_table_gen", - "dbgpt.agent.commands.disply_type.show_text_gen", + "dbgpt.agent.plugin.commands.built_in.disply_type.show_chart_gen", + "dbgpt.agent.plugin.commands.built_in.disply_type.show_table_gen", + "dbgpt.agent.plugin.commands.built_in.disply_type.show_text_gen", ] command_disply_registry = CommandRegistry() for command in command_disply_commands: @@ -133,7 +133,7 @@ def _initialize_db(try_to_create_db: Optional[bool] = False) -> str: f"{urlquote(CFG.LOCAL_DB_PASSWORD)}@" f"{CFG.LOCAL_DB_HOST}:" f"{str(CFG.LOCAL_DB_PORT)}/" - f"{db_name}?charset=utf8mb4&collation=utf8mb4_unicode_ci" + f"{db_name}?charset=utf8mb4" ) # Try to create database, if failed, will raise exception _create_mysql_database(db_name, db_url, try_to_create_db) diff --git a/dbgpt/app/component_configs.py b/dbgpt/app/component_configs.py index 8e791d72f..d477a19e4 100644 --- a/dbgpt/app/component_configs.py +++ b/dbgpt/app/component_configs.py @@ -34,10 +34,14 @@ def initialize_components( # system_app.register(DefaultRAGGraphFactory) - from dbgpt.agent.controller import module_agent + from dbgpt.serve.agent.hub.controller import module_agent system_app.register_instance(module_agent) + from dbgpt.serve.agent.agents.controller import multi_agents + + system_app.register_instance(multi_agents) + _initialize_embedding_model( param, system_app, embedding_model_name, embedding_model_path ) diff --git a/dbgpt/app/dbgpt_server.py b/dbgpt/app/dbgpt_server.py index 388d014b2..c8e7893e9 100644 --- a/dbgpt/app/dbgpt_server.py +++ b/dbgpt/app/dbgpt_server.py @@ -26,8 +26,7 @@ # fastapi import time cost about 0.05s from fastapi.staticfiles import StaticFiles -from fastapi import FastAPI, applications -from fastapi.openapi.docs import get_swagger_ui_html +from fastapi import FastAPI from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware @@ -46,18 +45,13 @@ CFG = Config() - -def swagger_monkey_patch(*args, **kwargs): - return get_swagger_ui_html( - *args, - **kwargs, - swagger_js_url="https://cdn.bootcdn.net/ajax/libs/swagger-ui/4.10.3/swagger-ui-bundle.js", - swagger_css_url="https://cdn.bootcdn.net/ajax/libs/swagger-ui/4.10.3/swagger-ui.css", - ) - - -app = FastAPI() -applications.get_swagger_ui_html = swagger_monkey_patch +app = FastAPI( + title="DBGPT OPEN API", + description="This is dbgpt, with auto docs for the API and everything", + version="0.5.0", + openapi_tags=[], +) +# applications.get_swagger_ui_html = swagger_monkey_patch system_app = SystemApp(app) @@ -94,7 +88,7 @@ def mount_routers(app: FastAPI): def mount_static_files(app: FastAPI): - from dbgpt.agent.commands.disply_type.show_chart_gen import ( + from dbgpt.agent.plugin.commands.built_in.disply_type import ( static_message_img_path, ) diff --git a/dbgpt/app/initialization/db_model_initialization.py b/dbgpt/app/initialization/db_model_initialization.py index f236ef649..1d925201a 100644 --- a/dbgpt/app/initialization/db_model_initialization.py +++ b/dbgpt/app/initialization/db_model_initialization.py @@ -1,7 +1,7 @@ """Import all models to make sure they are registered with SQLAlchemy. """ -from dbgpt.agent.db.my_plugin_db import MyPluginEntity -from dbgpt.agent.db.plugin_hub_db import PluginHubEntity +from dbgpt.serve.agent.db.my_plugin_db import MyPluginEntity +from dbgpt.serve.agent.db.plugin_hub_db import PluginHubEntity from dbgpt.app.knowledge.chunk_db import DocumentChunkEntity from dbgpt.app.knowledge.document_db import KnowledgeDocumentEntity from dbgpt.app.knowledge.space_db import KnowledgeSpaceEntity diff --git a/dbgpt/app/openapi/api_v1/api_v1.py b/dbgpt/app/openapi/api_v1/api_v1.py index 2fb4200b9..6c25e6d2c 100644 --- a/dbgpt/app/openapi/api_v1/api_v1.py +++ b/dbgpt/app/openapi/api_v1/api_v1.py @@ -45,6 +45,7 @@ DefaultExecutorFactory, ) + router = APIRouter() CFG = Config() CHAT_FACTORY = ChatFactory() diff --git a/dbgpt/app/scene/chat_agent/chat.py b/dbgpt/app/scene/chat_agent/chat.py index 7ea748faa..a46d70f37 100644 --- a/dbgpt/app/scene/chat_agent/chat.py +++ b/dbgpt/app/scene/chat_agent/chat.py @@ -3,10 +3,10 @@ from dbgpt.app.scene import BaseChat, ChatScene from dbgpt._private.config import Config -from dbgpt.agent.commands.command_mange import ApiCall -from dbgpt.agent import PluginPromptGenerator +from dbgpt.agent.plugin.commands.command_mange import ApiCall +from dbgpt.agent.plugin.generator import PluginPromptGenerator from dbgpt.component import ComponentType -from dbgpt.agent.controller import ModuleAgent +from dbgpt.serve.agent.hub.controller import ModuleAgent from dbgpt.util.tracer import root_tracer, trace CFG = Config() diff --git a/dbgpt/app/scene/chat_data/chat_excel/excel_analyze/chat.py b/dbgpt/app/scene/chat_data/chat_excel/excel_analyze/chat.py index 3cbed545e..4ace7e0b5 100644 --- a/dbgpt/app/scene/chat_data/chat_excel/excel_analyze/chat.py +++ b/dbgpt/app/scene/chat_data/chat_excel/excel_analyze/chat.py @@ -4,7 +4,7 @@ from typing import Dict from dbgpt.app.scene import BaseChat, ChatScene from dbgpt._private.config import Config -from dbgpt.agent.commands.command_mange import ApiCall +from dbgpt.agent.plugin.commands.command_mange import ApiCall from dbgpt.app.scene.chat_data.chat_excel.excel_reader import ExcelReader from dbgpt.app.scene.chat_data.chat_excel.excel_learning.chat import ExcelLearning from dbgpt.util.path_utils import has_path diff --git a/dbgpt/app/scene/chat_data/chat_excel/excel_learning/chat.py b/dbgpt/app/scene/chat_data/chat_excel/excel_learning/chat.py index 72a39f4cf..db9927fd3 100644 --- a/dbgpt/app/scene/chat_data/chat_excel/excel_learning/chat.py +++ b/dbgpt/app/scene/chat_data/chat_excel/excel_learning/chat.py @@ -3,7 +3,7 @@ from dbgpt.core.interface.message import ViewMessage, AIMessage from dbgpt.app.scene import BaseChat, ChatScene -from dbgpt.util.json_utils import DateTimeEncoder +from dbgpt.util.json_utils import EnhancedJSONEncoder from dbgpt.util.executor_utils import blocking_func_to_async from dbgpt.util.tracer import trace @@ -45,7 +45,7 @@ async def generate_input_values(self) -> Dict: datas.insert(0, colunms) input_values = { - "data_example": json.dumps(datas, cls=DateTimeEncoder), + "data_example": json.dumps(datas, cls=EnhancedJSONEncoder), "file_name": self.excel_reader.excel_file_name, } return input_values diff --git a/dbgpt/app/scene/chat_db/auto_execute/chat.py b/dbgpt/app/scene/chat_db/auto_execute/chat.py index 13790795b..3a3269d7e 100644 --- a/dbgpt/app/scene/chat_db/auto_execute/chat.py +++ b/dbgpt/app/scene/chat_db/auto_execute/chat.py @@ -1,6 +1,6 @@ from typing import Dict -from dbgpt.agent.commands.command_mange import ApiCall +from dbgpt.agent.plugin.commands.command_mange import ApiCall from dbgpt.app.scene import BaseChat, ChatScene from dbgpt._private.config import Config from dbgpt.util.executor_utils import blocking_func_to_async diff --git a/dbgpt/app/scene/chat_execution/chat.py b/dbgpt/app/scene/chat_execution/chat.py index 8c484f53c..b1c4f75c1 100644 --- a/dbgpt/app/scene/chat_execution/chat.py +++ b/dbgpt/app/scene/chat_execution/chat.py @@ -2,8 +2,8 @@ from dbgpt.app.scene import BaseChat, ChatScene from dbgpt._private.config import Config -from dbgpt.agent.commands.command import execute_command -from dbgpt.agent import PluginPromptGenerator +from dbgpt.agent.plugin.commands.command import execute_command +from dbgpt.agent.plugin.generator import PluginPromptGenerator from dbgpt.util.tracer import trace CFG = Config() diff --git a/dbgpt/component.py b/dbgpt/component.py index 84ea9afde..d10d500e0 100644 --- a/dbgpt/component.py +++ b/dbgpt/component.py @@ -52,6 +52,7 @@ class ComponentType(str, Enum): MODEL_API_SERVER = "dbgpt_model_api_server" MODEL_CACHE_MANAGER = "dbgpt_model_cache_manager" AGENT_HUB = "dbgpt_agent_hub" + MULTI_AGENTS = "dbgpt_multi_agents" EXECUTOR_DEFAULT = "dbgpt_thread_pool_default" TRACER = "dbgpt_tracer" TRACER_SPAN_STORAGE = "dbgpt_tracer_span_storage" diff --git a/dbgpt/core/interface/output_parser.py b/dbgpt/core/interface/output_parser.py index 05a5003d9..1fdac4510 100644 --- a/dbgpt/core/interface/output_parser.py +++ b/dbgpt/core/interface/output_parser.py @@ -84,7 +84,7 @@ def parse_model_nostream_resp(self, response: ResponseTye, sep: str): ai_response = ai_response.replace("\*", "*") ai_response = ai_response.replace("\t", "") - ai_response = ai_response.strip().replace("\\n", " ").replace("\n", " ") + # ai_response = ai_response.strip().replace("\\n", " ").replace("\n", " ") print("un_stream ai response:", ai_response) return ai_response else: diff --git a/dbgpt/datasource/rdbms/base.py b/dbgpt/datasource/rdbms/base.py index a8d0846f4..d38330988 100644 --- a/dbgpt/datasource/rdbms/base.py +++ b/dbgpt/datasource/rdbms/base.py @@ -334,7 +334,7 @@ def query_ex(self, query, fetch: str = "all"): """ print(f"Query[{query}]") if not query: - return [] + return [], None cursor = self.session.execute(text(query)) if cursor.returns_rows: if fetch == "all": @@ -347,7 +347,7 @@ def query_ex(self, query, fetch: str = "all"): result = list(result) return field_names, result - return [] + return [], None def run(self, command: str, fetch: str = "all") -> List: """Execute a SQL command and return a string representing the results.""" diff --git a/dbgpt/model/proxy/llms/chatgpt.py b/dbgpt/model/proxy/llms/chatgpt.py index 54b2ae2c9..1b98e8a8a 100755 --- a/dbgpt/model/proxy/llms/chatgpt.py +++ b/dbgpt/model/proxy/llms/chatgpt.py @@ -91,34 +91,43 @@ def _initialize_openai_v1(params: ProxyModelParameters): return openai_params, api_type, api_version, proxies -def _build_request(model: ProxyModel, params): - history = [] +def __convert_2_gpt_messages(messages: List[ModelMessage]): + chat_round = 0 + gpt_messages = [] + last_usr_message = "" + system_messages = [] + + for message in messages: + if message.role == ModelMessageRoleType.HUMAN or message.role == "user": + last_usr_message = message.content + elif message.role == ModelMessageRoleType.SYSTEM: + system_messages.append(message.content) + elif message.role == ModelMessageRoleType.AI or message.role == "assistant": + last_ai_message = message.content + gpt_messages.append({"role": "user", "content": last_usr_message}) + gpt_messages.append({"role": "assistant", "content": last_ai_message}) + + if len(system_messages) > 0: + if len(system_messages) < 2: + gpt_messages.insert(0, {"role": "system", "content": system_messages[0]}) + gpt_messages.append({"role": "user", "content": last_usr_message}) + else: + gpt_messages.append({"role": "user", "content": system_messages[1]}) + else: + last_message = messages[-1] + if last_message.role == ModelMessageRoleType.HUMAN: + gpt_messages.append({"role": "user", "content": last_message.content}) + return gpt_messages + + +def _build_request(model: ProxyModel, params): model_params = model.get_params() logger.info(f"Model: {model}, model_params: {model_params}") messages: List[ModelMessage] = params["messages"] - # Add history conversation - for message in messages: - if message.role == ModelMessageRoleType.HUMAN: - history.append({"role": "user", "content": message.content}) - elif message.role == ModelMessageRoleType.SYSTEM: - history.append({"role": "system", "content": message.content}) - elif message.role == ModelMessageRoleType.AI: - history.append({"role": "assistant", "content": message.content}) - else: - pass - - # Move the last user's information to the end - last_user_input_index = None - for i in range(len(history) - 1, -1, -1): - if history[i]["role"] == "user": - last_user_input_index = i - break - if last_user_input_index: - last_user_input = history.pop(last_user_input_index) - history.append(last_user_input) + history = __convert_2_gpt_messages(messages) payloads = { "temperature": params.get("temperature"), "max_tokens": params.get("max_new_tokens"), diff --git a/dbgpt/model/proxy/llms/tongyi.py b/dbgpt/model/proxy/llms/tongyi.py index 33c11ce4e..902b26c4a 100644 --- a/dbgpt/model/proxy/llms/tongyi.py +++ b/dbgpt/model/proxy/llms/tongyi.py @@ -25,6 +25,7 @@ def __convert_2_tongyi_messages(messages: List[ModelMessage]): if len(system_messages) > 0: if len(system_messages) < 2: tongyi_messages.insert(0, {"role": "system", "content": system_messages[0]}) + tongyi_messages.append({"role": "user", "content": last_usr_message}) else: tongyi_messages.append({"role": "user", "content": system_messages[1]}) else: diff --git a/dbgpt/serve/agent/agents/__init__.py b/dbgpt/serve/agent/agents/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbgpt/serve/agent/agents/controller.py b/dbgpt/serve/agent/agents/controller.py new file mode 100644 index 000000000..a6bc7d706 --- /dev/null +++ b/dbgpt/serve/agent/agents/controller.py @@ -0,0 +1,338 @@ +import logging +import json +import asyncio +import uuid +from collections import defaultdict +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union +from fastapi import ( + APIRouter, + Body, + UploadFile, + File, +) +from fastapi.responses import StreamingResponse + +from abc import ABC +from typing import List +from dbgpt.core.awel import BaseOperator, SimpleCallDataInputSource, InputOperator, DAG +from dbgpt.model.operator.model_operator import ModelOperator, ModelStreamOperator +from dbgpt.app.openapi.api_view_model import Result, ConversationVo +from dbgpt.util.json_utils import EnhancedJSONEncoder +from dbgpt.serve.agent.model import ( + PluginHubParam, + PagenationFilter, + PagenationResult, + PluginHubFilter, +) + +from dbgpt.agent.common.schema import Status +from dbgpt.agent.agents.agents_mange import AgentsMange +from dbgpt.agent.agents.planner_agent import PlannerAgent +from dbgpt.agent.agents.user_proxy_agent import UserProxyAgent +from dbgpt.agent.agents.plan_group_chat import PlanChat, PlanChatManager +from dbgpt.agent.agents.agent import AgentContext +from dbgpt.agent.memory.gpts_memory import GptsMemory + +from .db_gpts_memory import MetaDbGptsPlansMemory, MetaDbGptsMessageMemory + +from ..db.gpts_mange_db import GptsInstanceDao, GptsInstanceEntity +from ..db.gpts_conversations_db import GptsConversationsDao, GptsConversationsEntity + +from .dbgpts import DbGptsCompletion, DbGptsTaskStep, DbGptsMessage, DbGptsInstance +from dbgpt.component import BaseComponent, ComponentType, SystemApp +from dbgpt.agent.agents.agents_mange import agent_mange +from dbgpt._private.config import Config +from dbgpt.model.cluster.controller.controller import BaseModelController +from dbgpt.agent.memory.gpts_memory import GptsMessage +from dbgpt.model.cluster import WorkerManager, WorkerManagerFactory + +from dbgpt.model.cluster.client import DefaultLLMClient + +CFG = Config() + +import asyncio + +router = APIRouter() +logger = logging.getLogger(__name__) + + +class MultiAgents(BaseComponent, ABC): + name = ComponentType.MULTI_AGENTS + + def init_app(self, system_app: SystemApp): + system_app.app.include_router(router, prefix="/api", tags=["Multi-Agents"]) + + def __init__(self): + self.gpts_intance = GptsInstanceDao() + self.gpts_conversations = GptsConversationsDao() + self.memory = GptsMemory( + plans_memory=MetaDbGptsPlansMemory(), + message_memory=MetaDbGptsMessageMemory(), + ) + + def gpts_create(self, entity: GptsInstanceEntity): + self.gpts_intance.add(entity) + + async def plan_chat( + self, + name: str, + user_query: str, + conv_id: str, + user_code: str = None, + sys_code: str = None, + ): + gpts_instance: GptsInstanceEntity = self.gpts_intance.get_by_name(name) + if gpts_instance is None: + raise ValueError(f"can't find dbgpts!{name}") + agents_names = json.loads(gpts_instance.gpts_agents) + llm_models_priority = json.loads(gpts_instance.gpts_models) + resource_db = ( + json.loads(gpts_instance.resource_db) if gpts_instance.resource_db else None + ) + resource_knowledge = ( + json.loads(gpts_instance.resource_knowledge) + if gpts_instance.resource_knowledge + else None + ) + resource_internet = ( + json.loads(gpts_instance.resource_internet) + if gpts_instance.resource_internet + else None + ) + + ### init chat param + worker_manager = CFG.SYSTEM_APP.get_component( + ComponentType.WORKER_MANAGER_FACTORY, WorkerManagerFactory + ).create() + llm_task = DefaultLLMClient(worker_manager) + context: AgentContext = AgentContext(conv_id=conv_id, llm_provider=llm_task) + context.gpts_name = gpts_instance.gpts_name + context.resource_db = resource_db + context.resource_internet = resource_internet + context.resource_knowledge = resource_knowledge + context.agents = agents_names + + context.llm_models = await llm_task.models() + context.model_priority = llm_models_priority + + agent_map = defaultdict() + + ### default plan excute mode + agents = [] + for name in agents_names: + cls = agent_mange.get_by_name(name) + agent = cls( + agent_context=context, + memory=self.memory, + ) + agents.append(agent) + agent_map[name] = agent + + groupchat = PlanChat(agents=agents, messages=[], max_round=50) + planner = PlannerAgent( + agent_context=context, + memory=self.memory, + plan_chat=groupchat, + ) + agent_map[planner.name] = planner + + manager = PlanChatManager( + agent_context=context, + memory=self.memory, + plan_chat=groupchat, + planner=planner, + ) + agent_map[manager.name] = manager + + user_proxy = UserProxyAgent(memory=self.memory, agent_context=context) + agent_map[user_proxy.name] = user_proxy + + gpts_conversation = self.gpts_conversations.get_by_conv_id(conv_id) + if gpts_conversation is None: + self.gpts_conversations.add( + GptsConversationsEntity( + conv_id=conv_id, + user_goal=user_query, + gpts_name=gpts_instance.gpts_name, + state=Status.RUNNING.value, + max_auto_reply_round=context.max_chat_round, + auto_reply_count=0, + user_code=user_code, + sys_code=sys_code, + ) + ) + + ## dbgpts conversation save + try: + await user_proxy.a_initiate_chat( + recipient=manager, + message=user_query, + memory=self.memory, + ) + except Exception as e: + logger.error(f"chat abnormal termination！{str(e)}", e) + self.gpts_conversations.update(conv_id, Status.FAILED.value) + + else: + # retry chat + self.gpts_conversations.update(conv_id, Status.RUNNING.value) + try: + await user_proxy.a_retry_chat( + recipient=manager, + agent_map=agent_map, + memory=self.memory, + ) + except Exception as e: + logger.error(f"chat abnormal termination！{str(e)}", e) + self.gpts_conversations.update(conv_id, Status.FAILED.value) + + self.gpts_conversations.update(conv_id, Status.COMPLETE.value) + return conv_id + + async def chat_completions( + self, conv_id: str, user_code: str = None, system_app: str = None + ): + is_complete = False + while True: + gpts_conv = self.gpts_conversations.get_by_conv_id(conv_id) + if gpts_conv: + is_complete = ( + True + if gpts_conv.state + in [ + Status.COMPLETE.value, + Status.WAITING.value, + Status.FAILED.value, + ] + else False + ) + yield await self.memory.one_plan_chat_competions(conv_id) + if is_complete: + return + else: + await asyncio.sleep(5) + + async def stable_message( + self, conv_id: str, user_code: str = None, system_app: str = None + ): + gpts_conv = self.gpts_conversations.get_by_conv_id(conv_id) + if gpts_conv: + is_complete = ( + True + if gpts_conv.state + in [Status.COMPLETE.value, Status.WAITING.value, Status.FAILED.value] + else False + ) + if is_complete: + return await self.self.memory.one_plan_chat_competions(conv_id) + else: + raise ValueError( + "The conversation has not been completed yet, so we cannot directly obtain information." + ) + else: + raise ValueError("No conversation record found!") + + def gpts_conv_list(self, user_code: str = None, system_app: str = None): + return self.gpts_conversations.get_convs(user_code, system_app) + + +multi_agents = MultiAgents() + + +@router.post("/v1/dbbgpts/agents/list", response_model=Result[str]) +async def agents_list(): + logger.info("agents_list!") + try: + agents = agent_mange.all_agents() + return Result.succ(agents) + except Exception as e: + return Result.failed(code="E30001", msg=str(e)) + + +@router.post("/v1/dbbgpts/create", response_model=Result[str]) +async def create_dbgpts(gpts_instance: DbGptsInstance = Body()): + logger.info(f"create_dbgpts:{gpts_instance}") + try: + multi_agents.gpts_create( + GptsInstanceEntity( + gpts_name=gpts_instance.gpts_name, + gpts_describe=gpts_instance.gpts_describe, + resource_db=json.dumps(gpts_instance.resource_db.to_dict()), + resource_internet=json.dumps(gpts_instance.resource_internet.to_dict()), + resource_knowledge=json.dumps( + gpts_instance.resource_knowledge.to_dict() + ), + gpts_agents=json.dumps(gpts_instance.gpts_agents), + gpts_models=json.dumps(gpts_instance.gpts_models), + language=gpts_instance.language, + user_code=gpts_instance.user_code, + sys_code=gpts_instance.sys_code, + ) + ) + return Result.succ(None) + except Exception as e: + logger.error(f"create_dbgpts failed:{str(e)}") + return Result.failed(msg=str(e), code="E300002") + + +async def stream_generator(conv_id: str): + async for chunk in multi_agents.chat_completions(conv_id): + if chunk: + yield f"data: {chunk}\n\n" + + +@router.post("/v1/dbbgpts/chat/plan/completions", response_model=Result[str]) +async def dgpts_completions( + gpts_name: str, + user_query: str, + conv_id: str = None, + user_code: str = None, + sys_code: str = None, +): + logger.info(f"dgpts_completions:{gpts_name},{user_query},{conv_id}") + if conv_id is None: + conv_id = str(uuid.uuid1()) + asyncio.create_task( + multi_agents.plan_chat(gpts_name, user_query, conv_id, user_code, sys_code) + ) + + headers = { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "Transfer-Encoding": "chunked", + } + return StreamingResponse( + stream_generator(conv_id), + headers=headers, + media_type="text/plain", + ) + + +@router.post("/v1/dbbgpts/plan/chat/cancel", response_model=Result[str]) +async def dgpts_plan_chat_cancel( + conv_id: str = None, user_code: str = None, sys_code: str = None +): + pass + + +@router.get("/v1/dbbgpts/chat/plan/messages", response_model=Result[str]) +async def plan_chat_messages(conv_id: str, user_code: str = None, sys_code: str = None): + logger.info(f"plan_chat_messages:{conv_id},{user_code},{sys_code}") + + headers = { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "Transfer-Encoding": "chunked", + } + return StreamingResponse( + stream_generator(conv_id), + headers=headers, + media_type="text/plain", + ) + + +@router.post("/v1/dbbgpts/chat/feedback", response_model=Result[str]) +async def dgpts_chat_feedback(filter: PagenationFilter[PluginHubFilter] = Body()): + pass diff --git a/dbgpt/serve/agent/agents/db_gpts_memory.py b/dbgpt/serve/agent/agents/db_gpts_memory.py new file mode 100644 index 000000000..ba6d9a6a8 --- /dev/null +++ b/dbgpt/serve/agent/agents/db_gpts_memory.py @@ -0,0 +1,121 @@ +from typing import List, Optional +from dbgpt.agent.memory.gpts_memory import ( + GptsPlansMemory, + GptsPlan, + GptsMessageMemory, + GptsMessage, +) + +from ..db.gpts_plans_db import GptsPlansEntity, GptsPlansDao +from ..db.gpts_messages_db import GptsMessagesDao, GptsMessagesEntity + + +class MetaDbGptsPlansMemory(GptsPlansMemory): + def __init__(self): + self.gpts_plan = GptsPlansDao() + + def batch_save(self, plans: list[GptsPlan]): + self.gpts_plan.batch_save([item.to_dict() for item in plans]) + + def get_by_conv_id(self, conv_id: str) -> List[GptsPlan]: + db_results: List[GptsPlansEntity] = self.gpts_plan.get_by_conv_id( + conv_id=conv_id + ) + results = [] + for item in db_results: + results.append(GptsPlan.from_dict(item.__dict__)) + return results + + def get_by_conv_id_and_num( + self, conv_id: str, task_nums: List[int] + ) -> List[GptsPlan]: + db_results: List[GptsPlansEntity] = self.gpts_plan.get_by_conv_id_and_num( + conv_id=conv_id, task_nums=task_nums + ) + results = [] + for item in db_results: + results.append(GptsPlan.from_dict(item.__dict__)) + return results + + def get_todo_plans(self, conv_id: str) -> List[GptsPlan]: + db_results: List[GptsPlansEntity] = self.gpts_plan.get_todo_plans( + conv_id=conv_id + ) + results = [] + for item in db_results: + results.append(GptsPlan.from_dict(item.__dict__)) + return results + + def complete_task(self, conv_id: str, task_num: int, result: str): + self.gpts_plan.complete_task(conv_id=conv_id, task_num=task_num, result=result) + + def update_task( + self, + conv_id: str, + task_num: int, + state: str, + retry_times: int, + agent: str = None, + model: str = None, + result: str = None, + ): + self.gpts_plan.update_task( + conv_id=conv_id, + task_num=task_num, + state=state, + retry_times=retry_times, + agent=agent, + model=model, + result=result, + ) + + def remove_by_conv_id(self, conv_id: str): + self.gpts_plan.remove_by_conv_id(conv_id=conv_id) + + +class MetaDbGptsMessageMemory(GptsMessageMemory): + def __init__(self): + self.gpts_message = GptsMessagesDao() + + def append(self, message: GptsMessage): + self.gpts_message.append(message.to_dict()) + + def get_by_agent(self, conv_id: str, agent: str) -> Optional[List[GptsMessage]]: + db_results = self.gpts_message.get_by_agent(conv_id, agent) + results = [] + db_results = sorted(db_results, key=lambda x: x.rounds) + for item in db_results: + results.append(GptsMessage.from_dict(item.__dict__)) + return results + + def get_between_agents( + self, + conv_id: str, + agent1: str, + agent2: str, + current_gogal: Optional[str] = None, + ) -> Optional[List[GptsMessage]]: + db_results = self.gpts_message.get_between_agents( + conv_id, agent1, agent2, current_gogal + ) + results = [] + db_results = sorted(db_results, key=lambda x: x.rounds) + for item in db_results: + results.append(GptsMessage.from_dict(item.__dict__)) + return results + + def get_by_conv_id(self, conv_id: str) -> Optional[List[GptsMessage]]: + db_results = self.gpts_message.get_by_conv_id(conv_id) + + results = [] + db_results = sorted(db_results, key=lambda x: x.rounds) + for item in db_results: + results.append(GptsMessage.from_dict(item.__dict__)) + return results + + def get_last_message(self, conv_id: str) -> Optional[GptsMessage]: + db_result = self.gpts_message.get_last_message(conv_id) + if db_result: + return GptsMessage.from_dict(db_result.__dict__) + else: + return None diff --git a/dbgpt/serve/agent/agents/dbgpts.py b/dbgpt/serve/agent/agents/dbgpts.py new file mode 100644 index 000000000..37173fb73 --- /dev/null +++ b/dbgpt/serve/agent/agents/dbgpts.py @@ -0,0 +1,93 @@ +from __future__ import annotations + +from enum import Enum +from abc import ABC, abstractmethod +from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union +from dataclasses import dataclass, asdict, fields +import dataclasses + +from dbgpt.agent.agents.agent import AgentResource + + +class AgentMode(Enum): + PLAN_EXCUTE = "plan_excute" + + +@dataclass +class DbGptsInstance: + gpts_name: str + gpts_describe: str + gpts_agents: list[str] + resource_db: Optional[AgentResource] = None + resource_internet: Optional[AgentResource] = None + resource_knowledge: Optional[AgentResource] = None + gpts_models: Optional[Dict[str, List[str]]] = None + language: str = "en" + user_code: str = None + sys_code: str = None + + def to_dict(self) -> Dict[str, Any]: + return dataclasses.asdict(self) + + +@dataclass +class DbGptsMessage: + sender: str + receiver: str + content: str + action_report: str + + @staticmethod + def from_dict(d: Dict[str, Any]) -> DbGptsMessage: + return DbGptsMessage( + sender=d["sender"], + receiver=d["receiver"], + content=d["content"], + model_name=d["model_name"], + agent_name=d["agent_name"], + ) + + def to_dict(self) -> Dict[str, Any]: + return dataclasses.asdict(self) + + +@dataclass +class DbGptsTaskStep: + task_num: str + task_content: str + state: str + result: str + agent_name: str + model_name: str + + @staticmethod + def from_dict(d: Dict[str, Any]) -> DbGptsTaskStep: + return DbGptsTaskStep( + task_num=d["task_num"], + task_content=d["task_content"], + state=d["state"], + result=d["result"], + agent_name=d["agent_name"], + model_name=d["model_name"], + ) + + def to_dict(self) -> Dict[str, Any]: + return dataclasses.asdict(self) + + +@dataclass +class DbGptsCompletion: + conv_id: str + task_steps: Optional[List[DbGptsTaskStep]] + messages: Optional[List[DbGptsMessage]] + + @staticmethod + def from_dict(d: Dict[str, Any]) -> DbGptsCompletion: + return DbGptsCompletion( + conv_id=d.get("conv_id"), + task_steps=DbGptsTaskStep.from_dict(d["task_steps"]), + messages=DbGptsMessage.from_dict(d["messages"]), + ) + + def to_dict(self) -> Dict[str, Any]: + return dataclasses.asdict(self) diff --git a/dbgpt/serve/agent/db/__init__.py b/dbgpt/serve/agent/db/__init__.py new file mode 100644 index 000000000..c8db61a65 --- /dev/null +++ b/dbgpt/serve/agent/db/__init__.py @@ -0,0 +1,6 @@ +from .gpts_conversations_db import GptsConversationsDao, GptsConversationsEntity +from .gpts_mange_db import GptsInstanceDao, GptsInstanceEntity +from .gpts_messages_db import GptsMessagesDao, GptsMessagesEntity +from .gpts_plans_db import GptsPlansDao, GptsPlansEntity +from .my_plugin_db import MyPluginDao, MyPluginEntity +from .plugin_hub_db import PluginHubDao, PluginHubEntity diff --git a/dbgpt/serve/agent/db/gpts_conversations_db.py b/dbgpt/serve/agent/db/gpts_conversations_db.py new file mode 100644 index 000000000..67375592b --- /dev/null +++ b/dbgpt/serve/agent/db/gpts_conversations_db.py @@ -0,0 +1,95 @@ +from datetime import datetime +from typing import List +from sqlalchemy import Column, Integer, String, Index, DateTime, func, Text, desc +from sqlalchemy import UniqueConstraint + + +from dbgpt.storage.metadata import BaseDao, Model + + +class GptsConversationsEntity(Model): + __tablename__ = "gpts_conversations" + __table_args__ = { + "mysql_charset": "utf8mb4", + "mysql_collate": "utf8mb4_unicode_ci", + } + + id = Column(Integer, primary_key=True, comment="autoincrement id") + + conv_id = Column( + String(255), nullable=False, comment="The unique id of the conversation record" + ) + user_goal = Column(Text, nullable=False, comment="User's goals content") + + gpts_name = Column(String(255), nullable=False, comment="The gpts name") + state = Column(String(255), nullable=True, comment="The gpts state") + + max_auto_reply_round = Column( + Integer, nullable=False, comment="max auto reply round" + ) + auto_reply_count = Column(Integer, nullable=False, comment="auto reply count") + + user_code = Column(String(255), nullable=True, comment="user code") + sys_code = Column(String(255), nullable=True, comment="system app ") + + created_at = Column(DateTime, default=datetime.utcnow, comment="create time") + updated_at = Column( + DateTime, + default=datetime.utcnow, + onupdate=datetime.utcnow, + comment="last update time", + ) + + __table_args__ = ( + UniqueConstraint("conv_id", name="uk_gpts_conversations"), + Index("idx_gpts_name", "gpts_name"), + ) + + +class GptsConversationsDao(BaseDao): + def add(self, engity: GptsConversationsEntity): + session = self.get_raw_session() + session.add(engity) + session.commit() + id = engity.id + session.close() + return id + + def get_by_conv_id(self, conv_id: str): + session = self.get_raw_session() + gpts_conv = session.query(GptsConversationsEntity) + if conv_id: + gpts_conv = gpts_conv.filter(GptsConversationsEntity.conv_id == conv_id) + result = gpts_conv.first() + session.close() + return result + + def get_convs(self, user_code: str = None, system_app: str = None): + session = self.get_raw_session() + gpts_conversations = session.query(GptsConversationsEntity) + if user_code: + gpts_conversations = gpts_conversations.filter( + GptsConversationsEntity.user_code == user_code + ) + if system_app: + gpts_conversations = gpts_conversations.filter( + GptsConversationsEntity.system_app == system_app + ) + + result = ( + gpts_conversations.limit(20) + .order_by(desc(GptsConversationsEntity.id)) + .all() + ) + session.close() + return result + + def update(self, conv_id: str, state: str): + session = self.get_raw_session() + gpts_convs = session.query(GptsConversationsEntity) + gpts_convs = gpts_convs.filter(GptsConversationsEntity.conv_id == conv_id) + gpts_convs.update( + {GptsConversationsEntity.state: state}, synchronize_session="fetch" + ) + session.commit() + session.close() diff --git a/dbgpt/serve/agent/db/gpts_mange_db.py b/dbgpt/serve/agent/db/gpts_mange_db.py new file mode 100644 index 000000000..91f578e9f --- /dev/null +++ b/dbgpt/serve/agent/db/gpts_mange_db.py @@ -0,0 +1,78 @@ +from datetime import datetime +from typing import List +from sqlalchemy import Column, Integer, String, Index, DateTime, func, Text, Boolean +from sqlalchemy import UniqueConstraint + +from dbgpt.storage.metadata import BaseDao, Model + + +class GptsInstanceEntity(Model): + __tablename__ = "gpts_instance" + __table_args__ = { + "mysql_charset": "utf8mb4", + "mysql_collate": "utf8mb4_unicode_ci", + } + id = Column(Integer, primary_key=True, comment="autoincrement id") + + gpts_name = Column(String(255), nullable=False, comment="Current AI assistant name") + gpts_describe = Column( + String(2255), nullable=False, comment="Current AI assistant describe" + ) + resource_db = Column( + Text, + nullable=True, + comment="List of structured database names contained in the current gpts", + ) + resource_internet = Column( + Text, + nullable=True, + comment="Is it possible to retrieve information from the internet", + ) + resource_knowledge = Column( + Text, + nullable=True, + comment="List of unstructured database names contained in the current gpts", + ) + gpts_agents = Column( + String(1000), + nullable=True, + comment="List of agents names contained in the current gpts", + ) + gpts_models = Column( + String(1000), + nullable=True, + comment="List of llm model names contained in the current gpts", + ) + language = Column(String(100), nullable=True, comment="gpts language") + + user_code = Column(String(255), nullable=False, comment="user code") + sys_code = Column(String(255), nullable=True, comment="system app code") + + created_at = Column(DateTime, default=datetime.utcnow, comment="create time") + updated_at = Column( + DateTime, + default=datetime.utcnow, + onupdate=datetime.utcnow, + comment="last update time", + ) + + __table_args__ = (UniqueConstraint("gpts_name", name="uk_gpts"),) + + +class GptsInstanceDao(BaseDao): + def add(self, engity: GptsInstanceEntity): + session = self.get_raw_session() + session.add(engity) + session.commit() + id = engity.id + session.close() + return id + + def get_by_name(self, name: str) -> GptsInstanceEntity: + session = self.get_raw_session() + gpts_instance = session.query(GptsInstanceEntity) + if name: + gpts_instance = gpts_instance.filter(GptsInstanceEntity.gpts_name == name) + result = gpts_instance.first() + session.close() + return result diff --git a/dbgpt/serve/agent/db/gpts_messages_db.py b/dbgpt/serve/agent/db/gpts_messages_db.py new file mode 100644 index 000000000..2945fe59e --- /dev/null +++ b/dbgpt/serve/agent/db/gpts_messages_db.py @@ -0,0 +1,160 @@ +from datetime import datetime +from typing import List, Optional +from sqlalchemy import ( + Column, + Integer, + String, + Index, + DateTime, + func, + Text, + or_, + and_, + desc, +) +from sqlalchemy import UniqueConstraint + +from dbgpt.storage.metadata import BaseDao, Model + + +class GptsMessagesEntity(Model): + __tablename__ = "gpts_messages" + __table_args__ = { + "mysql_charset": "utf8mb4", + "mysql_collate": "utf8mb4_unicode_ci", + } + id = Column(Integer, primary_key=True, comment="autoincrement id") + + conv_id = Column( + String(255), nullable=False, comment="The unique id of the conversation record" + ) + sender = Column( + String(255), + nullable=False, + comment="Who speaking in the current conversation turn", + ) + receiver = Column( + String(255), + nullable=False, + comment="Who receive message in the current conversation turn", + ) + model_name = Column(String(255), nullable=True, comment="message generate model") + rounds = Column(Integer, nullable=False, comment="dialogue turns") + content = Column(Text, nullable=True, comment="Content of the speech") + current_gogal = Column( + Text, nullable=True, comment="The target corresponding to the current message" + ) + context = Column(Text, nullable=True, comment="Current conversation context") + review_info = Column( + Text, nullable=True, comment="Current conversation review info" + ) + action_report = Column( + Text, nullable=True, comment="Current conversation action report" + ) + + role = Column( + String(255), nullable=True, comment="The role of the current message content" + ) + + created_at = Column(DateTime, default=datetime.utcnow, comment="create time") + updated_at = Column( + DateTime, + default=datetime.utcnow, + onupdate=datetime.utcnow, + comment="last update time", + ) + __table_args__ = (Index("idx_q_messages", "conv_id", "rounds", "sender"),) + + +class GptsMessagesDao(BaseDao): + def append(self, entity: dict): + session = self.get_raw_session() + message = GptsMessagesEntity( + conv_id=entity.get("conv_id"), + sender=entity.get("sender"), + receiver=entity.get("receiver"), + content=entity.get("content"), + role=entity.get("role", None), + model_name=entity.get("model_name", None), + context=entity.get("context", None), + rounds=entity.get("rounds", None), + current_gogal=entity.get("current_gogal", None), + review_info=entity.get("review_info", None), + action_report=entity.get("action_report", None), + ) + session.add(message) + session.commit() + id = message.id + session.close() + return id + + def get_by_agent( + self, conv_id: str, agent: str + ) -> Optional[List[GptsMessagesEntity]]: + session = self.get_raw_session() + gpts_messages = session.query(GptsMessagesEntity) + if agent: + gpts_messages = gpts_messages.filter( + GptsMessagesEntity.conv_id == conv_id + ).filter( + or_( + GptsMessagesEntity.sender == agent, + GptsMessagesEntity.receiver == agent, + ) + ) + result = gpts_messages.order_by(GptsMessagesEntity.rounds).all() + session.close() + return result + + def get_by_conv_id(self, conv_id: str) -> Optional[List[GptsMessagesEntity]]: + session = self.get_raw_session() + gpts_messages = session.query(GptsMessagesEntity) + if conv_id: + gpts_messages = gpts_messages.filter(GptsMessagesEntity.conv_id == conv_id) + result = gpts_messages.order_by(GptsMessagesEntity.rounds).all() + session.close() + return result + + def get_between_agents( + self, + conv_id: str, + agent1: str, + agent2: str, + current_gogal: Optional[str] = None, + ) -> Optional[List[GptsMessagesEntity]]: + session = self.get_raw_session() + gpts_messages = session.query(GptsMessagesEntity) + if agent1 and agent2: + gpts_messages = gpts_messages.filter( + GptsMessagesEntity.conv_id == conv_id + ).filter( + or_( + and_( + GptsMessagesEntity.sender == agent1, + GptsMessagesEntity.receiver == agent2, + ), + and_( + GptsMessagesEntity.sender == agent2, + GptsMessagesEntity.receiver == agent1, + ), + ) + ) + if current_gogal: + gpts_messages = gpts_messages.filter( + GptsMessagesEntity.current_gogal == current_gogal + ) + result = gpts_messages.order_by(GptsMessagesEntity.rounds).all() + session.close() + return result + + def get_last_message(self, conv_id: str) -> Optional[GptsMessagesEntity]: + session = self.get_raw_session() + gpts_messages = session.query(GptsMessagesEntity) + if conv_id: + gpts_messages = gpts_messages.filter( + GptsMessagesEntity.conv_id == conv_id + ).order_by(desc(GptsMessagesEntity.rounds)) + + result = gpts_messages.first() + session.close() + return result diff --git a/dbgpt/serve/agent/db/gpts_plans_db.py b/dbgpt/serve/agent/db/gpts_plans_db.py new file mode 100644 index 000000000..f01840cbf --- /dev/null +++ b/dbgpt/serve/agent/db/gpts_plans_db.py @@ -0,0 +1,156 @@ +from datetime import datetime +from typing import List +from sqlalchemy import Column, Integer, String, Index, DateTime, func, Text +from sqlalchemy import UniqueConstraint + +from dbgpt.storage.metadata import BaseDao, Model +from dbgpt.agent.common.schema import Status + + +class GptsPlansEntity(Model): + __tablename__ = "gpts_plans" + __table_args__ = { + "mysql_charset": "utf8mb4", + "mysql_collate": "utf8mb4_unicode_ci", + } + id = Column(Integer, primary_key=True, comment="autoincrement id") + + conv_id = Column( + String(255), nullable=False, comment="The unique id of the conversation record" + ) + sub_task_num = Column(Integer, nullable=False, comment="Subtask number") + sub_task_title = Column(String(255), nullable=False, comment="subtask title") + sub_task_content = Column(Text, nullable=False, comment="subtask content") + sub_task_agent = Column( + String(255), nullable=True, comment="Available agents corresponding to subtasks" + ) + resource_name = Column(String(255), nullable=True, comment="resource name") + rely = Column( + String(255), nullable=True, comment="Subtask dependencies，like: 1,2,3" + ) + + agent_model = Column( + String(255), + nullable=True, + comment="LLM model used by subtask processing agents", + ) + retry_times = Column(Integer, default=False, comment="number of retries") + max_retry_times = Column( + Integer, default=False, comment="Maximum number of retries" + ) + state = Column(String(255), nullable=True, comment="subtask status") + result = Column(Text(length=2**31 - 1), nullable=True, comment="subtask result") + + created_at = Column(DateTime, default=datetime.utcnow, comment="create time") + updated_at = Column( + DateTime, + default=datetime.utcnow, + onupdate=datetime.utcnow, + comment="last update time", + ) + __table_args__ = (UniqueConstraint("conv_id", "sub_task_num", name="uk_sub_task"),) + + +class GptsPlansDao(BaseDao): + def batch_save(self, plans: list[dict]): + session = self.get_raw_session() + session.bulk_insert_mappings(GptsPlansEntity, plans) + session.commit() + session.close() + + def get_by_conv_id(self, conv_id: str) -> list[GptsPlansEntity]: + session = self.get_raw_session() + gpts_plans = session.query(GptsPlansEntity) + if conv_id: + gpts_plans = gpts_plans.filter(GptsPlansEntity.conv_id == conv_id) + result = gpts_plans.all() + session.close() + return result + + def get_by_task_id(self, task_id: int) -> list[GptsPlansEntity]: + session = self.get_raw_session() + gpts_plans = session.query(GptsPlansEntity) + if task_id: + gpts_plans = gpts_plans.filter(GptsPlansEntity.id == task_id) + result = gpts_plans.first() + session.close() + return result + + def get_by_conv_id_and_num( + self, conv_id: str, task_nums: list + ) -> list[GptsPlansEntity]: + session = self.get_raw_session() + gpts_plans = session.query(GptsPlansEntity) + if conv_id: + gpts_plans = gpts_plans.filter(GptsPlansEntity.conv_id == conv_id).filter( + GptsPlansEntity.sub_task_num.in_(task_nums) + ) + result = gpts_plans.all() + session.close() + return result + + def get_todo_plans(self, conv_id: str) -> list[GptsPlansEntity]: + session = self.get_raw_session() + gpts_plans = session.query(GptsPlansEntity) + if not conv_id: + return [] + gpts_plans = gpts_plans.filter(GptsPlansEntity.conv_id == conv_id).filter( + GptsPlansEntity.state.in_([Status.TODO.value, Status.RETRYING.value]) + ) + result = gpts_plans.order_by(GptsPlansEntity.sub_task_num).all() + session.close() + return result + + def complete_task(self, conv_id: str, task_num: int, result: str): + session = self.get_raw_session() + gpts_plans = session.query(GptsPlansEntity) + gpts_plans = gpts_plans.filter(GptsPlansEntity.conv_id == conv_id).filter( + GptsPlansEntity.sub_task_num == task_num + ) + gpts_plans.update( + { + GptsPlansEntity.state: Status.COMPLETE.value, + GptsPlansEntity.result: result, + }, + synchronize_session="fetch", + ) + session.commit() + session.close() + + def update_task( + self, + conv_id: str, + task_num: int, + state: str, + retry_times: int, + agent: str = None, + model: str = None, + result: str = None, + ): + session = self.get_raw_session() + gpts_plans = session.query(GptsPlansEntity) + gpts_plans = gpts_plans.filter(GptsPlansEntity.conv_id == conv_id).filter( + GptsPlansEntity.sub_task_num == task_num + ) + update_param = {} + update_param[GptsPlansEntity.state] = state + update_param[GptsPlansEntity.retry_times] = retry_times + update_param[GptsPlansEntity.result] = result + if agent: + update_param[GptsPlansEntity.sub_task_agent] = agent + if model: + update_param[GptsPlansEntity.agent_model] = model + + gpts_plans.update(update_param, synchronize_session="fetch") + session.commit() + session.close() + + def remove_by_conv_id(self, conv_id: str): + session = self.get_raw_session() + if conv_id is None: + raise Exception("conv_id is None") + + gpts_plans = session.query(GptsPlansEntity) + gpts_plans.filter(GptsPlansEntity.conv_id == conv_id).delete() + session.commit() + session.close() diff --git a/dbgpt/agent/db/my_plugin_db.py b/dbgpt/serve/agent/db/my_plugin_db.py similarity index 100% rename from dbgpt/agent/db/my_plugin_db.py rename to dbgpt/serve/agent/db/my_plugin_db.py diff --git a/dbgpt/agent/db/plugin_hub_db.py b/dbgpt/serve/agent/db/plugin_hub_db.py similarity index 100% rename from dbgpt/agent/db/plugin_hub_db.py rename to dbgpt/serve/agent/db/plugin_hub_db.py diff --git a/dbgpt/serve/agent/dbgpts/__init__.py b/dbgpt/serve/agent/dbgpts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbgpt/serve/agent/hub/__init__.py b/dbgpt/serve/agent/hub/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbgpt/agent/hub/agent_hub.py b/dbgpt/serve/agent/hub/agent_hub.py similarity index 98% rename from dbgpt/agent/hub/agent_hub.py rename to dbgpt/serve/agent/hub/agent_hub.py index 0359d81fd..a075d6625 100644 --- a/dbgpt/agent/hub/agent_hub.py +++ b/dbgpt/serve/agent/hub/agent_hub.py @@ -9,8 +9,9 @@ from ..db.plugin_hub_db import PluginHubEntity, PluginHubDao from ..db.my_plugin_db import MyPluginDao, MyPluginEntity -from ..common.schema import PluginStorageType -from ..plugins_util import scan_plugins, update_from_git + +from dbgpt.agent.common.schema import PluginStorageType +from dbgpt.agent.plugin.plugins_util import scan_plugins, update_from_git logger = logging.getLogger(__name__) Default_User = "default" diff --git a/dbgpt/agent/controller.py b/dbgpt/serve/agent/hub/controller.py similarity index 94% rename from dbgpt/agent/controller.py rename to dbgpt/serve/agent/hub/controller.py index dec2a8d32..ed5d2ed05 100644 --- a/dbgpt/agent/controller.py +++ b/dbgpt/serve/agent/hub/controller.py @@ -12,16 +12,17 @@ Result, ) -from .model import ( +from dbgpt.serve.agent.model import ( PluginHubParam, PagenationFilter, PagenationResult, PluginHubFilter, ) -from .hub.agent_hub import AgentHub -from .db.plugin_hub_db import PluginHubEntity -from .plugins_util import scan_plugins -from .commands.generator import PluginPromptGenerator +from dbgpt.serve.agent.hub.agent_hub import AgentHub +from dbgpt.serve.agent.db.plugin_hub_db import PluginHubEntity + +from dbgpt.agent.plugin.plugins_util import scan_plugins +from dbgpt.agent.plugin.generator import PluginPromptGenerator from dbgpt.configs.model_config import PLUGINS_DIR from dbgpt.component import BaseComponent, ComponentType, SystemApp diff --git a/dbgpt/agent/model.py b/dbgpt/serve/agent/model.py similarity index 100% rename from dbgpt/agent/model.py rename to dbgpt/serve/agent/model.py diff --git a/dbgpt/storage/metadata/meta_data.py b/dbgpt/storage/metadata/meta_data.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbgpt/util/code_utils.py b/dbgpt/util/code_utils.py new file mode 100644 index 000000000..d05dfc2ae --- /dev/null +++ b/dbgpt/util/code_utils.py @@ -0,0 +1,468 @@ +import logging +import os +import pathlib +import re +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor, TimeoutError +from hashlib import md5 +from typing import Callable, Dict, List, Optional, Tuple, Union + +# Regular expression for finding a code block +# ```[ \t]*(\w+)?[ \t]*\r?\n(.*?)[ \t]*\r?\n``` Matches multi-line code blocks. +# The [ \t]* matches the potential spaces before language name. +# The (\w+)? matches the language, where the ? indicates it is optional. +# The [ \t]* matches the potential spaces (not newlines) after language name. +# The \r?\n makes sure there is a linebreak after ```. +# The (.*?) matches the code itself (non-greedy). +# The \r?\n makes sure there is a linebreak before ```. +# The [ \t]* matches the potential spaces before closing ``` (the spec allows indentation). +CODE_BLOCK_PATTERN = r"```[ \t]*(\w+)?[ \t]*\r?\n(.*?)\r?\n[ \t]*```" +WORKING_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "extensions") +UNKNOWN = "unknown" +TIMEOUT_MSG = "Timeout" +DEFAULT_TIMEOUT = 60 +WIN32 = sys.platform == "win32" +PATH_SEPARATOR = WIN32 and "\\" or "/" + +logger = logging.getLogger(__name__) + + +def content_str(content: Union[str, List]) -> str: + if type(content) is str: + return content + rst = "" + for item in content: + if item["type"] == "text": + rst += item["text"] + else: + assert ( + isinstance(item, dict) and item["type"] == "image_url" + ), "Wrong content format." + rst += "" + return rst + + +def infer_lang(code): + """infer the language for the code. + TODO: make it robust. + """ + if ( + code.startswith("python ") + or code.startswith("pip") + or code.startswith("python3 ") + ): + return "sh" + + # check if code is a valid python code + try: + compile(code, "test", "exec") + return "python" + except SyntaxError: + # not a valid python code + return UNKNOWN + + +# TODO: In the future move, to better support https://spec.commonmark.org/0.30/#fenced-code-blocks +# perhaps by using a full Markdown parser. +def extract_code( + text: Union[str, List], + pattern: str = CODE_BLOCK_PATTERN, + detect_single_line_code: bool = False, +) -> List[Tuple[str, str]]: + """Extract code from a text. + + Args: + text (str or List): The content to extract code from. The content can be + a string or a list, as returned by standard GPT or multimodal GPT. + pattern (str, optional): The regular expression pattern for finding the + code block. Defaults to CODE_BLOCK_PATTERN. + detect_single_line_code (bool, optional): Enable the new feature for + extracting single line code. Defaults to False. + + Returns: + list: A list of tuples, each containing the language and the code. + If there is no code block in the input text, the language would be "unknown". + If there is code block but the language is not specified, the language would be "". + """ + text = content_str(text) + if not detect_single_line_code: + match = re.findall(pattern, text, flags=re.DOTALL) + return match if match else [(UNKNOWN, text)] + + # Extract both multi-line and single-line code block, separated by the | operator + # `([^`]+)`: Matches inline code. + code_pattern = re.compile(CODE_BLOCK_PATTERN + r"|`([^`]+)`") + code_blocks = code_pattern.findall(text) + + # Extract the individual code blocks and languages from the matched groups + extracted = [] + for lang, group1, group2 in code_blocks: + if group1: + extracted.append((lang.strip(), group1.strip())) + elif group2: + extracted.append(("", group2.strip())) + + return extracted + + +if __name__ == "__main__": + print( + extract_code( + """```python import requests from bs4 import BeautifulSoup from datetime import datetime, timedelta # Define the search query query = "LLM application" # Define the time range (last week) end_date = datetime.now().strftime("%Y-%m-%d") start_date = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d") # Create the search URL url = f"https://arxiv.org/search/advanced?advanced=&terms-0-operator=AND&terms-0-term={query}&terms-0-field=title&classification-physics_archives=all&classification-include_cross_list=include&date-filter_by=specific_date&date-year=&date-from_date={start_date}&date-to_date={end_date}&date-date_type=submitted_date&abstracts=show&size=200&order=-announced_date_first" # Send a GET request to the search URL response = requests.get(url) # Parse the HTML content soup = BeautifulSoup(response.content, "html.parser") # Find all the paper titles and authors titles = soup.find_all("p", class_="title is-5 mathjax") authors = soup.find_all("p", class_="authors") # Print the results for i in range(len(titles)): print(f"Title: {titles[i].text.strip()}") print(f"Authors: {authors[i].text.strip()}") print("-------------------------") ``` This code uses the `requests` library to send a GET request to the advanced search page of arXiv. It searches for papers with the specified query ("LLM application") that were submitted in the last week. The code then uses `BeautifulSoup` to parse the HTML content of the search results page and extracts the paper titles and authors. Finally, it prints the titles and authors of the found papers.""" + ) + ) + + +_IMPROVE_FUNCTION_CONFIG = { + "prompt": """Improve the function '{func_name}' to achieve the objective '{objective}'. +The current implementation of the function is as follows: +{file_string}""", + "model": "DEFAULT_MODEL", + "request_timeout": 600, +} + + +_IMPROVE_CODE_CONFIG = { + "prompt": """Analyze the code in the following files and return a list of suggestions for improvement{followup}, to achieve the objective of '{objective}'. +{code} +""", + "model": "DEFAULT_MODEL", + "request_timeout": 900, +} + + +def timeout_handler(signum, frame): + raise TimeoutError("Timed out!") + + +def _cmd(lang): + if lang.startswith("python") or lang in ["bash", "sh", "powershell"]: + return lang + if lang in ["shell"]: + return "sh" + if lang in ["ps1"]: + return "powershell" + raise NotImplementedError(f"{lang} not recognized in code execution") + + +def execute_code( + code: Optional[str] = None, + timeout: Optional[int] = None, + filename: Optional[str] = None, + work_dir: Optional[str] = None, + use_docker: Optional[Union[List[str], str, bool]] = None, + lang: Optional[str] = "python", +) -> Tuple[int, str, str]: + """Execute code in a docker container. + This function is not tested on MacOS. + + Args: + code (Optional, str): The code to execute. + If None, the code from the file specified by filename will be executed. + Either code or filename must be provided. + timeout (Optional, int): The maximum execution time in seconds. + If None, a default timeout will be used. The default timeout is 600 seconds. On Windows, the timeout is not enforced when use_docker=False. + filename (Optional, str): The file name to save the code or where the code is stored when `code` is None. + If None, a file with a randomly generated name will be created. + The randomly generated file will be deleted after execution. + The file name must be a relative path. Relative paths are relative to the working directory. + work_dir (Optional, str): The working directory for the code execution. + If None, a default working directory will be used. + The default working directory is the "extensions" directory under + "path_to_autogen". + use_docker (Optional, list, str or bool): The docker image to use for code execution. + If a list or a str of image name(s) is provided, the code will be executed in a docker container + with the first image successfully pulled. + If None, False or empty, the code will be executed in the current environment. + Default is None, which will be converted into an empty list when docker package is available. + Expected behaviour: + - If `use_docker` is explicitly set to True and the docker package is available, the code will run in a Docker container. + - If `use_docker` is explicitly set to True but the Docker package is missing, an error will be raised. + - If `use_docker` is not set (i.e., left default to None) and the Docker package is not available, a warning will be displayed, but the code will run natively. + If the code is executed in the current environment, + the code must be trusted. + lang (Optional, str): The language of the code. Default is "python". + + Returns: + int: 0 if the code executes successfully. + str: The error message if the code fails to execute; the stdout otherwise. + image: The docker image name after container run when docker is used. + """ + if all((code is None, filename is None)): + error_msg = f"Either {code=} or {filename=} must be provided." + logger.error(error_msg) + raise AssertionError(error_msg) + + # Warn if use_docker was unspecified (or None), and cannot be provided (the default). + # In this case the current behavior is to fall back to run natively, but this behavior + # is subject to change. + + try: + import docker + + try: + docker.version + except AttributeError: + docker = None + except ImportError: + docker = None + + if use_docker is None: + if docker is None: + use_docker = False + logger.warning( + "execute_code was called without specifying a value for use_docker. Since the python docker package is not available, code will be run natively. Note: this fallback behavior is subject to change" + ) + else: + # Default to true + use_docker = True + + timeout = timeout or DEFAULT_TIMEOUT + original_filename = filename + if WIN32 and lang in ["sh", "shell"] and (not use_docker): + lang = "ps1" + if filename is None: + code_hash = md5(code.encode()).hexdigest() + # create a file with a automatically generated name + filename = f"tmp_code_{code_hash}.{'py' if lang.startswith('python') else lang}" + if work_dir is None: + work_dir = WORKING_DIR + filepath = os.path.join(work_dir, filename) + file_dir = os.path.dirname(filepath) + os.makedirs(file_dir, exist_ok=True) + if code is not None: + with open(filepath, "w", encoding="utf-8") as fout: + fout.write(code) + # check if already running in a docker container + in_docker_container = os.path.exists("/.dockerenv") + if not use_docker or in_docker_container: + # already running in a docker container + cmd = [ + sys.executable if lang.startswith("python") else _cmd(lang), + f".\\{filename}" if WIN32 else filename, + ] + if WIN32: + logger.warning( + "SIGALRM is not supported on Windows. No timeout will be enforced." + ) + result = subprocess.run( + cmd, + cwd=work_dir, + capture_output=True, + text=True, + ) + else: + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit( + subprocess.run, + cmd, + cwd=work_dir, + capture_output=True, + text=True, + ) + try: + result = future.result(timeout=timeout) + except TimeoutError: + if original_filename is None: + os.remove(filepath) + return 1, TIMEOUT_MSG, None + if original_filename is None: + os.remove(filepath) + if result.returncode: + logs = result.stderr + if original_filename is None: + abs_path = str(pathlib.Path(filepath).absolute()) + logs = logs.replace(str(abs_path), "").replace(filename, "") + else: + abs_path = str(pathlib.Path(work_dir).absolute()) + PATH_SEPARATOR + logs = logs.replace(str(abs_path), "") + else: + logs = result.stdout + return result.returncode, logs, None + + # create a docker client + client = docker.from_env() + image_list = ( + ["python:3-alpine", "python:3", "python:3-windowsservercore"] + if use_docker is True + else [use_docker] + if isinstance(use_docker, str) + else use_docker + ) + for image in image_list: + # check if the image exists + try: + client.images.get(image) + break + except docker.errors.ImageNotFound: + # pull the image + print("Pulling image", image) + try: + client.images.pull(image) + break + except docker.errors.DockerException: + print("Failed to pull image", image) + # get a randomized str based on current time to wrap the exit code + exit_code_str = f"exitcode{time.time()}" + abs_path = pathlib.Path(work_dir).absolute() + cmd = [ + "sh", + "-c", + f"{_cmd(lang)} {filename}; exit_code=$?; echo -n {exit_code_str}; echo -n $exit_code; echo {exit_code_str}", + ] + # create a docker container + container = client.containers.run( + image, + command=cmd, + working_dir="/workspace", + detach=True, + # get absolute path to the working directory + volumes={abs_path: {"bind": "/workspace", "mode": "rw"}}, + ) + start_time = time.time() + while container.status != "exited" and time.time() - start_time < timeout: + # Reload the container object + container.reload() + if container.status != "exited": + container.stop() + container.remove() + if original_filename is None: + os.remove(filepath) + return 1, TIMEOUT_MSG, image + # get the container logs + logs = container.logs().decode("utf-8").rstrip() + # commit the image + tag = filename.replace("/", "") + container.commit(repository="python", tag=tag) + # remove the container + container.remove() + # check if the code executed successfully + exit_code = container.attrs["State"]["ExitCode"] + if exit_code == 0: + # extract the exit code from the logs + pattern = re.compile(f"{exit_code_str}(\\d+){exit_code_str}") + match = pattern.search(logs) + exit_code = 1 if match is None else int(match.group(1)) + # remove the exit code from the logs + logs = logs if match is None else pattern.sub("", logs) + + if original_filename is None: + os.remove(filepath) + if exit_code: + logs = logs.replace( + f"/workspace/{filename if original_filename is None else ''}", "" + ) + # return the exit code, logs and image + return exit_code, logs, f"python:{tag}" + + +_GENERATE_ASSERTIONS_CONFIG = { + "prompt": """Given the signature and docstring, write the exactly same number of assertion(s) for the provided example(s) in the docstring, without assertion messages. + +func signature: +{definition} +assertions:""", + "model": "FAST_MODEL", + "max_tokens": 256, + "stop": "\n\n", +} + + +def _remove_check(response): + """Remove the check function from the response.""" + # find the position of the check function + pos = response.find("def check(") + if pos == -1: + return response + return response[:pos] + + +def eval_function_completions( + responses: List[str], + definition: str, + test: Optional[str] = None, + entry_point: Optional[str] = None, + assertions: Optional[Union[str, Callable[[str], Tuple[str, float]]]] = None, + timeout: Optional[float] = 3, + use_docker: Optional[bool] = True, +) -> Dict: + """(openai<1) Select a response from a list of responses for the function completion task (using generated assertions), and/or evaluate if the task is successful using a gold test. + + Args: + responses (list): The list of responses. + definition (str): The input definition. + test (Optional, str): The test code. + entry_point (Optional, str): The name of the function. + assertions (Optional, str or Callable): The assertion code which serves as a filter of the responses, or an assertion generator. + When provided, only the responses that pass the assertions will be considered for the actual test (if provided). + timeout (Optional, float): The timeout for executing the code. + + Returns: + dict: The success metrics. + """ + n = len(responses) + if assertions is None: + # no assertion filter + success_list = [] + for i in range(n): + response = _remove_check(responses[i]) + code = ( + f"{response}\n{test}\ncheck({entry_point})" + if response.startswith("def") + else f"{definition}{response}\n{test}\ncheck({entry_point})" + ) + success = execute_code(code, timeout=timeout, use_docker=use_docker)[0] == 0 + success_list.append(success) + return { + "expected_success": 1 - pow(1 - sum(success_list) / n, n), + "success": any(s for s in success_list), + } + if callable(assertions) and n > 1: + # assertion generator + assertions, gen_cost = assertions(definition) + else: + assertions, gen_cost = None, 0 + if n > 1 or test is None: + for i in range(n): + response = responses[i] = _remove_check(responses[i]) + code = ( + f"{response}\n{assertions}" + if response.startswith("def") + else f"{definition}{response}\n{assertions}" + ) + succeed_assertions = ( + execute_code(code, timeout=timeout, use_docker=use_docker)[0] == 0 + ) + if succeed_assertions: + break + else: + # just test, no need to check assertions + succeed_assertions = False + i, response = 0, responses[0] + if test is None: + # no test code + return { + "index_selected": i, + "succeed_assertions": succeed_assertions, + "gen_cost": gen_cost, + "assertions": assertions, + } + code_test = ( + f"{response}\n{test}\ncheck({entry_point})" + if response.startswith("def") + else f"{definition}{response}\n{test}\ncheck({entry_point})" + ) + success = execute_code(code_test, timeout=timeout, use_docker=use_docker)[0] == 0 + return { + "index_selected": i, + "succeed_assertions": succeed_assertions, + "success": success, + "gen_cost": gen_cost, + "assertions": assertions, + } + + +_FUNC_COMPLETION_PROMPT = "# Python 3{definition}" +_FUNC_COMPLETION_STOP = ["\nclass", "\ndef", "\nif", "\nprint"] diff --git a/dbgpt/util/error_types.py b/dbgpt/util/error_types.py new file mode 100644 index 000000000..5bd8f5823 --- /dev/null +++ b/dbgpt/util/error_types.py @@ -0,0 +1,16 @@ +class LLMChatError(Exception): + """ + llm conversation result generates exception + """ + + def __init__(self, message="LLM Chat Generrate Error!", original_exception=None): + super().__init__(message) + self.message = message + self.original_exception = original_exception + + def __str__(self): + if self.original_exception: + # 返回自定义异常信息和原始异常信息 + return f"{self.message}({self.original_exception})" + else: + return self.message diff --git a/dbgpt/util/json_utils.py b/dbgpt/util/json_utils.py index c4b348315..e861e7284 100644 --- a/dbgpt/util/json_utils.py +++ b/dbgpt/util/json_utils.py @@ -1,5 +1,16 @@ +"""Utilities for the json_fixes package.""" import json from datetime import date, datetime +from dataclasses import dataclass, asdict, is_dataclass +import os.path +import re +import logging + +from jsonschema import Draft7Validator + +logger = logging.getLogger(__name__) + +LLM_DEFAULT_RESPONSE_FORMAT = "llm_response_format_1" def serialize(obj): @@ -7,8 +18,105 @@ def serialize(obj): return obj.isoformat() -class DateTimeEncoder(json.JSONEncoder): +class EnhancedJSONEncoder(json.JSONEncoder): def default(self, obj): + if is_dataclass(obj): + return asdict(obj) if isinstance(obj, datetime): return obj.isoformat() return super().default(obj) + + +def extract_char_position(error_message: str) -> int: + """Extract the character position from the JSONDecodeError message. + + Args: + error_message (str): The error message from the JSONDecodeError + exception. + + Returns: + int: The character position. + """ + + char_pattern = re.compile(r"\(char (\d+)\)") + if match := char_pattern.search(error_message): + return int(match[1]) + else: + raise ValueError("Character position not found in the error message.") + + +def find_json_objects(text): + json_objects = [] + inside_string = False + escape_character = False + stack = [] + start_index = -1 + + for i, char in enumerate(text): + # Handle escape characters + if char == "\\" and not escape_character: + escape_character = True + continue + + # Toggle inside_string flag + if char == '"' and not escape_character: + inside_string = not inside_string + + if not inside_string and char == "\n": + continue + if inside_string and char == "\n": + char = "\\n" + if inside_string and char == "\t": + char = "\\t" + + # Handle opening brackets + if char in "{[" and not inside_string: + stack.append(char) + if len(stack) == 1: + start_index = i + # Handle closing brackets + if char in "}]" and not inside_string and stack: + if (char == "}" and stack[-1] == "{") or (char == "]" and stack[-1] == "["): + stack.pop() + if not stack: + end_index = i + 1 + try: + json_obj = json.loads(text[start_index:end_index]) + json_objects.append(json_obj) + except json.JSONDecodeError: + pass + # Reset escape_character flag + escape_character = False if escape_character else escape_character + + return json_objects + + +@staticmethod +def _format_json_str(jstr): + """Remove newlines outside of quotes, and handle JSON escape sequences. + + 1. this function removes the newline in the query outside of quotes otherwise json.loads(s) will fail. + Ex 1: + "{\n"tool": "python",\n"query": "print('hello')\nprint('world')"\n}" -> "{"tool": "python","query": "print('hello')\nprint('world')"}" + Ex 2: + "{\n \"location\": \"Boston, MA\"\n}" -> "{"location": "Boston, MA"}" + + 2. this function also handles JSON escape sequences inside quotes, + Ex 1: + '{"args": "a\na\na\ta"}' -> '{"args": "a\\na\\na\\ta"}' + """ + result = [] + inside_quotes = False + last_char = " " + for char in jstr: + if last_char != "\\" and char == '"': + inside_quotes = not inside_quotes + last_char = char + if not inside_quotes and char == "\n": + continue + if inside_quotes and char == "\n": + char = "\\n" + if inside_quotes and char == "\t": + char = "\\t" + result.append(char) + return "".join(result) diff --git a/dbgpt/util/string_utils.py b/dbgpt/util/string_utils.py index d62b10bea..1303e6e23 100644 --- a/dbgpt/util/string_utils.py +++ b/dbgpt/util/string_utils.py @@ -73,6 +73,15 @@ def extract_content_open_ending(long_string, s1, s2, is_include: bool = False): return match_map +def str_to_bool(s): + if s.lower() in ("true", "t", "1", "yes", "y"): + return True + elif s.lower() in ("false", "f", "0", "no", "n"): + return False + else: + return False + + def _to_str(x, charset="utf8", errors="strict"): if x is None or isinstance(x, str): return x diff --git a/examples/agents/__init__.py b/examples/agents/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/examples/agents/auto_plan_agent_dialogue_example.py b/examples/agents/auto_plan_agent_dialogue_example.py new file mode 100644 index 000000000..1f968db45 --- /dev/null +++ b/examples/agents/auto_plan_agent_dialogue_example.py @@ -0,0 +1,73 @@ +"""Agents: auto plan agents example? + + Examples: + + Execute the following command in the terminal: + Set env params. + .. code-block:: shell + + export OPENAI_API_KEY=sk-xx + export OPENAI_API_BASE=https://xx:80/v1 + + run example. + ..code-block:: shell + python examples/agents/auto_plan_agent_dialogue_example.py +""" + + +from dbgpt.agent.agents.planner_agent import PlannerAgent +from dbgpt.agent.agents.user_proxy_agent import UserProxyAgent +from dbgpt.agent.agents.plan_group_chat import PlanChat, PlanChatManager + +from dbgpt.agent.agents.expand.code_assistant_agent import CodeAssistantAgent +from dbgpt.agent.agents.expand.plugin_assistant_agent import PluginAgent +from dbgpt.agent.agents.agents_mange import agent_mange + +from dbgpt.agent.agents.agent import AgentContext +from dbgpt.agent.memory.gpts_memory import GptsMemory +from dbgpt.core.interface.llm import ModelMetadata + +import asyncio +import os + +if __name__ == "__main__": + from dbgpt.model import OpenAILLMClient + + llm_client = OpenAILLMClient() + context: AgentContext = AgentContext(conv_id="test456", llm_provider=llm_client) + # context.llm_models = [ModelMetadata(model="gpt-3.5-turbo")] + context.llm_models = [ModelMetadata(model="gpt-4-vision-preview")] + context.gpts_name = "代码分析助手" + + default_memory = GptsMemory() + coder = CodeAssistantAgent(memory=default_memory, agent_context=context) + ## TODO add other agent + + groupchat = PlanChat(agents=[coder], messages=[], max_round=50) + planner = PlannerAgent( + agent_context=context, + memory=default_memory, + plan_chat=groupchat, + ) + + manager = PlanChatManager( + plan_chat=groupchat, + planner=planner, + agent_context=context, + memory=default_memory, + ) + + user_proxy = UserProxyAgent(memory=default_memory, agent_context=context) + + asyncio.run( + user_proxy.a_initiate_chat( + recipient=manager, + reviewer=user_proxy, + message="Obtain simple information about issues in the repository 'eosphoros-ai/DB-GPT' in the past three days and analyze the data. Create a Markdown table grouped by day and status.", + # message="Find papers on gpt-4 in the past three weeks on arxiv, and organize their titles, authors, and links into a markdown table", + # message="find papers on LLM applications from arxiv in the last month, create a markdown table of different domains.", + ) + ) + + ## dbgpt-vis message infos + print(asyncio.run(default_memory.one_plan_chat_competions("test456"))) diff --git a/examples/agents/single_agent_dialogue_example.py b/examples/agents/single_agent_dialogue_example.py new file mode 100644 index 000000000..5fb32da1e --- /dev/null +++ b/examples/agents/single_agent_dialogue_example.py @@ -0,0 +1,48 @@ +"""Agents: single agents about CodeAssistantAgent? + + Examples: + + Execute the following command in the terminal: + Set env params. + .. code-block:: shell + + export OPENAI_API_KEY=sk-xx + export OPENAI_API_BASE=https://xx:80/v1 + + run example. + ..code-block:: shell + python examples/agents/single_agent_dialogue_example.py +""" + +from dbgpt.agent.agents.expand.code_assistant_agent import CodeAssistantAgent +from dbgpt.agent.agents.user_proxy_agent import UserProxyAgent +from dbgpt.agent.memory.gpts_memory import GptsMemory +from dbgpt.agent.agents.agent import AgentContext +from dbgpt.core.interface.llm import ModelMetadata +import asyncio +import os + + +if __name__ == "__main__": + from dbgpt.model import OpenAILLMClient + + llm_client = OpenAILLMClient() + context: AgentContext = AgentContext(conv_id="test456", llm_provider=llm_client) + context.llm_models = [ModelMetadata(model="gpt-3.5-turbo")] + + default_memory = GptsMemory() + coder = CodeAssistantAgent(memory=default_memory, agent_context=context) + + user_proxy = UserProxyAgent(memory=default_memory, agent_context=context) + + asyncio.run( + user_proxy.a_initiate_chat( + recipient=coder, + reviewer=user_proxy, + message="式计算下321 * 123等于多少", # 用python代码的方式计算下321 * 123等于多少 + # message="download data from https://raw.githubusercontent.com/uwdata/draco/master/data/cars.csv and plot a visualization that tells us about the relationship between weight and horsepower. Save the plot to a file. Print the fields in a dataset before visualizing it.", + ) + ) + + ## dbgpt-vis message infos + print(asyncio.run(default_memory.one_plan_chat_competions("test456"))) diff --git a/examples/notebook/agent_auto_plan_dialogue_example.ipynb b/examples/notebook/agent_auto_plan_dialogue_example.ipynb new file mode 100644 index 000000000..db1c4d16d --- /dev/null +++ b/examples/notebook/agent_auto_plan_dialogue_example.ipynb @@ -0,0 +1,681 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "from dbgpt.agent.agents.planner_agent import PlannerAgent\n", + "from dbgpt.agent.agents.user_proxy_agent import UserProxyAgent\n", + "from dbgpt.agent.agents.plan_group_chat import PlanChat, PlanChatManager\n", + "\n", + "from dbgpt.agent.agents.expand.code_assistant_agent import CodeAssistantAgent\n", + "from dbgpt.agent.agents.expand.plugin_assistant_agent import PluginAgent\n", + "from dbgpt.agent.agents.agents_mange import agent_mange\n", + "\n", + "from dbgpt.agent.agents.agent import AgentContext\n", + "from dbgpt.agent.memory.gpts_memory import GptsMemory\n", + "from dbgpt.core.interface.llm import ModelMetadata\n", + "\n", + "import asyncio\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from dbgpt.model import OpenAILLMClient" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "os.environ[\"OPENAI_API_KEY\"] = \"sk-ElhG3036tcvECTOYO9nHqAkUIWtqT55JXs1cTjODz1bnUQDz\"\n", + "os.environ[\"OPENAI_API_BASE\"] = \"https://api.chatanywhere.tech/v1\"" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mUser\u001b[0m (to plan_manager)-[]:\n", + "\n", + "\"Obtain simple information about issues in the repository 'eosphoros-ai/DB-GPT' in the past three days and analyze the data. Create a Markdown table grouped by day and status.\"\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mplan_manager\u001b[0m (to Planner)-[]:\n", + "\n", + "\"Obtain simple information about issues in the repository 'eosphoros-ai/DB-GPT' in the past three days and analyze the data. Create a Markdown table grouped by day and status.\"\n", + "\n", + "--------------------------------------------------------------------------------\n", + "un_stream ai response: [{\n", + " \"serial_number\": \"1\",\n", + " \"agent\": \"CodeEngineer\",\n", + " \"content\": \"Write a Python script to use the GitHub API to retrieve issues from the 'eosphoros-ai/DB-GPT' repository that were created in the past three days. The script should extract the issue's creation date and status.\",\n", + " \"rely\": \"\"\n", + "},\n", + "{\n", + " \"serial_number\": \"2\",\n", + " \"agent\": \"CodeEngineer\",\n", + " \"content\": \"Write a Python script to process the retrieved issues data to group them by creation date and status. Then, format the grouped data into a Markdown table.\",\n", + " \"rely\": \"1\"\n", + "}]\n", + "\u001b[33mPlanner\u001b[0m (to plan_manager)-[gpt-4-vision-preview]:\n", + "\n", + "\"[{\\n \\\"serial_number\\\": \\\"1\\\",\\n \\\"agent\\\": \\\"CodeEngineer\\\",\\n \\\"content\\\": \\\"Write a Python script to use the GitHub API to retrieve issues from the 'eosphoros-ai/DB-GPT' repository that were created in the past three days. The script should extract the issue's creation date and status.\\\",\\n \\\"rely\\\": \\\"\\\"\\n},\\n{\\n \\\"serial_number\\\": \\\"2\\\",\\n \\\"agent\\\": \\\"CodeEngineer\\\",\\n \\\"content\\\": \\\"Write a Python script to process the retrieved issues data to group them by creation date and status. Then, format the grouped data into a Markdown table.\\\",\\n \\\"rely\\\": \\\"1\\\"\\n}]\"\n", + "\u001b[32m>>>>>>>>Planner Review info: \n", + " Pass.None\u001b[0m\n", + "\u001b[34m>>>>>>>>Planner Action report: \n", + "execution succeeded,\n", + "1,Write a Python script to use the GitHub API to retrieve issues from the 'eosphoros-ai/DB-GPT' repository that were created in the past three days. The script should extract the issue's creation date and status.\n", + "2,Write a Python script to process the retrieved issues data to group them by creation date and status. Then, format the grouped data into a Markdown table.\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GroupChat is underpopulated with 1 agents. It is recommended to set speaker_selection_method to 'round_robin' or allow_repeat_speaker to False.Or, use direct communication instead.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mplan_manager\u001b[0m (to CodeEngineer)-[]:\n", + "\n", + "\"Write a Python script to use the GitHub API to retrieve issues from the 'eosphoros-ai/DB-GPT' repository that were created in the past three days. The script should extract the issue's creation date and status.\"\n", + "\n", + "--------------------------------------------------------------------------------\n", + "un_stream ai response: ```python\n", + "# filename: github_issues.py\n", + "import requests\n", + "from datetime import datetime, timedelta\n", + "\n", + "# Function to get issues created in the past three days from a GitHub repository\n", + "def get_recent_issues(owner, repo):\n", + " # Calculate the date for three days ago\n", + " three_days_ago = datetime.utcnow() - timedelta(days=3)\n", + " # Format the date in ISO 8601 format\n", + " since = three_days_ago.isoformat()\n", + "\n", + " # GitHub API URL for issues of the given repository\n", + " issues_url = f\"https://api.github.com/repos/{owner}/{repo}/issues\"\n", + " \n", + " # Parameters to get issues created in the past three days\n", + " params = {\n", + " 'since': since,\n", + " 'state': 'all', # Get both open and closed issues\n", + " }\n", + " \n", + " # Send a request to the GitHub API\n", + " response = requests.get(issues_url, params=params)\n", + " # Raise an exception if the request was unsuccessful\n", + " response.raise_for_status()\n", + " \n", + " # Parse the JSON response\n", + " issues = response.json()\n", + " \n", + " # Extract and print the creation date and status of each issue\n", + " for issue in issues:\n", + " created_at = issue['created_at']\n", + " state = issue['state']\n", + " print(f\"Issue created at: {created_at}, Status: {state}\")\n", + "\n", + "# Replace 'owner' and 'repo' with the actual owner and repository name\n", + "get_recent_issues('eosphoros-ai', 'DB-GPT')\n", + "```\n", + "\n", + "When you run this script, it will print out the creation date and status of issues from the 'eosphoros-ai/DB-GPT' repository that were created in the past three days. Make sure to have internet access while running the script as it needs to make requests to the GitHub API.\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", + "\u001b[33mCodeEngineer\u001b[0m (to plan_manager)-[gpt-4-vision-preview]:\n", + "\n", + "\"```python\\n# filename: github_issues.py\\nimport requests\\nfrom datetime import datetime, timedelta\\n\\n# Function to get issues created in the past three days from a GitHub repository\\ndef get_recent_issues(owner, repo):\\n # Calculate the date for three days ago\\n three_days_ago = datetime.utcnow() - timedelta(days=3)\\n # Format the date in ISO 8601 format\\n since = three_days_ago.isoformat()\\n\\n # GitHub API URL for issues of the given repository\\n issues_url = f\\\"https://api.github.com/repos/{owner}/{repo}/issues\\\"\\n \\n # Parameters to get issues created in the past three days\\n params = {\\n 'since': since,\\n 'state': 'all', # Get both open and closed issues\\n }\\n \\n # Send a request to the GitHub API\\n response = requests.get(issues_url, params=params)\\n # Raise an exception if the request was unsuccessful\\n response.raise_for_status()\\n \\n # Parse the JSON response\\n issues = response.json()\\n \\n # Extract and print the creation date and status of each issue\\n for issue in issues:\\n created_at = issue['created_at']\\n state = issue['state']\\n print(f\\\"Issue created at: {created_at}, Status: {state}\\\")\\n\\n# Replace 'owner' and 'repo' with the actual owner and repository name\\nget_recent_issues('eosphoros-ai', 'DB-GPT')\\n```\\n\\nWhen you run this script, it will print out the creation date and status of issues from the 'eosphoros-ai/DB-GPT' repository that were created in the past three days. Make sure to have internet access while running the script as it needs to make requests to the GitHub API.\"\n", + "\u001b[32m>>>>>>>>CodeEngineer Review info: \n", + " Pass.None\u001b[0m\n", + "\u001b[34m>>>>>>>>CodeEngineer Action report: \n", + "execution failed,\n", + "exitcode: 1 (execution failed)\n", + " \n", + "Traceback (most recent call last):\n", + " File \"github_issues.py\", line 2, in \n", + " import requests\n", + "ModuleNotFoundError: No module named 'requests'\n", + "\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GroupChat is underpopulated with 1 agents. It is recommended to set speaker_selection_method to 'round_robin' or allow_repeat_speaker to False.Or, use direct communication instead.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mplan_manager\u001b[0m (to CodeEngineer)-[]:\n", + "\n", + "\"exitcode: 1 (execution failed)\\n \\nTraceback (most recent call last):\\n File \\\"github_issues.py\\\", line 2, in \\n import requests\\nModuleNotFoundError: No module named 'requests'\\n\"\n", + "\n", + "--------------------------------------------------------------------------------\n", + "un_stream ai response: It appears that the `requests` module is not installed on your system. You can install it using `pip`, the Python package manager. Here's the command to install the `requests` module:\n", + "\n", + "```sh\n", + "pip install requests\n", + "```\n", + "\n", + "After installing the `requests` module, you can try running the script again. Here is the complete script including the installation of the `requests` module:\n", + "\n", + "```python\n", + "# filename: github_issues.py\n", + "import requests\n", + "from datetime import datetime, timedelta\n", + "\n", + "# Function to get issues created in the past three days from a GitHub repository\n", + "def get_recent_issues(owner, repo):\n", + " # Calculate the date for three days ago\n", + " three_days_ago = datetime.utcnow() - timedelta(days=3)\n", + " # Format the date in ISO 8601 format\n", + " since = three_days_ago.isoformat()\n", + "\n", + " # GitHub API URL for issues of the given repository\n", + " issues_url = f\"https://api.github.com/repos/{owner}/{repo}/issues\"\n", + " \n", + " # Parameters to get issues created in the past three days\n", + " params = {\n", + " 'since': since,\n", + " 'state': 'all', # Get both open and closed issues\n", + " }\n", + " \n", + " # Send a request to the GitHub API\n", + " response = requests.get(issues_url, params=params)\n", + " # Raise an exception if the request was unsuccessful\n", + " response.raise_for_status()\n", + " \n", + " # Parse the JSON response\n", + " issues = response.json()\n", + " \n", + " # Extract and print the creation date and status of each issue\n", + " for issue in issues:\n", + " created_at = issue['created_at']\n", + " state = issue['state']\n", + " print(f\"Issue created at: {created_at}, Status: {state}\")\n", + "\n", + "# Replace 'owner' and 'repo' with the actual owner and repository name\n", + "get_recent_issues('eosphoros-ai', 'DB-GPT')\n", + "```\n", + "\n", + "Please run the above Python script after you have installed the `requests` module.\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is sh)...\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 1 (inferred language is python)...\u001b[0m\n", + "un_stream ai response: False\n", + "\u001b[33mCodeEngineer\u001b[0m (to plan_manager)-[gpt-4-vision-preview]:\n", + "\n", + "\"It appears that the `requests` module is not installed on your system. You can install it using `pip`, the Python package manager. Here's the command to install the `requests` module:\\n\\n```sh\\npip install requests\\n```\\n\\nAfter installing the `requests` module, you can try running the script again. Here is the complete script including the installation of the `requests` module:\\n\\n```python\\n# filename: github_issues.py\\nimport requests\\nfrom datetime import datetime, timedelta\\n\\n# Function to get issues created in the past three days from a GitHub repository\\ndef get_recent_issues(owner, repo):\\n # Calculate the date for three days ago\\n three_days_ago = datetime.utcnow() - timedelta(days=3)\\n # Format the date in ISO 8601 format\\n since = three_days_ago.isoformat()\\n\\n # GitHub API URL for issues of the given repository\\n issues_url = f\\\"https://api.github.com/repos/{owner}/{repo}/issues\\\"\\n \\n # Parameters to get issues created in the past three days\\n params = {\\n 'since': since,\\n 'state': 'all', # Get both open and closed issues\\n }\\n \\n # Send a request to the GitHub API\\n response = requests.get(issues_url, params=params)\\n # Raise an exception if the request was unsuccessful\\n response.raise_for_status()\\n \\n # Parse the JSON response\\n issues = response.json()\\n \\n # Extract and print the creation date and status of each issue\\n for issue in issues:\\n created_at = issue['created_at']\\n state = issue['state']\\n print(f\\\"Issue created at: {created_at}, Status: {state}\\\")\\n\\n# Replace 'owner' and 'repo' with the actual owner and repository name\\nget_recent_issues('eosphoros-ai', 'DB-GPT')\\n```\\n\\nPlease run the above Python script after you have installed the `requests` module.\"\n", + "\u001b[32m>>>>>>>>CodeEngineer Review info: \n", + " Pass.None\u001b[0m\n", + "\u001b[34m>>>>>>>>CodeEngineer Action report: \n", + "execution succeeded,\n", + "\n", + "Collecting requests\n", + " Obtaining dependency information for requests from https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata\n", + " Downloading requests-2.31.0-py3-none-any.whl.metadata (4.6 kB)\n", + "Collecting charset-normalizer<4,>=2 (from requests)\n", + " Obtaining dependency information for charset-normalizer<4,>=2 from https://files.pythonhosted.org/packages/45/59/3d27019d3b447a88fe7e7d004a1e04be220227760264cc41b405e863891b/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl.metadata\n", + " Downloading charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl.metadata (33 kB)\n", + "Collecting idna<4,>=2.5 (from requests)\n", + " Obtaining dependency information for idna<4,>=2.5 from https://files.pythonhosted.org/packages/c2/e7/a82b05cf63a603df6e68d59ae6a68bf5064484a0718ea5033660af4b54a9/idna-3.6-py3-none-any.whl.metadata\n", + " Downloading idna-3.6-py3-none-any.whl.metadata (9.9 kB)\n", + "Collecting urllib3<3,>=1.21.1 (from requests)\n", + " Obtaining dependency information for urllib3<3,>=1.21.1 from https://files.pythonhosted.org/packages/96/94/c31f58c7a7f470d5665935262ebd7455c7e4c7782eb525658d3dbf4b9403/urllib3-2.1.0-py3-none-any.whl.metadata\n", + " Downloading urllib3-2.1.0-py3-none-any.whl.metadata (6.4 kB)\n", + "Collecting certifi>=2017.4.17 (from requests)\n", + " Obtaining dependency information for certifi>=2017.4.17 from https://files.pythonhosted.org/packages/64/62/428ef076be88fa93716b576e4a01f919d25968913e817077a386fcbe4f42/certifi-2023.11.17-py3-none-any.whl.metadata\n", + " Downloading certifi-2023.11.17-py3-none-any.whl.metadata (2.2 kB)\n", + "Downloading requests-2.31.0-py3-none-any.whl (62 kB)\n", + " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 62.6/62.6 kB 1.9 MB/s eta 0:00:00\n", + "Downloading certifi-2023.11.17-py3-none-any.whl (162 kB)\n", + " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 162.5/162.5 kB 1.9 MB/s eta 0:00:00\n", + "Downloading charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl (138 kB)\n", + " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 138.3/138.3 kB 4.6 MB/s eta 0:00:00\n", + "Downloading idna-3.6-py3-none-any.whl (61 kB)\n", + " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 61.6/61.6 kB 3.2 MB/s eta 0:00:00\n", + "Downloading urllib3-2.1.0-py3-none-any.whl (104 kB)\n", + " ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 104.6/104.6 kB 4.9 MB/s eta 0:00:00\n", + "Installing collected packages: urllib3, idna, charset-normalizer, certifi, requests\n", + "Successfully installed certifi-2023.11.17 charset-normalizer-3.3.2 idna-3.6 requests-2.31.0 urllib3-2.1.0\n", + "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n", + "\n", + "[notice] A new release of pip is available: 23.2.1 -> 23.3.2\n", + "[notice] To update, run: pip install --upgrade pip\n", + "\n", + "/workspace/github_issues.py:8: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).\n", + " three_days_ago = datetime.utcnow() - timedelta(days=3)\n", + "Issue created at: 2023-12-27T03:20:12Z, Status: open\n", + "Issue created at: 2023-12-26T16:13:42Z, Status: open\n", + "Issue created at: 2023-12-26T10:23:43Z, Status: open\n", + "Issue created at: 2023-12-26T07:25:37Z, Status: closed\n", + "Issue created at: 2023-12-26T05:01:26Z, Status: closed\n", + "Issue created at: 2023-12-26T03:54:15Z, Status: open\n", + "Issue created at: 2023-12-26T02:19:48Z, Status: open\n", + "Issue created at: 2023-12-25T12:26:40Z, Status: open\n", + "Issue created at: 2023-12-25T12:00:52Z, Status: open\n", + "Issue created at: 2023-12-25T08:36:56Z, Status: open\n", + "Issue created at: 2023-12-25T08:35:48Z, Status: open\n", + "Issue created at: 2023-12-25T08:34:13Z, Status: closed\n", + "Issue created at: 2023-12-25T06:33:37Z, Status: closed\n", + "Issue created at: 2023-12-23T11:47:40Z, Status: closed\n", + "Issue created at: 2023-12-23T03:01:23Z, Status: open\n", + "Issue created at: 2023-12-19T06:38:57Z, Status: open\n", + "Issue created at: 2023-12-11T08:34:08Z, Status: open\n", + "Issue created at: 2023-11-26T03:55:22Z, Status: open\n", + "Issue created at: 2023-11-25T12:38:23Z, Status: open\n", + "Issue created at: 2023-11-25T02:25:13Z, Status: open\n", + "Issue created at: 2023-11-25T02:09:58Z, Status: open\n", + "Issue created at: 2023-11-24T08:48:24Z, Status: open\n", + "Issue created at: 2023-11-24T06:03:56Z, Status: open\n", + "Issue created at: 2023-11-23T16:25:33Z, Status: open\n", + "Issue created at: 2023-11-23T08:54:36Z, Status: open\n", + "Issue created at: 2023-11-23T08:26:27Z, Status: open\n", + "Issue created at: 2023-11-22T06:42:34Z, Status: open\n", + "Issue created at: 2023-10-26T01:45:37Z, Status: open\n", + "Issue created at: 2023-08-30T06:02:32Z, Status: closed\n", + "Issue created at: 2023-06-18T01:11:22Z, Status: open\n", + "\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GroupChat is underpopulated with 1 agents. It is recommended to set speaker_selection_method to 'round_robin' or allow_repeat_speaker to False.Or, use direct communication instead.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mplan_manager\u001b[0m (to CodeEngineer)-[]:\n", + "\n", + "\"The execution result of the code you wrote is judged as not answering the task question. Please re-understand and complete the task.\"\n", + "\n", + "--------------------------------------------------------------------------------\n", + "un_stream ai response: I apologize for the inconvenience. Let me provide you with a Python script that includes a check to ensure the `requests` module is installed. If it's not installed, the script will attempt to install it using `pip`. Please run the following Python script:\n", + "\n", + "```python\n", + "# filename: github_issues.py\n", + "import subprocess\n", + "import pkg_resources\n", + "from datetime import datetime, timedelta\n", + "\n", + "# Ensure the 'requests' package is installed\n", + "required_packages = {'requests'}\n", + "installed_packages = {pkg.key for pkg in pkg_resources.working_set}\n", + "missing_packages = required_packages - installed_packages\n", + "\n", + "if missing_packages:\n", + " subprocess.check_call(['pip', 'install', *missing_packages])\n", + "\n", + "import requests\n", + "\n", + "# Function to get issues created in the past three days from a GitHub repository\n", + "def get_recent_issues(owner, repo):\n", + " # Calculate the date for three days ago\n", + " three_days_ago = datetime.utcnow() - timedelta(days=3)\n", + " # Format the date in ISO 8601 format\n", + " since = three_days_ago.isoformat()\n", + "\n", + " # GitHub API URL for issues of the given repository\n", + " issues_url = f\"https://api.github.com/repos/{owner}/{repo}/issues\"\n", + " \n", + " # Parameters to get issues created in the past three days\n", + " params = {\n", + " 'since': since,\n", + " 'state': 'all', # Get both open and closed issues\n", + " }\n", + " \n", + " # Send a request to the GitHub API\n", + " response = requests.get(issues_url, params=params)\n", + " # Raise an exception if the request was unsuccessful\n", + " response.raise_for_status()\n", + " \n", + " # Parse the JSON response\n", + " issues = response.json()\n", + " \n", + " # Extract and print the creation date and status of each issue\n", + " for issue in issues:\n", + " created_at = issue['created_at']\n", + " state = issue['state']\n", + " print(f\"Issue created at: {created_at}, Status: {state}\")\n", + "\n", + "# Replace 'owner' and 'repo' with the actual owner and repository name\n", + "get_recent_issues('eosphoros-ai', 'DB-GPT')\n", + "```\n", + "\n", + "When you run this script, it will check for the `requests` module and install it if necessary. Then it will retrieve the issues from the 'eosphoros-ai/DB-GPT' repository created in the past three days and print their creation date and status.\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", + "un_stream ai response: False\n", + "\u001b[33mCodeEngineer\u001b[0m (to plan_manager)-[gpt-4-vision-preview]:\n", + "\n", + "\"I apologize for the inconvenience. Let me provide you with a Python script that includes a check to ensure the `requests` module is installed. If it's not installed, the script will attempt to install it using `pip`. Please run the following Python script:\\n\\n```python\\n# filename: github_issues.py\\nimport subprocess\\nimport pkg_resources\\nfrom datetime import datetime, timedelta\\n\\n# Ensure the 'requests' package is installed\\nrequired_packages = {'requests'}\\ninstalled_packages = {pkg.key for pkg in pkg_resources.working_set}\\nmissing_packages = required_packages - installed_packages\\n\\nif missing_packages:\\n subprocess.check_call(['pip', 'install', *missing_packages])\\n\\nimport requests\\n\\n# Function to get issues created in the past three days from a GitHub repository\\ndef get_recent_issues(owner, repo):\\n # Calculate the date for three days ago\\n three_days_ago = datetime.utcnow() - timedelta(days=3)\\n # Format the date in ISO 8601 format\\n since = three_days_ago.isoformat()\\n\\n # GitHub API URL for issues of the given repository\\n issues_url = f\\\"https://api.github.com/repos/{owner}/{repo}/issues\\\"\\n \\n # Parameters to get issues created in the past three days\\n params = {\\n 'since': since,\\n 'state': 'all', # Get both open and closed issues\\n }\\n \\n # Send a request to the GitHub API\\n response = requests.get(issues_url, params=params)\\n # Raise an exception if the request was unsuccessful\\n response.raise_for_status()\\n \\n # Parse the JSON response\\n issues = response.json()\\n \\n # Extract and print the creation date and status of each issue\\n for issue in issues:\\n created_at = issue['created_at']\\n state = issue['state']\\n print(f\\\"Issue created at: {created_at}, Status: {state}\\\")\\n\\n# Replace 'owner' and 'repo' with the actual owner and repository name\\nget_recent_issues('eosphoros-ai', 'DB-GPT')\\n```\\n\\nWhen you run this script, it will check for the `requests` module and install it if necessary. Then it will retrieve the issues from the 'eosphoros-ai/DB-GPT' repository created in the past three days and print their creation date and status.\"\n", + "\u001b[32m>>>>>>>>CodeEngineer Review info: \n", + " Pass.None\u001b[0m\n", + "\u001b[34m>>>>>>>>CodeEngineer Action report: \n", + "execution succeeded,\n", + "\n", + "/workspace/github_issues.py:3: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html\n", + " import pkg_resources\n", + "/workspace/github_issues.py:19: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).\n", + " three_days_ago = datetime.utcnow() - timedelta(days=3)\n", + "Issue created at: 2023-12-27T03:20:12Z, Status: open\n", + "Issue created at: 2023-12-26T16:13:42Z, Status: open\n", + "Issue created at: 2023-12-26T10:23:43Z, Status: open\n", + "Issue created at: 2023-12-26T07:25:37Z, Status: closed\n", + "Issue created at: 2023-12-26T05:01:26Z, Status: closed\n", + "Issue created at: 2023-12-26T03:54:15Z, Status: open\n", + "Issue created at: 2023-12-26T02:19:48Z, Status: open\n", + "Issue created at: 2023-12-25T12:26:40Z, Status: open\n", + "Issue created at: 2023-12-25T12:00:52Z, Status: open\n", + "Issue created at: 2023-12-25T08:36:56Z, Status: open\n", + "Issue created at: 2023-12-25T08:35:48Z, Status: open\n", + "Issue created at: 2023-12-25T08:34:13Z, Status: closed\n", + "Issue created at: 2023-12-25T06:33:37Z, Status: closed\n", + "Issue created at: 2023-12-23T11:47:40Z, Status: closed\n", + "Issue created at: 2023-12-23T03:01:23Z, Status: open\n", + "Issue created at: 2023-12-19T06:38:57Z, Status: open\n", + "Issue created at: 2023-12-11T08:34:08Z, Status: open\n", + "Issue created at: 2023-11-26T03:55:22Z, Status: open\n", + "Issue created at: 2023-11-25T12:38:23Z, Status: open\n", + "Issue created at: 2023-11-25T02:25:13Z, Status: open\n", + "Issue created at: 2023-11-25T02:09:58Z, Status: open\n", + "Issue created at: 2023-11-24T08:48:24Z, Status: open\n", + "Issue created at: 2023-11-24T06:03:56Z, Status: open\n", + "Issue created at: 2023-11-23T16:25:33Z, Status: open\n", + "Issue created at: 2023-11-23T08:54:36Z, Status: open\n", + "Issue created at: 2023-11-23T08:26:27Z, Status: open\n", + "Issue created at: 2023-11-22T06:42:34Z, Status: open\n", + "Issue created at: 2023-10-26T01:45:37Z, Status: open\n", + "Issue created at: 2023-08-30T06:02:32Z, Status: closed\n", + "Issue created at: 2023-06-18T01:11:22Z, Status: open\n", + "\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GroupChat is underpopulated with 1 agents. It is recommended to set speaker_selection_method to 'round_robin' or allow_repeat_speaker to False.Or, use direct communication instead.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mplan_manager\u001b[0m (to CodeEngineer)-[]:\n", + "\n", + "\"The execution result of the code you wrote is judged as not answering the task question. Please re-understand and complete the task.\"\n", + "\n", + "--------------------------------------------------------------------------------\n", + "un_stream ai response: I apologize for the oversight. Let's try a different approach to ensure we get the issues created in the past three days from the 'eosphoros-ai/DB-GPT' repository. I will provide you with an updated Python script that correctly retrieves and filters the issues based on their creation date.\n", + "\n", + "```python\n", + "# filename: github_issues.py\n", + "import requests\n", + "from datetime import datetime, timedelta\n", + "\n", + "# Function to get issues created in the past three days from a GitHub repository\n", + "def get_recent_issues(owner, repo):\n", + " # Calculate the date for three days ago\n", + " three_days_ago = datetime.utcnow() - timedelta(days=3)\n", + " # Format the date in ISO 8601 format\n", + " since = three_days_ago.isoformat()\n", + "\n", + " # GitHub API URL for issues of the given repository\n", + " issues_url = f\"https://api.github.com/repos/{owner}/{repo}/issues\"\n", + "\n", + " # Parameters to get all issues, GitHub API does not allow filtering by creation date directly\n", + " params = {\n", + " 'state': 'all', # Get both open and closed issues\n", + " }\n", + "\n", + " # Send a request to the GitHub API\n", + " response = requests.get(issues_url, params=params)\n", + " # Raise an exception if the request was unsuccessful\n", + " response.raise_for_status()\n", + "\n", + " # Parse the JSON response\n", + " issues = response.json()\n", + "\n", + " # Filter and print the creation date and status of each issue created in the past three days\n", + " for issue in issues:\n", + " created_at = datetime.strptime(issue['created_at'], \"%Y-%m-%dT%H:%M:%SZ\")\n", + " if created_at > three_days_ago:\n", + " state = issue['state']\n", + " print(f\"Issue created at: {created_at}, Status: {state}\")\n", + "\n", + "# Replace 'owner' and 'repo' with the actual owner and repository name\n", + "get_recent_issues('eosphoros-ai', 'DB-GPT')\n", + "```\n", + "\n", + "This script retrieves all issues from the specified repository and then filters them based on the creation date to find the ones that were created in the past three days. It then prints the creation date and status of each issue that meets the criteria. Please execute this updated script.\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", + "un_stream ai response: True\n", + "\u001b[33mCodeEngineer\u001b[0m (to plan_manager)-[gpt-4-vision-preview]:\n", + "\n", + "\"I apologize for the oversight. Let's try a different approach to ensure we get the issues created in the past three days from the 'eosphoros-ai/DB-GPT' repository. I will provide you with an updated Python script that correctly retrieves and filters the issues based on their creation date.\\n\\n```python\\n# filename: github_issues.py\\nimport requests\\nfrom datetime import datetime, timedelta\\n\\n# Function to get issues created in the past three days from a GitHub repository\\ndef get_recent_issues(owner, repo):\\n # Calculate the date for three days ago\\n three_days_ago = datetime.utcnow() - timedelta(days=3)\\n # Format the date in ISO 8601 format\\n since = three_days_ago.isoformat()\\n\\n # GitHub API URL for issues of the given repository\\n issues_url = f\\\"https://api.github.com/repos/{owner}/{repo}/issues\\\"\\n\\n # Parameters to get all issues, GitHub API does not allow filtering by creation date directly\\n params = {\\n 'state': 'all', # Get both open and closed issues\\n }\\n\\n # Send a request to the GitHub API\\n response = requests.get(issues_url, params=params)\\n # Raise an exception if the request was unsuccessful\\n response.raise_for_status()\\n\\n # Parse the JSON response\\n issues = response.json()\\n\\n # Filter and print the creation date and status of each issue created in the past three days\\n for issue in issues:\\n created_at = datetime.strptime(issue['created_at'], \\\"%Y-%m-%dT%H:%M:%SZ\\\")\\n if created_at > three_days_ago:\\n state = issue['state']\\n print(f\\\"Issue created at: {created_at}, Status: {state}\\\")\\n\\n# Replace 'owner' and 'repo' with the actual owner and repository name\\nget_recent_issues('eosphoros-ai', 'DB-GPT')\\n```\\n\\nThis script retrieves all issues from the specified repository and then filters them based on the creation date to find the ones that were created in the past three days. It then prints the creation date and status of each issue that meets the criteria. Please execute this updated script.\"\n", + "\u001b[32m>>>>>>>>CodeEngineer Review info: \n", + " Pass.None\u001b[0m\n", + "\u001b[34m>>>>>>>>CodeEngineer Action report: \n", + "execution succeeded,\n", + "\n", + "/workspace/github_issues.py:8: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).\n", + " three_days_ago = datetime.utcnow() - timedelta(days=3)\n", + "Issue created at: 2023-12-27 03:20:12, Status: open\n", + "Issue created at: 2023-12-26 16:13:42, Status: open\n", + "Issue created at: 2023-12-26 10:23:43, Status: open\n", + "Issue created at: 2023-12-26 07:25:37, Status: closed\n", + "Issue created at: 2023-12-26 05:01:26, Status: closed\n", + "Issue created at: 2023-12-26 03:54:15, Status: open\n", + "Issue created at: 2023-12-26 02:19:48, Status: open\n", + "Issue created at: 2023-12-25 12:26:40, Status: open\n", + "Issue created at: 2023-12-25 12:00:52, Status: open\n", + "Issue created at: 2023-12-25 08:36:56, Status: open\n", + "Issue created at: 2023-12-25 08:35:48, Status: open\n", + "Issue created at: 2023-12-25 08:34:13, Status: closed\n", + "Issue created at: 2023-12-25 06:33:37, Status: closed\n", + "\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "GroupChat is underpopulated with 1 agents. It is recommended to set speaker_selection_method to 'round_robin' or allow_repeat_speaker to False.Or, use direct communication instead.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mplan_manager\u001b[0m (to CodeEngineer)-[]:\n", + "\n", + "\"Read the result data of the dependent steps in the above historical message to complete the current goal:Write a Python script to process the retrieved issues data to group them by creation date and status. Then, format the grouped data into a Markdown table.\"\n", + "\n", + "--------------------------------------------------------------------------------\n", + "un_stream ai response: ```python\n", + "# Let's process the retrieved issues data to group them by creation date and status.\n", + "# Then, we will format the grouped data into a Markdown table.\n", + "\n", + "# The data from the previous step\n", + "issues_data = [\n", + " {\"created_at\": \"2023-12-27 03:20:12\", \"status\": \"open\"},\n", + " {\"created_at\": \"2023-12-26 16:13:42\", \"status\": \"open\"},\n", + " {\"created_at\": \"2023-12-26 10:23:43\", \"status\": \"open\"},\n", + " {\"created_at\": \"2023-12-26 07:25:37\", \"status\": \"closed\"},\n", + " {\"created_at\": \"2023-12-26 05:01:26\", \"status\": \"closed\"},\n", + " {\"created_at\": \"2023-12-26 03:54:15\", \"status\": \"open\"},\n", + " {\"created_at\": \"2023-12-26 02:19:48\", \"status\": \"open\"},\n", + " {\"created_at\": \"2023-12-25 12:26:40\", \"status\": \"open\"},\n", + " {\"created_at\": \"2023-12-25 12:00:52\", \"status\": \"open\"},\n", + " {\"created_at\": \"2023-12-25 08:36:56\", \"status\": \"open\"},\n", + " {\"created_at\": \"2023-12-25 08:35:48\", \"status\": \"open\"},\n", + " {\"created_at\": \"2023-12-25 08:34:13\", \"status\": \"closed\"},\n", + " {\"created_at\": \"2023-12-25 06:33:37\", \"status\": \"closed\"},\n", + "]\n", + "\n", + "# Grouping the issues by creation date and status\n", + "from collections import defaultdict\n", + "grouped_issues = defaultdict(lambda: defaultdict(list))\n", + "\n", + "for issue in issues_data:\n", + " date = issue[\"created_at\"].split(' ')[0] # Extract just the date part\n", + " grouped_issues[date][issue[\"status\"]].append(issue)\n", + "\n", + "# Format the grouped data into a Markdown table\n", + "markdown_table = \"Date | Open Issues | Closed Issues\\n\"\n", + "markdown_table += \"--- | --- | ---\\n\"\n", + "\n", + "for date, statuses in sorted(grouped_issues.items()):\n", + " open_issues = len(statuses[\"open\"]) if \"open\" in statuses else 0\n", + " closed_issues = len(statuses[\"closed\"]) if \"closed\" in statuses else 0\n", + " markdown_table += f\"{date} | {open_issues} | {closed_issues}\\n\"\n", + "\n", + "print(markdown_table)\n", + "```\n", + "\n", + "When you run this script, it will print out a Markdown table with the number of open and closed issues grouped by their creation dates.\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", + "un_stream ai response: True\n", + "\u001b[33mCodeEngineer\u001b[0m (to plan_manager)-[gpt-4-vision-preview]:\n", + "\n", + "\"```python\\n# Let's process the retrieved issues data to group them by creation date and status.\\n# Then, we will format the grouped data into a Markdown table.\\n\\n# The data from the previous step\\nissues_data = [\\n {\\\"created_at\\\": \\\"2023-12-27 03:20:12\\\", \\\"status\\\": \\\"open\\\"},\\n {\\\"created_at\\\": \\\"2023-12-26 16:13:42\\\", \\\"status\\\": \\\"open\\\"},\\n {\\\"created_at\\\": \\\"2023-12-26 10:23:43\\\", \\\"status\\\": \\\"open\\\"},\\n {\\\"created_at\\\": \\\"2023-12-26 07:25:37\\\", \\\"status\\\": \\\"closed\\\"},\\n {\\\"created_at\\\": \\\"2023-12-26 05:01:26\\\", \\\"status\\\": \\\"closed\\\"},\\n {\\\"created_at\\\": \\\"2023-12-26 03:54:15\\\", \\\"status\\\": \\\"open\\\"},\\n {\\\"created_at\\\": \\\"2023-12-26 02:19:48\\\", \\\"status\\\": \\\"open\\\"},\\n {\\\"created_at\\\": \\\"2023-12-25 12:26:40\\\", \\\"status\\\": \\\"open\\\"},\\n {\\\"created_at\\\": \\\"2023-12-25 12:00:52\\\", \\\"status\\\": \\\"open\\\"},\\n {\\\"created_at\\\": \\\"2023-12-25 08:36:56\\\", \\\"status\\\": \\\"open\\\"},\\n {\\\"created_at\\\": \\\"2023-12-25 08:35:48\\\", \\\"status\\\": \\\"open\\\"},\\n {\\\"created_at\\\": \\\"2023-12-25 08:34:13\\\", \\\"status\\\": \\\"closed\\\"},\\n {\\\"created_at\\\": \\\"2023-12-25 06:33:37\\\", \\\"status\\\": \\\"closed\\\"},\\n]\\n\\n# Grouping the issues by creation date and status\\nfrom collections import defaultdict\\ngrouped_issues = defaultdict(lambda: defaultdict(list))\\n\\nfor issue in issues_data:\\n date = issue[\\\"created_at\\\"].split(' ')[0] # Extract just the date part\\n grouped_issues[date][issue[\\\"status\\\"]].append(issue)\\n\\n# Format the grouped data into a Markdown table\\nmarkdown_table = \\\"Date | Open Issues | Closed Issues\\\\n\\\"\\nmarkdown_table += \\\"--- | --- | ---\\\\n\\\"\\n\\nfor date, statuses in sorted(grouped_issues.items()):\\n open_issues = len(statuses[\\\"open\\\"]) if \\\"open\\\" in statuses else 0\\n closed_issues = len(statuses[\\\"closed\\\"]) if \\\"closed\\\" in statuses else 0\\n markdown_table += f\\\"{date} | {open_issues} | {closed_issues}\\\\n\\\"\\n\\nprint(markdown_table)\\n```\\n\\nWhen you run this script, it will print out a Markdown table with the number of open and closed issues grouped by their creation dates.\"\n", + "\u001b[32m>>>>>>>>CodeEngineer Review info: \n", + " Pass.None\u001b[0m\n", + "\u001b[34m>>>>>>>>CodeEngineer Action report: \n", + "execution succeeded,\n", + "\n", + "Date | Open Issues | Closed Issues\n", + "--- | --- | ---\n", + "2023-12-25 | 4 | 2\n", + "2023-12-26 | 4 | 2\n", + "2023-12-27 | 1 | 0\n", + "\n", + "\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mplan_manager\u001b[0m (to User)-[None]:\n", + "\n", + "\"Obtain simple information about issues in the repository 'eosphoros-ai/DB-GPT' in the past three days and analyze the data. Create a Markdown table grouped by day and status.\"\n", + "\u001b[32m>>>>>>>>plan_manager Review info: \n", + " Pass.None\u001b[0m\n", + "\u001b[34m>>>>>>>>plan_manager Action report: \n", + "execution succeeded,\n", + "TERMINATE\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "```agent-messages\n", + "[{\"sender\": \"User\", \"receiver\": \"plan_manager\", \"model\": null, \"markdown\": \"Obtain simple information about issues in the repository 'eosphoros-ai/DB-GPT' in the past three days and analyze the data. Create a Markdown table grouped by day and status.\"}, {\"sender\": \"plan_manager\", \"receiver\": \"Planner\", \"model\": null, \"markdown\": \"Obtain simple information about issues in the repository 'eosphoros-ai/DB-GPT' in the past three days and analyze the data. Create a Markdown table grouped by day and status.\"}, {\"sender\": \"Planner\", \"receiver\": \"plan_manager\", \"model\": \"gpt-4-vision-preview\", \"markdown\": \"1,Write a Python script to use the GitHub API to retrieve issues from the 'eosphoros-ai/DB-GPT' repository that were created in the past three days. The script should extract the issue's creation date and status.\\n2,Write a Python script to process the retrieved issues data to group them by creation date and status. Then, format the grouped data into a Markdown table.\"}, {\"sender\": \"plan_manager\", \"receiver\": \"User\", \"model\": null, \"markdown\": \"TERMINATE\"}]\n", + "```\n", + "```agent-plans\n", + "[{\"name\": \"Write a Python script to use the GitHub API to retrieve issues from the 'eosphoros-ai/DB-GPT' repository that were created in the past three days. The script should extract the issue's creation date and status.\", \"num\": \"1\", \"status\": \"complete\", \"agent\": \"CodeEngineer\", \"markdown\": \"```agent-messages\\n[{\\\"sender\\\": \\\"plan_manager\\\", \\\"receiver\\\": \\\"CodeEngineer\\\", \\\"model\\\": null, \\\"markdown\\\": \\\"Write a Python script to use the GitHub API to retrieve issues from the 'eosphoros-ai/DB-GPT' repository that were created in the past three days. The script should extract the issue's creation date and status.\\\"}, {\\\"sender\\\": \\\"CodeEngineer\\\", \\\"receiver\\\": \\\"plan_manager\\\", \\\"model\\\": \\\"gpt-4-vision-preview\\\", \\\"markdown\\\": \\\"```vis-code\\\\n{\\\\\\\"exit_success\\\\\\\": false, \\\\\\\"language\\\\\\\": \\\\\\\"python\\\\\\\", \\\\\\\"code\\\\\\\": [[\\\\\\\"python\\\\\\\", \\\\\\\"# filename: github_issues.py\\\\\\\\nimport requests\\\\\\\\nfrom datetime import datetime, timedelta\\\\\\\\n\\\\\\\\n# Function to get issues created in the past three days from a GitHub repository\\\\\\\\ndef get_recent_issues(owner, repo):\\\\\\\\n # Calculate the date for three days ago\\\\\\\\n three_days_ago = datetime.utcnow() - timedelta(days=3)\\\\\\\\n # Format the date in ISO 8601 format\\\\\\\\n since = three_days_ago.isoformat()\\\\\\\\n\\\\\\\\n # GitHub API URL for issues of the given repository\\\\\\\\n issues_url = f\\\\\\\\\\\\\\\"https://api.github.com/repos/{owner}/{repo}/issues\\\\\\\\\\\\\\\"\\\\\\\\n \\\\\\\\n # Parameters to get issues created in the past three days\\\\\\\\n params = {\\\\\\\\n 'since': since,\\\\\\\\n 'state': 'all', # Get both open and closed issues\\\\\\\\n }\\\\\\\\n \\\\\\\\n # Send a request to the GitHub API\\\\\\\\n response = requests.get(issues_url, params=params)\\\\\\\\n # Raise an exception if the request was unsuccessful\\\\\\\\n response.raise_for_status()\\\\\\\\n \\\\\\\\n # Parse the JSON response\\\\\\\\n issues = response.json()\\\\\\\\n \\\\\\\\n # Extract and print the creation date and status of each issue\\\\\\\\n for issue in issues:\\\\\\\\n created_at = issue['created_at']\\\\\\\\n state = issue['state']\\\\\\\\n print(f\\\\\\\\\\\\\\\"Issue created at: {created_at}, Status: {state}\\\\\\\\\\\\\\\")\\\\\\\\n\\\\\\\\n# Replace 'owner' and 'repo' with the actual owner and repository name\\\\\\\\nget_recent_issues('eosphoros-ai', 'DB-GPT')\\\\\\\"]], \\\\\\\"log\\\\\\\": \\\\\\\"\\\\\\\\nTraceback (most recent call last):\\\\\\\\n File \\\\\\\\\\\\\\\"github_issues.py\\\\\\\\\\\\\\\", line 2, in \\\\\\\\n import requests\\\\\\\\nModuleNotFoundError: No module named 'requests'\\\\\\\\n\\\\\\\"}\\\\n```\\\"}, {\\\"sender\\\": \\\"plan_manager\\\", \\\"receiver\\\": \\\"CodeEngineer\\\", \\\"model\\\": null, \\\"markdown\\\": \\\"exitcode: 1 (execution failed)\\\\n \\\\nTraceback (most recent call last):\\\\n File \\\\\\\"github_issues.py\\\\\\\", line 2, in \\\\n import requests\\\\nModuleNotFoundError: No module named 'requests'\\\\n\\\"}, {\\\"sender\\\": \\\"CodeEngineer\\\", \\\"receiver\\\": \\\"plan_manager\\\", \\\"model\\\": \\\"gpt-4-vision-preview\\\", \\\"markdown\\\": \\\"```vis-code\\\\n{\\\\\\\"exit_success\\\\\\\": true, \\\\\\\"language\\\\\\\": \\\\\\\"sh\\\\\\\", \\\\\\\"code\\\\\\\": [[\\\\\\\"sh\\\\\\\", \\\\\\\"pip install requests\\\\\\\"], [\\\\\\\"python\\\\\\\", \\\\\\\"# filename: github_issues.py\\\\\\\\nimport requests\\\\\\\\nfrom datetime import datetime, timedelta\\\\\\\\n\\\\\\\\n# Function to get issues created in the past three days from a GitHub repository\\\\\\\\ndef get_recent_issues(owner, repo):\\\\\\\\n # Calculate the date for three days ago\\\\\\\\n three_days_ago = datetime.utcnow() - timedelta(days=3)\\\\\\\\n # Format the date in ISO 8601 format\\\\\\\\n since = three_days_ago.isoformat()\\\\\\\\n\\\\\\\\n # GitHub API URL for issues of the given repository\\\\\\\\n issues_url = f\\\\\\\\\\\\\\\"https://api.github.com/repos/{owner}/{repo}/issues\\\\\\\\\\\\\\\"\\\\\\\\n \\\\\\\\n # Parameters to get issues created in the past three days\\\\\\\\n params = {\\\\\\\\n 'since': since,\\\\\\\\n 'state': 'all', # Get both open and closed issues\\\\\\\\n }\\\\\\\\n \\\\\\\\n # Send a request to the GitHub API\\\\\\\\n response = requests.get(issues_url, params=params)\\\\\\\\n # Raise an exception if the request was unsuccessful\\\\\\\\n response.raise_for_status()\\\\\\\\n \\\\\\\\n # Parse the JSON response\\\\\\\\n issues = response.json()\\\\\\\\n \\\\\\\\n # Extract and print the creation date and status of each issue\\\\\\\\n for issue in issues:\\\\\\\\n created_at = issue['created_at']\\\\\\\\n state = issue['state']\\\\\\\\n print(f\\\\\\\\\\\\\\\"Issue created at: {created_at}, Status: {state}\\\\\\\\\\\\\\\")\\\\\\\\n\\\\\\\\n# Replace 'owner' and 'repo' with the actual owner and repository name\\\\\\\\nget_recent_issues('eosphoros-ai', 'DB-GPT')\\\\\\\"]], \\\\\\\"log\\\\\\\": \\\\\\\"\\\\\\\\nCollecting requests\\\\\\\\n Obtaining dependency information for requests from https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl.metadata\\\\\\\\n Downloading requests-2.31.0-py3-none-any.whl.metadata (4.6 kB)\\\\\\\\nCollecting charset-normalizer<4,>=2 (from requests)\\\\\\\\n Obtaining dependency information for charset-normalizer<4,>=2 from https://files.pythonhosted.org/packages/45/59/3d27019d3b447a88fe7e7d004a1e04be220227760264cc41b405e863891b/charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl.metadata\\\\\\\\n Downloading charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl.metadata (33 kB)\\\\\\\\nCollecting idna<4,>=2.5 (from requests)\\\\\\\\n Obtaining dependency information for idna<4,>=2.5 from https://files.pythonhosted.org/packages/c2/e7/a82b05cf63a603df6e68d59ae6a68bf5064484a0718ea5033660af4b54a9/idna-3.6-py3-none-any.whl.metadata\\\\\\\\n Downloading idna-3.6-py3-none-any.whl.metadata (9.9 kB)\\\\\\\\nCollecting urllib3<3,>=1.21.1 (from requests)\\\\\\\\n Obtaining dependency information for urllib3<3,>=1.21.1 from https://files.pythonhosted.org/packages/96/94/c31f58c7a7f470d5665935262ebd7455c7e4c7782eb525658d3dbf4b9403/urllib3-2.1.0-py3-none-any.whl.metadata\\\\\\\\n Downloading urllib3-2.1.0-py3-none-any.whl.metadata (6.4 kB)\\\\\\\\nCollecting certifi>=2017.4.17 (from requests)\\\\\\\\n Obtaining dependency information for certifi>=2017.4.17 from https://files.pythonhosted.org/packages/64/62/428ef076be88fa93716b576e4a01f919d25968913e817077a386fcbe4f42/certifi-2023.11.17-py3-none-any.whl.metadata\\\\\\\\n Downloading certifi-2023.11.17-py3-none-any.whl.metadata (2.2 kB)\\\\\\\\nDownloading requests-2.31.0-py3-none-any.whl (62 kB)\\\\\\\\n \\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501 62.6/62.6 kB 1.9 MB/s eta 0:00:00\\\\\\\\nDownloading certifi-2023.11.17-py3-none-any.whl (162 kB)\\\\\\\\n \\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501 162.5/162.5 kB 1.9 MB/s eta 0:00:00\\\\\\\\nDownloading charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl (138 kB)\\\\\\\\n \\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501 138.3/138.3 kB 4.6 MB/s eta 0:00:00\\\\\\\\nDownloading idna-3.6-py3-none-any.whl (61 kB)\\\\\\\\n \\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501 61.6/61.6 kB 3.2 MB/s eta 0:00:00\\\\\\\\nDownloading urllib3-2.1.0-py3-none-any.whl (104 kB)\\\\\\\\n \\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501\\\\\\\\u2501 104.6/104.6 kB 4.9 MB/s eta 0:00:00\\\\\\\\nInstalling collected packages: urllib3, idna, charset-normalizer, certifi, requests\\\\\\\\nSuccessfully installed certifi-2023.11.17 charset-normalizer-3.3.2 idna-3.6 requests-2.31.0 urllib3-2.1.0\\\\\\\\nWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\\\\\\\\n\\\\\\\\n[notice] A new release of pip is available: 23.2.1 -> 23.3.2\\\\\\\\n[notice] To update, run: pip install --upgrade pip\\\\\\\\n\\\\\\\\n/workspace/github_issues.py:8: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).\\\\\\\\n three_days_ago = datetime.utcnow() - timedelta(days=3)\\\\\\\\nIssue created at: 2023-12-27T03:20:12Z, Status: open\\\\\\\\nIssue created at: 2023-12-26T16:13:42Z, Status: open\\\\\\\\nIssue created at: 2023-12-26T10:23:43Z, Status: open\\\\\\\\nIssue created at: 2023-12-26T07:25:37Z, Status: closed\\\\\\\\nIssue created at: 2023-12-26T05:01:26Z, Status: closed\\\\\\\\nIssue created at: 2023-12-26T03:54:15Z, Status: open\\\\\\\\nIssue created at: 2023-12-26T02:19:48Z, Status: open\\\\\\\\nIssue created at: 2023-12-25T12:26:40Z, Status: open\\\\\\\\nIssue created at: 2023-12-25T12:00:52Z, Status: open\\\\\\\\nIssue created at: 2023-12-25T08:36:56Z, Status: open\\\\\\\\nIssue created at: 2023-12-25T08:35:48Z, Status: open\\\\\\\\nIssue created at: 2023-12-25T08:34:13Z, Status: closed\\\\\\\\nIssue created at: 2023-12-25T06:33:37Z, Status: closed\\\\\\\\nIssue created at: 2023-12-23T11:47:40Z, Status: closed\\\\\\\\nIssue created at: 2023-12-23T03:01:23Z, Status: open\\\\\\\\nIssue created at: 2023-12-19T06:38:57Z, Status: open\\\\\\\\nIssue created at: 2023-12-11T08:34:08Z, Status: open\\\\\\\\nIssue created at: 2023-11-26T03:55:22Z, Status: open\\\\\\\\nIssue created at: 2023-11-25T12:38:23Z, Status: open\\\\\\\\nIssue created at: 2023-11-25T02:25:13Z, Status: open\\\\\\\\nIssue created at: 2023-11-25T02:09:58Z, Status: open\\\\\\\\nIssue created at: 2023-11-24T08:48:24Z, Status: open\\\\\\\\nIssue created at: 2023-11-24T06:03:56Z, Status: open\\\\\\\\nIssue created at: 2023-11-23T16:25:33Z, Status: open\\\\\\\\nIssue created at: 2023-11-23T08:54:36Z, Status: open\\\\\\\\nIssue created at: 2023-11-23T08:26:27Z, Status: open\\\\\\\\nIssue created at: 2023-11-22T06:42:34Z, Status: open\\\\\\\\nIssue created at: 2023-10-26T01:45:37Z, Status: open\\\\\\\\nIssue created at: 2023-08-30T06:02:32Z, Status: closed\\\\\\\\nIssue created at: 2023-06-18T01:11:22Z, Status: open\\\\\\\\n\\\\\\\"}\\\\n```\\\"}, {\\\"sender\\\": \\\"plan_manager\\\", \\\"receiver\\\": \\\"CodeEngineer\\\", \\\"model\\\": null, \\\"markdown\\\": \\\"The execution result of the code you wrote is judged as not answering the task question. Please re-understand and complete the task.\\\"}, {\\\"sender\\\": \\\"CodeEngineer\\\", \\\"receiver\\\": \\\"plan_manager\\\", \\\"model\\\": \\\"gpt-4-vision-preview\\\", \\\"markdown\\\": \\\"```vis-code\\\\n{\\\\\\\"exit_success\\\\\\\": true, \\\\\\\"language\\\\\\\": \\\\\\\"python\\\\\\\", \\\\\\\"code\\\\\\\": [[\\\\\\\"python\\\\\\\", \\\\\\\"# filename: github_issues.py\\\\\\\\nimport subprocess\\\\\\\\nimport pkg_resources\\\\\\\\nfrom datetime import datetime, timedelta\\\\\\\\n\\\\\\\\n# Ensure the 'requests' package is installed\\\\\\\\nrequired_packages = {'requests'}\\\\\\\\ninstalled_packages = {pkg.key for pkg in pkg_resources.working_set}\\\\\\\\nmissing_packages = required_packages - installed_packages\\\\\\\\n\\\\\\\\nif missing_packages:\\\\\\\\n subprocess.check_call(['pip', 'install', *missing_packages])\\\\\\\\n\\\\\\\\nimport requests\\\\\\\\n\\\\\\\\n# Function to get issues created in the past three days from a GitHub repository\\\\\\\\ndef get_recent_issues(owner, repo):\\\\\\\\n # Calculate the date for three days ago\\\\\\\\n three_days_ago = datetime.utcnow() - timedelta(days=3)\\\\\\\\n # Format the date in ISO 8601 format\\\\\\\\n since = three_days_ago.isoformat()\\\\\\\\n\\\\\\\\n # GitHub API URL for issues of the given repository\\\\\\\\n issues_url = f\\\\\\\\\\\\\\\"https://api.github.com/repos/{owner}/{repo}/issues\\\\\\\\\\\\\\\"\\\\\\\\n \\\\\\\\n # Parameters to get issues created in the past three days\\\\\\\\n params = {\\\\\\\\n 'since': since,\\\\\\\\n 'state': 'all', # Get both open and closed issues\\\\\\\\n }\\\\\\\\n \\\\\\\\n # Send a request to the GitHub API\\\\\\\\n response = requests.get(issues_url, params=params)\\\\\\\\n # Raise an exception if the request was unsuccessful\\\\\\\\n response.raise_for_status()\\\\\\\\n \\\\\\\\n # Parse the JSON response\\\\\\\\n issues = response.json()\\\\\\\\n \\\\\\\\n # Extract and print the creation date and status of each issue\\\\\\\\n for issue in issues:\\\\\\\\n created_at = issue['created_at']\\\\\\\\n state = issue['state']\\\\\\\\n print(f\\\\\\\\\\\\\\\"Issue created at: {created_at}, Status: {state}\\\\\\\\\\\\\\\")\\\\\\\\n\\\\\\\\n# Replace 'owner' and 'repo' with the actual owner and repository name\\\\\\\\nget_recent_issues('eosphoros-ai', 'DB-GPT')\\\\\\\"]], \\\\\\\"log\\\\\\\": \\\\\\\"\\\\\\\\n/workspace/github_issues.py:3: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html\\\\\\\\n import pkg_resources\\\\\\\\n/workspace/github_issues.py:19: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).\\\\\\\\n three_days_ago = datetime.utcnow() - timedelta(days=3)\\\\\\\\nIssue created at: 2023-12-27T03:20:12Z, Status: open\\\\\\\\nIssue created at: 2023-12-26T16:13:42Z, Status: open\\\\\\\\nIssue created at: 2023-12-26T10:23:43Z, Status: open\\\\\\\\nIssue created at: 2023-12-26T07:25:37Z, Status: closed\\\\\\\\nIssue created at: 2023-12-26T05:01:26Z, Status: closed\\\\\\\\nIssue created at: 2023-12-26T03:54:15Z, Status: open\\\\\\\\nIssue created at: 2023-12-26T02:19:48Z, Status: open\\\\\\\\nIssue created at: 2023-12-25T12:26:40Z, Status: open\\\\\\\\nIssue created at: 2023-12-25T12:00:52Z, Status: open\\\\\\\\nIssue created at: 2023-12-25T08:36:56Z, Status: open\\\\\\\\nIssue created at: 2023-12-25T08:35:48Z, Status: open\\\\\\\\nIssue created at: 2023-12-25T08:34:13Z, Status: closed\\\\\\\\nIssue created at: 2023-12-25T06:33:37Z, Status: closed\\\\\\\\nIssue created at: 2023-12-23T11:47:40Z, Status: closed\\\\\\\\nIssue created at: 2023-12-23T03:01:23Z, Status: open\\\\\\\\nIssue created at: 2023-12-19T06:38:57Z, Status: open\\\\\\\\nIssue created at: 2023-12-11T08:34:08Z, Status: open\\\\\\\\nIssue created at: 2023-11-26T03:55:22Z, Status: open\\\\\\\\nIssue created at: 2023-11-25T12:38:23Z, Status: open\\\\\\\\nIssue created at: 2023-11-25T02:25:13Z, Status: open\\\\\\\\nIssue created at: 2023-11-25T02:09:58Z, Status: open\\\\\\\\nIssue created at: 2023-11-24T08:48:24Z, Status: open\\\\\\\\nIssue created at: 2023-11-24T06:03:56Z, Status: open\\\\\\\\nIssue created at: 2023-11-23T16:25:33Z, Status: open\\\\\\\\nIssue created at: 2023-11-23T08:54:36Z, Status: open\\\\\\\\nIssue created at: 2023-11-23T08:26:27Z, Status: open\\\\\\\\nIssue created at: 2023-11-22T06:42:34Z, Status: open\\\\\\\\nIssue created at: 2023-10-26T01:45:37Z, Status: open\\\\\\\\nIssue created at: 2023-08-30T06:02:32Z, Status: closed\\\\\\\\nIssue created at: 2023-06-18T01:11:22Z, Status: open\\\\\\\\n\\\\\\\"}\\\\n```\\\"}, {\\\"sender\\\": \\\"plan_manager\\\", \\\"receiver\\\": \\\"CodeEngineer\\\", \\\"model\\\": null, \\\"markdown\\\": \\\"The execution result of the code you wrote is judged as not answering the task question. Please re-understand and complete the task.\\\"}, {\\\"sender\\\": \\\"CodeEngineer\\\", \\\"receiver\\\": \\\"plan_manager\\\", \\\"model\\\": \\\"gpt-4-vision-preview\\\", \\\"markdown\\\": \\\"```vis-code\\\\n{\\\\\\\"exit_success\\\\\\\": true, \\\\\\\"language\\\\\\\": \\\\\\\"python\\\\\\\", \\\\\\\"code\\\\\\\": [[\\\\\\\"python\\\\\\\", \\\\\\\"# filename: github_issues.py\\\\\\\\nimport requests\\\\\\\\nfrom datetime import datetime, timedelta\\\\\\\\n\\\\\\\\n# Function to get issues created in the past three days from a GitHub repository\\\\\\\\ndef get_recent_issues(owner, repo):\\\\\\\\n # Calculate the date for three days ago\\\\\\\\n three_days_ago = datetime.utcnow() - timedelta(days=3)\\\\\\\\n # Format the date in ISO 8601 format\\\\\\\\n since = three_days_ago.isoformat()\\\\\\\\n\\\\\\\\n # GitHub API URL for issues of the given repository\\\\\\\\n issues_url = f\\\\\\\\\\\\\\\"https://api.github.com/repos/{owner}/{repo}/issues\\\\\\\\\\\\\\\"\\\\\\\\n\\\\\\\\n # Parameters to get all issues, GitHub API does not allow filtering by creation date directly\\\\\\\\n params = {\\\\\\\\n 'state': 'all', # Get both open and closed issues\\\\\\\\n }\\\\\\\\n\\\\\\\\n # Send a request to the GitHub API\\\\\\\\n response = requests.get(issues_url, params=params)\\\\\\\\n # Raise an exception if the request was unsuccessful\\\\\\\\n response.raise_for_status()\\\\\\\\n\\\\\\\\n # Parse the JSON response\\\\\\\\n issues = response.json()\\\\\\\\n\\\\\\\\n # Filter and print the creation date and status of each issue created in the past three days\\\\\\\\n for issue in issues:\\\\\\\\n created_at = datetime.strptime(issue['created_at'], \\\\\\\\\\\\\\\"%Y-%m-%dT%H:%M:%SZ\\\\\\\\\\\\\\\")\\\\\\\\n if created_at > three_days_ago:\\\\\\\\n state = issue['state']\\\\\\\\n print(f\\\\\\\\\\\\\\\"Issue created at: {created_at}, Status: {state}\\\\\\\\\\\\\\\")\\\\\\\\n\\\\\\\\n# Replace 'owner' and 'repo' with the actual owner and repository name\\\\\\\\nget_recent_issues('eosphoros-ai', 'DB-GPT')\\\\\\\"]], \\\\\\\"log\\\\\\\": \\\\\\\"\\\\\\\\n/workspace/github_issues.py:8: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).\\\\\\\\n three_days_ago = datetime.utcnow() - timedelta(days=3)\\\\\\\\nIssue created at: 2023-12-27 03:20:12, Status: open\\\\\\\\nIssue created at: 2023-12-26 16:13:42, Status: open\\\\\\\\nIssue created at: 2023-12-26 10:23:43, Status: open\\\\\\\\nIssue created at: 2023-12-26 07:25:37, Status: closed\\\\\\\\nIssue created at: 2023-12-26 05:01:26, Status: closed\\\\\\\\nIssue created at: 2023-12-26 03:54:15, Status: open\\\\\\\\nIssue created at: 2023-12-26 02:19:48, Status: open\\\\\\\\nIssue created at: 2023-12-25 12:26:40, Status: open\\\\\\\\nIssue created at: 2023-12-25 12:00:52, Status: open\\\\\\\\nIssue created at: 2023-12-25 08:36:56, Status: open\\\\\\\\nIssue created at: 2023-12-25 08:35:48, Status: open\\\\\\\\nIssue created at: 2023-12-25 08:34:13, Status: closed\\\\\\\\nIssue created at: 2023-12-25 06:33:37, Status: closed\\\\\\\\n\\\\\\\"}\\\\n```\\\"}]\\n```\"}, {\"name\": \"Write a Python script to process the retrieved issues data to group them by creation date and status. Then, format the grouped data into a Markdown table.\", \"num\": \"2\", \"status\": \"complete\", \"agent\": \"CodeEngineer\", \"markdown\": \"```agent-messages\\n[{\\\"sender\\\": \\\"plan_manager\\\", \\\"receiver\\\": \\\"CodeEngineer\\\", \\\"model\\\": null, \\\"markdown\\\": \\\"Read the result data of the dependent steps in the above historical message to complete the current goal:Write a Python script to process the retrieved issues data to group them by creation date and status. Then, format the grouped data into a Markdown table.\\\"}, {\\\"sender\\\": \\\"CodeEngineer\\\", \\\"receiver\\\": \\\"plan_manager\\\", \\\"model\\\": \\\"gpt-4-vision-preview\\\", \\\"markdown\\\": \\\"```vis-code\\\\n{\\\\\\\"exit_success\\\\\\\": true, \\\\\\\"language\\\\\\\": \\\\\\\"python\\\\\\\", \\\\\\\"code\\\\\\\": [[\\\\\\\"python\\\\\\\", \\\\\\\"# Let's process the retrieved issues data to group them by creation date and status.\\\\\\\\n# Then, we will format the grouped data into a Markdown table.\\\\\\\\n\\\\\\\\n# The data from the previous step\\\\\\\\nissues_data = [\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-27 03:20:12\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"open\\\\\\\\\\\\\\\"},\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-26 16:13:42\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"open\\\\\\\\\\\\\\\"},\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-26 10:23:43\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"open\\\\\\\\\\\\\\\"},\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-26 07:25:37\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"closed\\\\\\\\\\\\\\\"},\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-26 05:01:26\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"closed\\\\\\\\\\\\\\\"},\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-26 03:54:15\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"open\\\\\\\\\\\\\\\"},\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-26 02:19:48\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"open\\\\\\\\\\\\\\\"},\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-25 12:26:40\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"open\\\\\\\\\\\\\\\"},\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-25 12:00:52\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"open\\\\\\\\\\\\\\\"},\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-25 08:36:56\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"open\\\\\\\\\\\\\\\"},\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-25 08:35:48\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"open\\\\\\\\\\\\\\\"},\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-25 08:34:13\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"closed\\\\\\\\\\\\\\\"},\\\\\\\\n {\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"2023-12-25 06:33:37\\\\\\\\\\\\\\\", \\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\": \\\\\\\\\\\\\\\"closed\\\\\\\\\\\\\\\"},\\\\\\\\n]\\\\\\\\n\\\\\\\\n# Grouping the issues by creation date and status\\\\\\\\nfrom collections import defaultdict\\\\\\\\ngrouped_issues = defaultdict(lambda: defaultdict(list))\\\\\\\\n\\\\\\\\nfor issue in issues_data:\\\\\\\\n date = issue[\\\\\\\\\\\\\\\"created_at\\\\\\\\\\\\\\\"].split(' ')[0] # Extract just the date part\\\\\\\\n grouped_issues[date][issue[\\\\\\\\\\\\\\\"status\\\\\\\\\\\\\\\"]].append(issue)\\\\\\\\n\\\\\\\\n# Format the grouped data into a Markdown table\\\\\\\\nmarkdown_table = \\\\\\\\\\\\\\\"Date | Open Issues | Closed Issues\\\\\\\\\\\\\\\\n\\\\\\\\\\\\\\\"\\\\\\\\nmarkdown_table += \\\\\\\\\\\\\\\"--- | --- | ---\\\\\\\\\\\\\\\\n\\\\\\\\\\\\\\\"\\\\\\\\n\\\\\\\\nfor date, statuses in sorted(grouped_issues.items()):\\\\\\\\n open_issues = len(statuses[\\\\\\\\\\\\\\\"open\\\\\\\\\\\\\\\"]) if \\\\\\\\\\\\\\\"open\\\\\\\\\\\\\\\" in statuses else 0\\\\\\\\n closed_issues = len(statuses[\\\\\\\\\\\\\\\"closed\\\\\\\\\\\\\\\"]) if \\\\\\\\\\\\\\\"closed\\\\\\\\\\\\\\\" in statuses else 0\\\\\\\\n markdown_table += f\\\\\\\\\\\\\\\"{date} | {open_issues} | {closed_issues}\\\\\\\\\\\\\\\\n\\\\\\\\\\\\\\\"\\\\\\\\n\\\\\\\\nprint(markdown_table)\\\\\\\"]], \\\\\\\"log\\\\\\\": \\\\\\\"\\\\\\\\nDate | Open Issues | Closed Issues\\\\\\\\n--- | --- | ---\\\\\\\\n2023-12-25 | 4 | 2\\\\\\\\n2023-12-26 | 4 | 2\\\\\\\\n2023-12-27 | 1 | 0\\\\\\\\n\\\\\\\\n\\\\\\\"}\\\\n```\\\"}]\\n```\"}]\n", + "```\n" + ] + } + ], + "source": [ + "llm_client = OpenAILLMClient()\n", + "context: AgentContext = AgentContext(conv_id=\"test\", llm_provider=llm_client)\n", + "# context.llm_models = [ModelMetadata(model=\"gpt-3.5-turbo\")]\n", + "context.llm_models = [ModelMetadata(model=\"gpt-4-vision-preview\")]\n", + "context.gpts_name = \"代码分析助手\"\n", + "\n", + "default_memory = GptsMemory()\n", + "coder = CodeAssistantAgent(memory=default_memory, agent_context=context)\n", + "## TODO add other agent\n", + "\n", + "groupchat = PlanChat(agents=[coder], messages=[], max_round=50)\n", + "planner = PlannerAgent(\n", + " agent_context=context,\n", + " memory=default_memory,\n", + " plan_chat=groupchat,\n", + ")\n", + "\n", + "manager = PlanChatManager(\n", + " plan_chat=groupchat,\n", + " planner=planner,\n", + " agent_context=context,\n", + " memory=default_memory,\n", + ")\n", + "\n", + "user_proxy = UserProxyAgent(memory=default_memory, agent_context=context)\n", + "\n", + "\n", + "await user_proxy.a_initiate_chat(\n", + " recipient=manager,\n", + " reviewer=user_proxy,\n", + " message=\"Obtain simple information about issues in the repository 'eosphoros-ai/DB-GPT' in the past three days and analyze the data. Create a Markdown table grouped by day and status.\",\n", + " # message=\"Find papers on gpt-4 in the past three weeks on arxiv, and organize their titles, authors, and links into a markdown table\",\n", + " # message=\"find papers on LLM applications from arxiv in the last month, create a markdown table of different domains.\",\n", + " )\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "dbgpt_env", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "f8b6b0e04f284afd2fbb5e4163e7d03bbdc845eaeb6e8c78fae04fce6b51dae6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}