From e61dfbbe562d920cf7637fd9d931eb946cad62a1 Mon Sep 17 00:00:00 2001 From: Fangyin Cheng Date: Thu, 28 Dec 2023 00:51:11 +0800 Subject: [PATCH 1/6] chore: Agent json dumpls support no ascii encoding --- dbgpt/agent/agents/base_agent.py | 8 ++++---- dbgpt/agent/agents/expand/code_assistant_agent.py | 2 +- dbgpt/agent/agents/expand/dashboard_assistant_agent.py | 2 +- dbgpt/agent/agents/expand/data_scientist_agent.py | 2 +- dbgpt/agent/agents/llm/llm_client.py | 2 +- dbgpt/serve/agent/db/gpts_conversations_db.py | 5 ----- dbgpt/serve/agent/db/gpts_mange_db.py | 4 ---- dbgpt/serve/agent/db/gpts_messages_db.py | 4 ---- dbgpt/serve/agent/db/gpts_plans_db.py | 4 ---- 9 files changed, 8 insertions(+), 25 deletions(-) diff --git a/dbgpt/agent/agents/base_agent.py b/dbgpt/agent/agents/base_agent.py index db0753d25..e42528046 100644 --- a/dbgpt/agent/agents/base_agent.py +++ b/dbgpt/agent/agents/base_agent.py @@ -249,13 +249,13 @@ def append_message(self, message: Optional[Dict], role, sender: Agent) -> bool: rounds=self.consecutive_auto_reply_counter, current_gogal=oai_message.get("current_gogal", None), content=oai_message.get("content", None), - context=json.dumps(oai_message["context"]) + context=json.dumps(oai_message["context"], ensure_ascii=False) if "context" in oai_message else None, - review_info=json.dumps(oai_message["review_info"]) + review_info=json.dumps(oai_message["review_info"], ensure_ascii=False) if "review_info" in oai_message else None, - action_report=json.dumps(oai_message["action_report"]) + action_report=json.dumps(oai_message["action_report"], ensure_ascii=False) if "action_report" in oai_message else None, model_name=oai_message.get("model_name", None), @@ -300,7 +300,7 @@ def _print_received_message(self, message: Union[Dict, str], sender: Agent): print(message["content"], flush=True) print(colored("*" * len(func_print), "green"), flush=True) else: - content = json.dumps(message.get("content")) + content = json.dumps(message.get("content"), ensure_ascii=False) if content is not None: if "context" in message: content = AIWrapper.instantiate( diff --git a/dbgpt/agent/agents/expand/code_assistant_agent.py b/dbgpt/agent/agents/expand/code_assistant_agent.py index 89d60209f..3e16bd472 100644 --- a/dbgpt/agent/agents/expand/code_assistant_agent.py +++ b/dbgpt/agent/agents/expand/code_assistant_agent.py @@ -109,7 +109,7 @@ def _vis_code_idea(self, code, exit_success, log, language): param["code"] = code param["log"] = log - return f"```vis-code\n{json.dumps(param)}\n```" + return f"```vis-code\n{json.dumps(param, ensure_ascii=False)}\n```" async def generate_code_execution_reply( self, diff --git a/dbgpt/agent/agents/expand/dashboard_assistant_agent.py b/dbgpt/agent/agents/expand/dashboard_assistant_agent.py index 88c0b03a5..76cf44f01 100644 --- a/dbgpt/agent/agents/expand/dashboard_assistant_agent.py +++ b/dbgpt/agent/agents/expand/dashboard_assistant_agent.py @@ -103,7 +103,7 @@ async def generate_dashboard_reply( else: try: chart_objs = json_objects[0] - content = json.dumps(chart_objs) + content = json.dumps(chart_objs, ensure_ascii=False) vis_client = ApiCall() view = vis_client.display_dashboard_vis( chart_objs, self.db_connect.run_to_df diff --git a/dbgpt/agent/agents/expand/data_scientist_agent.py b/dbgpt/agent/agents/expand/data_scientist_agent.py index 3574009cf..7b004d188 100644 --- a/dbgpt/agent/agents/expand/data_scientist_agent.py +++ b/dbgpt/agent/agents/expand/data_scientist_agent.py @@ -109,7 +109,7 @@ async def generate_analysis_chart_reply( rensponse_succ = False else: try: - content = json.dumps(json_objects[0]) + content = json.dumps(json_objects[0], ensure_ascii=False) except Exception as e: content = ( f"There is a format problem with the json of the answer，{str(e)}" diff --git a/dbgpt/agent/agents/llm/llm_client.py b/dbgpt/agent/agents/llm/llm_client.py index 47f478804..5caf17c94 100644 --- a/dbgpt/agent/agents/llm/llm_client.py +++ b/dbgpt/agent/agents/llm/llm_client.py @@ -103,7 +103,7 @@ def _get_key(self, config): if key in config: config, copied = config.copy() if not copied else config, True config.pop(key) - return json.dumps(config, sort_keys=True) + return json.dumps(config, sort_keys=True, ensure_ascii=False) async def create(self, **config): # merge the input config with the i-th config in the config list diff --git a/dbgpt/serve/agent/db/gpts_conversations_db.py b/dbgpt/serve/agent/db/gpts_conversations_db.py index 67375592b..92ecb76c2 100644 --- a/dbgpt/serve/agent/db/gpts_conversations_db.py +++ b/dbgpt/serve/agent/db/gpts_conversations_db.py @@ -9,11 +9,6 @@ class GptsConversationsEntity(Model): __tablename__ = "gpts_conversations" - __table_args__ = { - "mysql_charset": "utf8mb4", - "mysql_collate": "utf8mb4_unicode_ci", - } - id = Column(Integer, primary_key=True, comment="autoincrement id") conv_id = Column( diff --git a/dbgpt/serve/agent/db/gpts_mange_db.py b/dbgpt/serve/agent/db/gpts_mange_db.py index 91f578e9f..3db062b94 100644 --- a/dbgpt/serve/agent/db/gpts_mange_db.py +++ b/dbgpt/serve/agent/db/gpts_mange_db.py @@ -8,10 +8,6 @@ class GptsInstanceEntity(Model): __tablename__ = "gpts_instance" - __table_args__ = { - "mysql_charset": "utf8mb4", - "mysql_collate": "utf8mb4_unicode_ci", - } id = Column(Integer, primary_key=True, comment="autoincrement id") gpts_name = Column(String(255), nullable=False, comment="Current AI assistant name") diff --git a/dbgpt/serve/agent/db/gpts_messages_db.py b/dbgpt/serve/agent/db/gpts_messages_db.py index 2945fe59e..26e87abfc 100644 --- a/dbgpt/serve/agent/db/gpts_messages_db.py +++ b/dbgpt/serve/agent/db/gpts_messages_db.py @@ -19,10 +19,6 @@ class GptsMessagesEntity(Model): __tablename__ = "gpts_messages" - __table_args__ = { - "mysql_charset": "utf8mb4", - "mysql_collate": "utf8mb4_unicode_ci", - } id = Column(Integer, primary_key=True, comment="autoincrement id") conv_id = Column( diff --git a/dbgpt/serve/agent/db/gpts_plans_db.py b/dbgpt/serve/agent/db/gpts_plans_db.py index f01840cbf..d0c2bce9d 100644 --- a/dbgpt/serve/agent/db/gpts_plans_db.py +++ b/dbgpt/serve/agent/db/gpts_plans_db.py @@ -9,10 +9,6 @@ class GptsPlansEntity(Model): __tablename__ = "gpts_plans" - __table_args__ = { - "mysql_charset": "utf8mb4", - "mysql_collate": "utf8mb4_unicode_ci", - } id = Column(Integer, primary_key=True, comment="autoincrement id") conv_id = Column( From e899de55f9a78770ac12b961a51239dc519d2441 Mon Sep 17 00:00:00 2001 From: Fangyin Cheng Date: Thu, 28 Dec 2023 03:04:15 +0800 Subject: [PATCH 2/6] chore(agent): Fix agent typo --- dbgpt/agent/agents/agent.py | 33 +++++++-------- dbgpt/agent/agents/agents_mange.py | 2 +- dbgpt/agent/agents/base_agent.py | 20 +++------ .../agents/expand/code_assistant_agent.py | 22 ++++------ .../expand/dashboard_assistant_agent.py | 27 ++++-------- .../agents/expand/data_scientist_agent.py | 23 ++++------- .../agents/expand/plugin_assistant_agent.py | 21 +++------- .../agents/expand/sql_assistant_agent.py | 17 ++------ dbgpt/agent/agents/llm/llm_client.py | 4 +- dbgpt/agent/agents/plan_group_chat.py | 9 ++-- dbgpt/agent/agents/planner_agent.py | 5 ++- dbgpt/agent/agents/user_proxy_agent.py | 9 +--- dbgpt/agent/memory/__init__.py | 0 dbgpt/agent/memory/base.py | 6 +-- dbgpt/agent/memory/gpts_memory.py | 4 +- dbgpt/agent/memory/gpts_memory_storage.py | 14 +++---- .../plugin/commands/built_in/audio_text.py | 1 + .../plugin/commands/built_in/image_gen.py | 1 + dbgpt/agent/plugin/commands/command.py | 2 +- dbgpt/agent/plugin/commands/command_mange.py | 4 +- dbgpt/agent/plugin/plugin.py | 3 +- dbgpt/core/__init__.py | 41 +++++++++---------- dbgpt/core/_private/example_base.py | 3 +- 23 files changed, 109 insertions(+), 162 deletions(-) create mode 100644 dbgpt/agent/memory/__init__.py diff --git a/dbgpt/agent/agents/agent.py b/dbgpt/agent/agents/agent.py index ea31eb676..c9dfc2083 100644 --- a/dbgpt/agent/agents/agent.py +++ b/dbgpt/agent/agents/agent.py @@ -1,18 +1,16 @@ from __future__ import annotations import dataclasses -from collections import defaultdict -from dataclasses import asdict, dataclass, fields -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Tuple, Union -from ..memory.gpts_memory import GptsMemory from dbgpt.core import LLMClient from dbgpt.core.interface.llm import ModelMetadata +from ..memory.gpts_memory import GptsMemory + class Agent: - """ - An interface for AI agent. + """An interface for AI agent. An agent can communicate with other agents and perform actions. """ @@ -33,16 +31,16 @@ def __init__( self._memory = memory @property - def name(self): + def name(self) -> str: """Get the name of the agent.""" return self._name @property - def memory(self): + def memory(self) -> GptsMemory: return self._memory @property - def describe(self): + def describe(self) -> str: """Get the name of the agent.""" return self._describe @@ -53,7 +51,7 @@ async def a_send( reviewer: Agent, request_reply: Optional[bool] = True, is_recovery: Optional[bool] = False, - ): + ) -> None: """(Abstract async method) Send a message to another agent.""" async def a_receive( @@ -64,10 +62,12 @@ async def a_receive( request_reply: Optional[bool] = None, silent: Optional[bool] = False, is_recovery: Optional[bool] = False, - ): + ) -> None: """(Abstract async method) Receive a message from another agent.""" - async def a_review(self, message: Union[Dict, str], censored: Agent): + async def a_review( + self, message: Union[Dict, str], censored: Agent + ) -> Tuple[bool, Any]: """ Args: @@ -75,10 +75,11 @@ async def a_review(self, message: Union[Dict, str], censored: Agent): censored: Returns: - + bool: whether the message is censored + Any: the censored message """ - def reset(self): + def reset(self) -> None: """(Abstract method) Reset the agent.""" async def a_generate_reply( @@ -145,7 +146,7 @@ async def a_verify_reply( """ -@dataclass +@dataclasses.dataclass class AgentResource: type: str name: str @@ -165,7 +166,7 @@ def to_dict(self) -> Dict[str, Any]: return dataclasses.asdict(self) -@dataclass +@dataclasses.dataclass class AgentContext: conv_id: str llm_provider: LLMClient diff --git a/dbgpt/agent/agents/agents_mange.py b/dbgpt/agent/agents/agents_mange.py index eb20635ff..3d5ab7659 100644 --- a/dbgpt/agent/agents/agents_mange.py +++ b/dbgpt/agent/agents/agents_mange.py @@ -1,5 +1,5 @@ from collections import defaultdict -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union +from typing import Optional, Type from .agent import Agent from .expand.code_assistant_agent import CodeAssistantAgent diff --git a/dbgpt/agent/agents/base_agent.py b/dbgpt/agent/agents/base_agent.py index e42528046..0f80aa438 100644 --- a/dbgpt/agent/agents/base_agent.py +++ b/dbgpt/agent/agents/base_agent.py @@ -5,22 +5,14 @@ from typing import Any, Callable, Dict, List, Optional, Type, Union from dbgpt.agent.agents.llm.llm_client import AIWrapper -from dbgpt.core.awel import BaseOperator from dbgpt.core.interface.message import ModelMessageRoleType from dbgpt.util.error_types import LLMChatError +from dbgpt.util.utils import colored from ..memory.base import GptsMessage from ..memory.gpts_memory import GptsMemory from .agent import Agent, AgentContext -try: - from termcolor import colored -except ImportError: - - def colored(x, *args, **kwargs): - return x - - logger = logging.getLogger(__name__) @@ -360,7 +352,7 @@ def _gpts_message_to_ai_message( self, gpts_messages: Optional[List[GptsMessage]] ) -> List[Dict]: oai_messages: List[Dict] = [] - ###Based on the current agent, all messages received are user, and all messages sent are assistant. + # Based on the current agent, all messages received are user, and all messages sent are assistant. for item in gpts_messages: role = "" if item.role: @@ -390,21 +382,21 @@ def _gpts_message_to_ai_message( return oai_messages def process_now_message(self, sender, current_gogal: Optional[str] = None): - ### Convert and tailor the information in collective memory into contextual memory available to the current Agent + # Convert and tailor the information in collective memory into contextual memory available to the current Agent current_gogal_messages = self._gpts_message_to_ai_message( self.memory.message_memory.get_between_agents( self.agent_context.conv_id, self.name, sender.name, current_gogal ) ) - ### relay messages + # relay messages cut_messages = [] cut_messages.extend(self._rely_messages) if len(current_gogal_messages) < self.dialogue_memory_rounds: cut_messages.extend(current_gogal_messages) else: - ### TODO 基于token预算来分配历史信息 + # TODO: allocate historical information based on token budget cut_messages.extend(current_gogal_messages[:2]) # end_round = self.dialogue_memory_rounds - 2 cut_messages.extend(current_gogal_messages[-3:]) @@ -488,7 +480,7 @@ async def a_receive( logger.info( "The generated answer failed to verify, so send it to yourself for optimization." ) - ### TODO 自优化最大轮次后，异常退出 + # TODO: Exit after the maximum number of rounds of self-optimization await sender.a_send( message=reply, recipient=self, reviewer=reviewer, silent=silent ) diff --git a/dbgpt/agent/agents/expand/code_assistant_agent.py b/dbgpt/agent/agents/expand/code_assistant_agent.py index 3e16bd472..2cbd5fc00 100644 --- a/dbgpt/agent/agents/expand/code_assistant_agent.py +++ b/dbgpt/agent/agents/expand/code_assistant_agent.py @@ -1,21 +1,14 @@ import json -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union +from typing import Callable, Dict, Literal, Optional, Union -from dbgpt.core.awel import BaseOperator +from dbgpt.core.interface.message import ModelMessageRoleType from dbgpt.util.code_utils import UNKNOWN, execute_code, extract_code, infer_lang from dbgpt.util.string_utils import str_to_bool +from dbgpt.util.utils import colored from ...memory.gpts_memory import GptsMemory from ..agent import Agent, AgentContext from ..base_agent import ConversableAgent -from dbgpt.core.interface.message import ModelMessageRoleType - -try: - from termcolor import colored -except ImportError: - - def colored(x, *args, **kwargs): - return x class CodeAssistantAgent(ConversableAgent): @@ -171,7 +164,7 @@ async def a_verify(self, message: Optional[Dict]): if action_report: task_result = action_report.get("content", "") - check_reult, model = await self.a_reasoning_reply( + check_result, model = await self.a_reasoning_reply( [ { "role": ModelMessageRoleType.HUMAN, @@ -183,11 +176,11 @@ async def a_verify(self, message: Optional[Dict]): } ] ) - sucess = str_to_bool(check_reult) + success = str_to_bool(check_result) fail_reason = None - if sucess == False: + if not success: fail_reason = "The execution result of the code you wrote is judged as not answering the task question. Please re-understand and complete the task." - return sucess, fail_reason + return success, fail_reason @property def use_docker(self) -> Union[bool, str, None]: @@ -219,6 +212,7 @@ def run_code(self, code, **kwargs): def execute_code_blocks(self, code_blocks): """Execute the code blocks and return the result.""" logs_all = "" + exitcode = -1 for i, code_block in enumerate(code_blocks): lang, code = code_block if not lang: diff --git a/dbgpt/agent/agents/expand/dashboard_assistant_agent.py b/dbgpt/agent/agents/expand/dashboard_assistant_agent.py index 76cf44f01..99f583985 100644 --- a/dbgpt/agent/agents/expand/dashboard_assistant_agent.py +++ b/dbgpt/agent/agents/expand/dashboard_assistant_agent.py @@ -1,6 +1,7 @@ import json -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union +from typing import Callable, Dict, Literal, Optional, Union +from dbgpt._private.config import Config from dbgpt.agent.plugin.commands.command_mange import ApiCall from dbgpt.util.json_utils import find_json_objects @@ -8,17 +9,7 @@ from ..agent import Agent, AgentContext from ..base_agent import ConversableAgent -try: - from termcolor import colored -except ImportError: - - def colored(x, *args, **kwargs): - return x - - -from dbgpt._private.config import Config -from dbgpt.core.awel import BaseOperator - +# TODO: remove global config CFG = Config() @@ -93,13 +84,13 @@ async def generate_dashboard_reply( "Please recheck your answer，no usable plans generated in correct format，" ) json_count = len(json_objects) - rensponse_succ = True + response_success = True view = None content = None if json_count != 1: - ### Answer failed, turn on automatic repair + # Answer failed, turn on automatic repair fail_reason += f"There are currently {json_count} json contents" - rensponse_succ = False + response_success = False else: try: chart_objs = json_objects[0] @@ -110,12 +101,12 @@ async def generate_dashboard_reply( ) except Exception as e: fail_reason += f"Return json structure error and cannot be converted to a sql-rendered chart，{str(e)}" - rensponse_succ = False + response_success = False - if not rensponse_succ: + if not response_success: content = fail_reason return True, { - "is_exe_success": rensponse_succ, + "is_exe_success": response_success, "content": content, "view": view, } diff --git a/dbgpt/agent/agents/expand/data_scientist_agent.py b/dbgpt/agent/agents/expand/data_scientist_agent.py index 7b004d188..50cd05961 100644 --- a/dbgpt/agent/agents/expand/data_scientist_agent.py +++ b/dbgpt/agent/agents/expand/data_scientist_agent.py @@ -1,24 +1,16 @@ import json import logging -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union +from typing import Callable, Dict, Literal, Optional, Union from dbgpt._private.config import Config from dbgpt.agent.plugin.commands.command_mange import ApiCall -from dbgpt.core.awel import BaseOperator from dbgpt.util.json_utils import find_json_objects from ...memory.gpts_memory import GptsMemory from ..agent import Agent, AgentContext from ..base_agent import ConversableAgent -try: - from termcolor import colored -except ImportError: - - def colored(x, *args, **kwargs): - return x - - +# TODO: remove global config CFG = Config() logger = logging.getLogger(__name__) @@ -101,12 +93,12 @@ async def generate_analysis_chart_reply( json_objects = find_json_objects(message) fail_reason = "The required json format answer was not generated." json_count = len(json_objects) - rensponse_succ = True + response_success = True view = None content = None if json_count != 1: - ### Answer failed, turn on automatic repair - rensponse_succ = False + # Answer failed, turn on automatic repair + response_success = False else: try: content = json.dumps(json_objects[0], ensure_ascii=False) @@ -114,7 +106,7 @@ async def generate_analysis_chart_reply( content = ( f"There is a format problem with the json of the answer，{str(e)}" ) - rensponse_succ = False + response_success = False try: vis_client = ApiCall() view = vis_client.display_only_sql_vis( @@ -124,13 +116,14 @@ async def generate_analysis_chart_reply( view = f"```vis-convert-error\n{content}\n```" return True, { - "is_exe_success": rensponse_succ, + "is_exe_success": response_success, "content": content, "view": view, } async def a_verify(self, message: Optional[Dict]): action_reply = message.get("action_report", None) + # TODO None has no method get if action_reply.get("is_exe_success", False) == False: return ( False, diff --git a/dbgpt/agent/agents/expand/plugin_assistant_agent.py b/dbgpt/agent/agents/expand/plugin_assistant_agent.py index 6e7845e96..f9ecd2586 100644 --- a/dbgpt/agent/agents/expand/plugin_assistant_agent.py +++ b/dbgpt/agent/agents/expand/plugin_assistant_agent.py @@ -1,22 +1,12 @@ -import json import logging -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union +from typing import Callable, Dict, Literal, Optional, Union -from dbgpt.core.awel import BaseOperator from dbgpt.util.json_utils import find_json_objects from ...memory.gpts_memory import GptsMemory from ..agent import Agent, AgentContext from ..base_agent import ConversableAgent -try: - from termcolor import colored -except ImportError: - - def colored(x, *args, **kwargs): - return x - - logger = logging.getLogger(__name__) @@ -88,6 +78,7 @@ def __init__( self.agent_context = agent_context async def a_system_fill_param(self): + # TODO no db_connect attribute params = { "tool_infos": self.db_connect.get_table_info(), "dialect": self.db_connect.db_type, @@ -106,12 +97,12 @@ async def tool_call( json_objects = find_json_objects(message) fail_reason = "The required json format answer was not generated." json_count = len(json_objects) - rensponse_succ = True + response_success = True view = None content = None if json_count != 1: - ### Answer failed, turn on automatic repair - rensponse_succ = False + # Answer failed, turn on automatic repair + response_success = False else: try: view = "" @@ -119,7 +110,7 @@ async def tool_call( view = f"```vis-convert-error\n{content}\n```" return True, { - "is_exe_success": rensponse_succ, + "is_exe_success": response_success, "content": content, "view": view, } diff --git a/dbgpt/agent/agents/expand/sql_assistant_agent.py b/dbgpt/agent/agents/expand/sql_assistant_agent.py index 0900a151c..cfd3fc528 100644 --- a/dbgpt/agent/agents/expand/sql_assistant_agent.py +++ b/dbgpt/agent/agents/expand/sql_assistant_agent.py @@ -1,22 +1,13 @@ -from typing import Callable, Dict, List, Literal, Optional, Union +from typing import Callable, Dict, Literal, Optional, Union +from dbgpt._private.config import Config from dbgpt.agent.agents.base_agent import ConversableAgent -from dbgpt.core.awel import BaseOperator - from dbgpt.agent.plugin.commands.command_mange import ApiCall + from ...memory.gpts_memory import GptsMemory from ..agent import Agent, AgentContext -try: - from termcolor import colored -except ImportError: - - def colored(x, *args, **kwargs): - return x - - -from dbgpt._private.config import Config - +# TODO: remove global config CFG = Config() diff --git a/dbgpt/agent/agents/llm/llm_client.py b/dbgpt/agent/agents/llm/llm_client.py index 5caf17c94..e67575021 100644 --- a/dbgpt/agent/agents/llm/llm_client.py +++ b/dbgpt/agent/agents/llm/llm_client.py @@ -97,9 +97,9 @@ def _get_key(self, config): Returns: tuple: A unique identifier which can be used as a key for a dict. """ - NON_CACHE_KEY = ["api_key", "base_url", "api_type", "api_version"] + non_cache_key = ["api_key", "base_url", "api_type", "api_version"] copied = False - for key in NON_CACHE_KEY: + for key in non_cache_key: if key in config: config, copied = config.copy() if not copied else config, True config.pop(key) diff --git a/dbgpt/agent/agents/plan_group_chat.py b/dbgpt/agent/agents/plan_group_chat.py index 4b55a4aaf..91cc7ee7b 100644 --- a/dbgpt/agent/agents/plan_group_chat.py +++ b/dbgpt/agent/agents/plan_group_chat.py @@ -1,19 +1,16 @@ -import json import logging -import random import re import sys from dataclasses import dataclass from typing import Dict, List, Optional, Union -from dbgpt.core.awel import BaseOperator -from dbgpt.util.string_utils import str_to_bool +from dbgpt.core.interface.message import ModelMessageRoleType from ..common.schema import Status -from ..memory.gpts_memory import GptsMemory, GptsMessage, GptsPlan +from ..memory.base import GptsPlan +from ..memory.gpts_memory import GptsMemory from .agent import Agent, AgentContext from .base_agent import ConversableAgent -from dbgpt.core.interface.message import ModelMessageRoleType logger = logging.getLogger(__name__) diff --git a/dbgpt/agent/agents/planner_agent.py b/dbgpt/agent/agents/planner_agent.py index a71d1b43a..3a00e7e16 100644 --- a/dbgpt/agent/agents/planner_agent.py +++ b/dbgpt/agent/agents/planner_agent.py @@ -3,13 +3,14 @@ from dbgpt._private.config import Config from dbgpt.agent.agents.plan_group_chat import PlanChat from dbgpt.agent.common.schema import Status -from dbgpt.core.awel import BaseOperator from dbgpt.util.json_utils import find_json_objects -from ..memory.gpts_memory import GptsMemory, GptsPlan +from ..memory.base import GptsPlan +from ..memory.gpts_memory import GptsMemory from .agent import Agent, AgentContext from .base_agent import ConversableAgent +# TODO: remove global config CFG = Config() diff --git a/dbgpt/agent/agents/user_proxy_agent.py b/dbgpt/agent/agents/user_proxy_agent.py index 5fb577fae..425f0d41d 100644 --- a/dbgpt/agent/agents/user_proxy_agent.py +++ b/dbgpt/agent/agents/user_proxy_agent.py @@ -1,16 +1,9 @@ -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union +from typing import Callable, Dict, List, Literal, Optional, Tuple, Union from ..memory.gpts_memory import GptsMemory from .agent import Agent, AgentContext from .base_agent import ConversableAgent -try: - from termcolor import colored -except ImportError: - - def colored(x, *args, **kwargs): - return x - class UserProxyAgent(ConversableAgent): """(In preview) A proxy agent for the user, that can execute code and provide feedback to the other agents.""" diff --git a/dbgpt/agent/memory/__init__.py b/dbgpt/agent/memory/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/dbgpt/agent/memory/base.py b/dbgpt/agent/memory/base.py index c72c47c76..c8e2c0360 100644 --- a/dbgpt/agent/memory/base.py +++ b/dbgpt/agent/memory/base.py @@ -1,10 +1,10 @@ from __future__ import annotations import dataclasses -from abc import ABC, abstractmethod -from dataclasses import asdict, dataclass, fields +from abc import ABC +from dataclasses import dataclass from datetime import datetime -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union +from typing import Any, Dict, List, Optional from dbgpt.agent.common.schema import Status diff --git a/dbgpt/agent/memory/gpts_memory.py b/dbgpt/agent/memory/gpts_memory.py index b86be2c28..e7cd1857f 100644 --- a/dbgpt/agent/memory/gpts_memory.py +++ b/dbgpt/agent/memory/gpts_memory.py @@ -2,11 +2,11 @@ import json from collections import defaultdict -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union +from typing import Dict, List, Optional from dbgpt.util.json_utils import EnhancedJSONEncoder -from .base import GptsMessage, GptsMessageMemory, GptsPlan, GptsPlansMemory +from .base import GptsMessage, GptsMessageMemory, GptsPlansMemory from .default_gpts_memory import DefaultGptsMessageMemory, DefaultGptsPlansMemory diff --git a/dbgpt/agent/memory/gpts_memory_storage.py b/dbgpt/agent/memory/gpts_memory_storage.py index 031b129d6..3b68b374c 100644 --- a/dbgpt/agent/memory/gpts_memory_storage.py +++ b/dbgpt/agent/memory/gpts_memory_storage.py @@ -1,8 +1,9 @@ import dataclasses -from dataclasses import asdict, dataclass, fields -import json -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, Optional +from dataclasses import dataclass +from datetime import datetime +from typing import Any, Dict, List, Optional + +from dbgpt.agent.common.schema import Status from dbgpt.core.interface.storage import ( InMemoryStorage, QuerySpec, @@ -10,10 +11,8 @@ StorageInterface, StorageItem, ) -from dbgpt.agent.common.schema import Status -from datetime import datetime -from .base import GptsMessageMemory, GptsPlansMemory, GptsPlan, GptsMessage +from .base import GptsMessage, GptsMessageMemory @dataclass @@ -290,6 +289,7 @@ class GptsMessageManager(GptsMessageMemory): Simple wrapper for the storage interface. + TODO: Import gpts storage with storage interface. """ def __init__(self, storage: Optional[StorageInterface[GptsMessage, Any]] = None): diff --git a/dbgpt/agent/plugin/commands/built_in/audio_text.py b/dbgpt/agent/plugin/commands/built_in/audio_text.py index 49fbaceed..f10f940f8 100644 --- a/dbgpt/agent/plugin/commands/built_in/audio_text.py +++ b/dbgpt/agent/plugin/commands/built_in/audio_text.py @@ -4,6 +4,7 @@ import requests from dbgpt._private.config import Config + from ..command_mange import command CFG = Config() diff --git a/dbgpt/agent/plugin/commands/built_in/image_gen.py b/dbgpt/agent/plugin/commands/built_in/image_gen.py index 7893ac1bb..b0db4b8db 100644 --- a/dbgpt/agent/plugin/commands/built_in/image_gen.py +++ b/dbgpt/agent/plugin/commands/built_in/image_gen.py @@ -8,6 +8,7 @@ from PIL import Image from dbgpt._private.config import Config + from ..command_mange import command logger = logging.getLogger(__name__) diff --git a/dbgpt/agent/plugin/commands/command.py b/dbgpt/agent/plugin/commands/command.py index bdcfebaa4..742129b71 100644 --- a/dbgpt/agent/plugin/commands/command.py +++ b/dbgpt/agent/plugin/commands/command.py @@ -5,9 +5,9 @@ from typing import Dict from dbgpt._private.config import Config +from dbgpt.agent.plugin.generator import PluginPromptGenerator from .exception_not_commands import NotCommands -from dbgpt.agent.plugin.generator import PluginPromptGenerator def _resolve_pathlike_command_args(command_args): diff --git a/dbgpt/agent/plugin/commands/command_mange.py b/dbgpt/agent/plugin/commands/command_mange.py index c5b0a4ee0..2ae26a91b 100644 --- a/dbgpt/agent/plugin/commands/command_mange.py +++ b/dbgpt/agent/plugin/commands/command_mange.py @@ -8,12 +8,12 @@ from typing import Any, Callable, List, Optional from dbgpt._private.pydantic import BaseModel - -from .command import execute_command from dbgpt.agent.common.schema import Status from dbgpt.util.json_utils import serialize from dbgpt.util.string_utils import extract_content, extract_content_open_ending +from .command import execute_command + # Unique identifier for auto-gpt commands AUTO_GPT_COMMAND_IDENTIFIER = "auto_gpt_command" logger = logging.getLogger(__name__) diff --git a/dbgpt/agent/plugin/plugin.py b/dbgpt/agent/plugin/plugin.py index bc8bafe89..7e1706583 100644 --- a/dbgpt/agent/plugin/plugin.py +++ b/dbgpt/agent/plugin/plugin.py @@ -1,7 +1,8 @@ import logging -from .generator import PluginPromptGenerator from typing import List +from .generator import PluginPromptGenerator + logger = logging.getLogger(__name__) diff --git a/dbgpt/core/__init__.py b/dbgpt/core/__init__.py index 4be0c1f17..b8c787af2 100644 --- a/dbgpt/core/__init__.py +++ b/dbgpt/core/__init__.py @@ -1,46 +1,45 @@ +from dbgpt.core.interface.cache import ( + CacheClient, + CacheConfig, + CacheKey, + CachePolicy, + CacheValue, +) from dbgpt.core.interface.llm import ( + LLMClient, ModelInferenceMetrics, + ModelMetadata, + ModelOutput, ModelRequest, ModelRequestContext, - ModelOutput, - LLMClient, - ModelMetadata, ) from dbgpt.core.interface.message import ( + ConversationIdentifier, + MessageIdentifier, + MessageStorageItem, ModelMessage, ModelMessageRoleType, OnceConversation, StorageConversation, - MessageStorageItem, - ConversationIdentifier, - MessageIdentifier, ) +from dbgpt.core.interface.output_parser import BaseOutputParser, SQLOutputParser from dbgpt.core.interface.prompt import ( - PromptTemplate, PromptManager, + PromptTemplate, StoragePromptTemplate, ) -from dbgpt.core.interface.output_parser import BaseOutputParser, SQLOutputParser from dbgpt.core.interface.serialization import Serializable, Serializer -from dbgpt.core.interface.cache import ( - CacheKey, - CacheValue, - CacheClient, - CachePolicy, - CacheConfig, -) from dbgpt.core.interface.storage import ( - ResourceIdentifier, - StorageItem, - StorageItemAdapter, - StorageInterface, - InMemoryStorage, DefaultStorageItemAdapter, + InMemoryStorage, QuerySpec, + ResourceIdentifier, StorageError, + StorageInterface, + StorageItem, + StorageItemAdapter, ) - __ALL__ = [ "ModelInferenceMetrics", "ModelRequest", diff --git a/dbgpt/core/_private/example_base.py b/dbgpt/core/_private/example_base.py index 2d2952493..687059b67 100644 --- a/dbgpt/core/_private/example_base.py +++ b/dbgpt/core/_private/example_base.py @@ -1,6 +1,7 @@ from abc import ABC -from typing import List from enum import Enum +from typing import List + from dbgpt._private.pydantic import BaseModel From 6151db82c056873d1c998153001d0402a5ab3f35 Mon Sep 17 00:00:00 2001 From: Fangyin Cheng Date: Thu, 28 Dec 2023 03:26:01 +0800 Subject: [PATCH 3/6] chore: Format a part of the code --- Makefile | 11 +++++++++- dbgpt/core/awel/operator/stream_operator.py | 8 +++++--- .../interface/operator/message_operator.py | 20 +++++++++++++++---- dbgpt/model/adapter/base.py | 9 ++++++--- dbgpt/model/proxy/llms/gemini.py | 2 +- dbgpt/serve/conversation/api/endpoints.py | 14 +++++++------ dbgpt/serve/conversation/api/schemas.py | 1 + dbgpt/serve/conversation/config.py | 3 +-- dbgpt/serve/conversation/models/models.py | 11 ++++++---- dbgpt/serve/conversation/serve.py | 16 ++++++++------- dbgpt/serve/conversation/service/service.py | 10 ++++++---- .../conversation/tests/test_endpoints.py | 10 +++++----- dbgpt/serve/conversation/tests/test_models.py | 7 +++++-- .../serve/conversation/tests/test_service.py | 6 ++++-- dbgpt/serve/core/__init__.py | 4 ++-- dbgpt/serve/core/config.py | 1 + dbgpt/serve/core/schemas.py | 2 +- dbgpt/serve/core/serve.py | 10 ++++++---- dbgpt/serve/core/service.py | 3 ++- dbgpt/serve/core/tests/conftest.py | 3 ++- dbgpt/serve/prompt/api/endpoints.py | 3 ++- dbgpt/serve/prompt/serve.py | 7 ++++++- .../default_serve_template/api/endpoints.py | 14 +++++++------ .../default_serve_template/api/schemas.py | 1 + .../default_serve_template/config.py | 3 +-- .../default_serve_template/models/models.py | 11 ++++++---- .../default_serve_template/serve.py | 13 +++++++----- .../default_serve_template/service/service.py | 10 ++++++---- .../tests/test_endpoints.py | 10 +++++----- .../tests/test_models.py | 7 +++++-- .../tests/test_service.py | 6 ++++-- 31 files changed, 151 insertions(+), 85 deletions(-) diff --git a/Makefile b/Makefile index 132cdc9bf..c66210930 100644 --- a/Makefile +++ b/Makefile @@ -24,11 +24,20 @@ fmt: setup ## Format Python code # TODO: Use isort to sort Python imports. # https://github.com/PyCQA/isort # $(VENV_BIN)/isort . + $(VENV_BIN)/isort dbgpt/agent/ + $(VENV_BIN)/isort dbgpt/core/ + $(VENV_BIN)/isort dbgpt/serve/core/ + $(VENV_BIN)/isort dbgpt/serve/conversation/ + $(VENV_BIN)/isort dbgpt/serve/utils/_template_files # https://github.com/psf/black - $(VENV_BIN)/black . + $(VENV_BIN)/black --extend-exclude="examples/notebook" . # TODO: Use blackdoc to format Python doctests. # https://blackdoc.readthedocs.io/en/latest/ # $(VENV_BIN)/blackdoc . + $(VENV_BIN)/blackdoc dbgpt/core/ + $(VENV_BIN)/blackdoc dbgpt/agent/ + $(VENV_BIN)/blackdoc dbgpt/model/ + $(VENV_BIN)/blackdoc dbgpt/serve/ # TODO: Type checking of Python code. # https://github.com/python/mypy # $(VENV_BIN)/mypy dbgpt diff --git a/dbgpt/core/awel/operator/stream_operator.py b/dbgpt/core/awel/operator/stream_operator.py index 73eb49479..5927a43fc 100644 --- a/dbgpt/core/awel/operator/stream_operator.py +++ b/dbgpt/core/awel/operator/stream_operator.py @@ -27,7 +27,7 @@ async def streamify(self, input_value: IN) -> AsyncIterator[OUT]: .. code-block:: python class MyStreamOperator(StreamifyAbsOperator[int, int]): - async def streamify(self, input_value: int) -> AsyncIterator[int] + async def streamify(self, input_value: int) -> AsyncIterator[int]: for i in range(input_value): yield i """ @@ -54,7 +54,7 @@ async def unstreamify(self, input_value: AsyncIterator[IN]) -> OUT: .. code-block:: python class MyUnstreamOperator(UnstreamifyAbsOperator[int, int]): - async def unstreamify(self, input_value: AsyncIterator[int]) -> int + async def unstreamify(self, input_value: AsyncIterator[int]) -> int: value_cnt = 0 async for v in input_value: value_cnt += 1 @@ -85,7 +85,9 @@ async def transform_stream( .. code-block:: python class MyTransformStreamOperator(TransformStreamAbsOperator[int, int]): - async def unstreamify(self, input_value: AsyncIterator[int]) -> AsyncIterator[int] + async def unstreamify( + self, input_value: AsyncIterator[int] + ) -> AsyncIterator[int]: async for v in input_value: yield v + 1 """ diff --git a/dbgpt/core/interface/operator/message_operator.py b/dbgpt/core/interface/operator/message_operator.py index 8e5b0131d..1a995da2b 100644 --- a/dbgpt/core/interface/operator/message_operator.py +++ b/dbgpt/core/interface/operator/message_operator.py @@ -283,6 +283,7 @@ def map_multi_round_messages( import asyncio from dbgpt.core.operator import ConversationMapperOperator + messages_by_round = [ [ ModelMessage(role="human", content="Hi", round_index=1), @@ -290,7 +291,9 @@ def map_multi_round_messages( ], [ ModelMessage(role="system", content="Error 404", round_index=2), - ModelMessage(role="human", content="What's the error?", round_index=2), + ModelMessage( + role="human", content="What's the error?", round_index=2 + ), ModelMessage(role="ai", content="Just a joke.", round_index=2), ], [ @@ -303,7 +306,9 @@ def map_multi_round_messages( ModelMessage(role="human", content="Hi", round_index=1), ModelMessage(role="ai", content="Hello!", round_index=1), ModelMessage(role="system", content="Error 404", round_index=2), - ModelMessage(role="human", content="What's the error?", round_index=2), + ModelMessage( + role="human", content="What's the error?", round_index=2 + ), ModelMessage(role="ai", content="Just a joke.", round_index=2), ModelMessage(role="human", content="Funny!", round_index=3), ] @@ -315,8 +320,13 @@ def map_multi_round_messages( class MyMapper(ConversationMapperOperator): def __init__(self, **kwargs): super().__init__(**kwargs) - def map_multi_round_messages(self, messages_by_round: List[List[ModelMessage]]) -> List[ModelMessage]: + + def map_multi_round_messages( + self, messages_by_round: List[List[ModelMessage]] + ) -> List[ModelMessage]: return messages_by_round[-1] + + operator = MyMapper() messages = operator.map_multi_round_messages(messages_by_round) assert messages == [ @@ -371,7 +381,9 @@ class BufferedConversationMapperOperator(ConversationMapperOperator): # No history messages = [ModelMessage(role="human", content="Hello", round_index=1)] operator = BufferedConversationMapperOperator(last_k_round=1) - assert operator.map_messages(messages) == [ModelMessage(role="human", content="Hello", round_index=1)] + assert operator.map_messages(messages) == [ + ModelMessage(role="human", content="Hello", round_index=1) + ] Transform with history messages diff --git a/dbgpt/model/adapter/base.py b/dbgpt/model/adapter/base.py index df1d9441a..a456826a2 100644 --- a/dbgpt/model/adapter/base.py +++ b/dbgpt/model/adapter/base.py @@ -158,17 +158,19 @@ def transform_model_messages( Default is the OpenAI format, example: .. code-block:: python + return_messages = [ {"role": "system", "content": "You are a helpful assistant"}, {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi"} + {"role": "assistant", "content": "Hi"}, ] But some model may need to transform the messages to other format(e.g. There is no system message), such as: .. code-block:: python + return_messages = [ {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi"} + {"role": "assistant", "content": "Hi"}, ] Args: messages (List[ModelMessage]): The model messages @@ -191,10 +193,11 @@ def _transform_to_no_system_messages( Merge the system messages to the last user message, example: .. code-block:: python + return_messages = [ {"role": "system", "content": "You are a helpful assistant"}, {"role": "user", "content": "Hello"}, - {"role": "assistant", "content": "Hi"} + {"role": "assistant", "content": "Hi"}, ] => return_messages = [ diff --git a/dbgpt/model/proxy/llms/gemini.py b/dbgpt/model/proxy/llms/gemini.py index 53122ce87..068c19cfb 100644 --- a/dbgpt/model/proxy/llms/gemini.py +++ b/dbgpt/model/proxy/llms/gemini.py @@ -95,7 +95,7 @@ def _transform_to_gemini_messages( assert user_prompt == "How are you?" assert gemini_hist == [ {"role": "user", "parts": {"text": "Hello"}}, - {"role": "model", "parts": {"text": "Hi there!"}} + {"role": "model", "parts": {"text": "Hi there!"}}, ] """ user_prompt, system_messages, history_messages = parse_model_messages(messages) diff --git a/dbgpt/serve/conversation/api/endpoints.py b/dbgpt/serve/conversation/api/endpoints.py index 10398e7e2..8ca494b98 100644 --- a/dbgpt/serve/conversation/api/endpoints.py +++ b/dbgpt/serve/conversation/api/endpoints.py @@ -1,15 +1,16 @@ -from typing import Optional, List from functools import cache -from fastapi import APIRouter, Depends, Query, HTTPException -from fastapi.security.http import HTTPAuthorizationCredentials, HTTPBearer +from typing import List, Optional +from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi.security.http import HTTPAuthorizationCredentials, HTTPBearer from dbgpt.component import SystemApp from dbgpt.serve.core import Result from dbgpt.util import PaginationResult -from .schemas import ServeRequest, ServerResponse + +from ..config import APP_NAME, SERVE_APP_NAME, SERVE_SERVICE_COMPONENT_NAME, ServeConfig from ..service.service import Service -from ..config import APP_NAME, SERVE_APP_NAME, ServeConfig, SERVE_SERVICE_COMPONENT_NAME +from .schemas import ServeRequest, ServerResponse router = APIRouter() @@ -54,8 +55,9 @@ async def check_api_key( .. code-block:: python import requests + client_api_key = "your_api_key" - headers = {"Authorization": "Bearer " + client_api_key } + headers = {"Authorization": "Bearer " + client_api_key} res = requests.get("http://test/hello", headers=headers) assert res.status_code == 200 diff --git a/dbgpt/serve/conversation/api/schemas.py b/dbgpt/serve/conversation/api/schemas.py index 353be0b13..d83c79e15 100644 --- a/dbgpt/serve/conversation/api/schemas.py +++ b/dbgpt/serve/conversation/api/schemas.py @@ -1,5 +1,6 @@ # Define your Pydantic schemas here from dbgpt._private.pydantic import BaseModel, Field + from ..config import SERVE_APP_NAME_HUMP diff --git a/dbgpt/serve/conversation/config.py b/dbgpt/serve/conversation/config.py index 3809846c5..60819389c 100644 --- a/dbgpt/serve/conversation/config.py +++ b/dbgpt/serve/conversation/config.py @@ -1,9 +1,8 @@ -from typing import Optional from dataclasses import dataclass, field +from typing import Optional from dbgpt.serve.core import BaseServeConfig - APP_NAME = "conversation" SERVE_APP_NAME = "dbgpt_serve_conversation" SERVE_APP_NAME_HUMP = "dbgpt_serve_Conversation" diff --git a/dbgpt/serve/conversation/models/models.py b/dbgpt/serve/conversation/models/models.py index 6328bafd6..fb333b051 100644 --- a/dbgpt/serve/conversation/models/models.py +++ b/dbgpt/serve/conversation/models/models.py @@ -1,12 +1,15 @@ """This is an auto-generated model file You can define your own models and DAOs here """ -from typing import Union, Any, Dict from datetime import datetime -from sqlalchemy import Column, Integer, String, Index, Text, DateTime -from dbgpt.storage.metadata import Model, BaseDao, db +from typing import Any, Dict, Union + +from sqlalchemy import Column, DateTime, Index, Integer, String, Text + +from dbgpt.storage.metadata import BaseDao, Model, db + from ..api.schemas import ServeRequest, ServerResponse -from ..config import ServeConfig, SERVER_APP_TABLE_NAME +from ..config import SERVER_APP_TABLE_NAME, ServeConfig class ServeEntity(Model): diff --git a/dbgpt/serve/conversation/serve.py b/dbgpt/serve/conversation/serve.py index 1578dd44e..2dd63ca6a 100644 --- a/dbgpt/serve/conversation/serve.py +++ b/dbgpt/serve/conversation/serve.py @@ -1,15 +1,17 @@ -from typing import List, Optional, Union import logging -from dbgpt.component import SystemApp +from typing import List, Optional, Union + from sqlalchemy import URL + +from dbgpt.component import SystemApp from dbgpt.core import StorageInterface -from dbgpt.storage.metadata import DatabaseManager from dbgpt.serve.core import BaseServe +from dbgpt.storage.metadata import DatabaseManager from .config import ( + APP_NAME, SERVE_APP_NAME, SERVE_APP_NAME_HUMP, - APP_NAME, SERVE_CONFIG_KEY_PREFIX, ServeConfig, ) @@ -72,16 +74,16 @@ def on_init(self): def before_start(self): """Called before the start of the application.""" # TODO: Your code here - from dbgpt.storage.metadata.db_storage import SQLAlchemyStorage - from dbgpt.util.serialization.json_serialization import JsonSerializer from dbgpt.storage.chat_history.chat_history_db import ( ChatHistoryEntity, ChatHistoryMessageEntity, ) from dbgpt.storage.chat_history.storage_adapter import ( - DBStorageConversationItemAdapter, DBMessageStorageItemAdapter, + DBStorageConversationItemAdapter, ) + from dbgpt.storage.metadata.db_storage import SQLAlchemyStorage + from dbgpt.util.serialization.json_serialization import JsonSerializer self._db_manager = self.create_or_get_db_manager() diff --git a/dbgpt/serve/conversation/service/service.py b/dbgpt/serve/conversation/service/service.py index d346822f6..71592bd13 100644 --- a/dbgpt/serve/conversation/service/service.py +++ b/dbgpt/serve/conversation/service/service.py @@ -1,11 +1,13 @@ -from typing import Optional, List +from typing import List, Optional + from dbgpt.component import BaseComponent, SystemApp +from dbgpt.serve.core import BaseService from dbgpt.storage.metadata import BaseDao from dbgpt.util.pagination_utils import PaginationResult -from dbgpt.serve.core import BaseService -from ..models.models import ServeDao, ServeEntity + from ..api.schemas import ServeRequest, ServerResponse -from ..config import SERVE_SERVICE_COMPONENT_NAME, SERVE_CONFIG_KEY_PREFIX, ServeConfig +from ..config import SERVE_CONFIG_KEY_PREFIX, SERVE_SERVICE_COMPONENT_NAME, ServeConfig +from ..models.models import ServeDao, ServeEntity class Service(BaseService[ServeEntity, ServeRequest, ServerResponse]): diff --git a/dbgpt/serve/conversation/tests/test_endpoints.py b/dbgpt/serve/conversation/tests/test_endpoints.py index 79f35c7a0..ba7b4f0cd 100644 --- a/dbgpt/serve/conversation/tests/test_endpoints.py +++ b/dbgpt/serve/conversation/tests/test_endpoints.py @@ -1,15 +1,15 @@ import pytest +from fastapi import FastAPI from httpx import AsyncClient -from fastapi import FastAPI from dbgpt.component import SystemApp +from dbgpt.serve.core.tests.conftest import asystem_app, client from dbgpt.storage.metadata import db from dbgpt.util import PaginationResult -from ..config import SERVE_CONFIG_KEY_PREFIX -from ..api.endpoints import router, init_endpoints -from ..api.schemas import ServeRequest, ServerResponse -from dbgpt.serve.core.tests.conftest import client, asystem_app +from ..api.endpoints import init_endpoints, router +from ..api.schemas import ServeRequest, ServerResponse +from ..config import SERVE_CONFIG_KEY_PREFIX @pytest.fixture(autouse=True) diff --git a/dbgpt/serve/conversation/tests/test_models.py b/dbgpt/serve/conversation/tests/test_models.py index c065909b2..218307f1a 100644 --- a/dbgpt/serve/conversation/tests/test_models.py +++ b/dbgpt/serve/conversation/tests/test_models.py @@ -1,9 +1,12 @@ from typing import List + import pytest + from dbgpt.storage.metadata import db -from ..config import ServeConfig + from ..api.schemas import ServeRequest, ServerResponse -from ..models.models import ServeEntity, ServeDao +from ..config import ServeConfig +from ..models.models import ServeDao, ServeEntity @pytest.fixture(autouse=True) diff --git a/dbgpt/serve/conversation/tests/test_service.py b/dbgpt/serve/conversation/tests/test_service.py index 003286c82..00177924d 100644 --- a/dbgpt/serve/conversation/tests/test_service.py +++ b/dbgpt/serve/conversation/tests/test_service.py @@ -1,11 +1,13 @@ from typing import List + import pytest + from dbgpt.component import SystemApp -from dbgpt.storage.metadata import db from dbgpt.serve.core.tests.conftest import system_app +from dbgpt.storage.metadata import db -from ..models.models import ServeEntity from ..api.schemas import ServeRequest, ServerResponse +from ..models.models import ServeEntity from ..service.service import Service diff --git a/dbgpt/serve/core/__init__.py b/dbgpt/serve/core/__init__.py index 23275525c..89c596180 100644 --- a/dbgpt/serve/core/__init__.py +++ b/dbgpt/serve/core/__init__.py @@ -1,6 +1,6 @@ -from dbgpt.serve.core.schemas import Result from dbgpt.serve.core.config import BaseServeConfig -from dbgpt.serve.core.service import BaseService +from dbgpt.serve.core.schemas import Result from dbgpt.serve.core.serve import BaseServe +from dbgpt.serve.core.service import BaseService __ALL__ = ["Result", "BaseServeConfig", "BaseService", "BaseServe"] diff --git a/dbgpt/serve/core/config.py b/dbgpt/serve/core/config.py index 4fd6d2247..9b78c44a2 100644 --- a/dbgpt/serve/core/config.py +++ b/dbgpt/serve/core/config.py @@ -1,4 +1,5 @@ from dataclasses import dataclass + from dbgpt.component import AppConfig from dbgpt.util import BaseParameters diff --git a/dbgpt/serve/core/schemas.py b/dbgpt/serve/core/schemas.py index b97a936d5..23e8b56c6 100644 --- a/dbgpt/serve/core/schemas.py +++ b/dbgpt/serve/core/schemas.py @@ -1,4 +1,4 @@ -from typing import TypeVar, Generic, Any, Optional +from typing import Any, Generic, Optional, TypeVar from dbgpt._private.pydantic import BaseModel, Field diff --git a/dbgpt/serve/core/serve.py b/dbgpt/serve/core/serve.py index 4d3983f16..b2bcfdb1f 100644 --- a/dbgpt/serve/core/serve.py +++ b/dbgpt/serve/core/serve.py @@ -1,8 +1,10 @@ -from abc import ABC -from typing import Optional, Union, List import logging -from dbgpt.component import BaseComponent, SystemApp, ComponentType +from abc import ABC +from typing import List, Optional, Union + from sqlalchemy import URL + +from dbgpt.component import BaseComponent, ComponentType, SystemApp from dbgpt.storage.metadata import DatabaseManager logger = logging.getLogger(__name__) @@ -36,7 +38,7 @@ def create_or_get_db_manager(self) -> DatabaseManager: Returns: DatabaseManager: The database manager """ - from dbgpt.storage.metadata import Model, db, UnifiedDBManagerFactory + from dbgpt.storage.metadata import Model, UnifiedDBManagerFactory, db # If you need to use the database, you can get the database manager here db_manager_factory: UnifiedDBManagerFactory = self._system_app.get_component( diff --git a/dbgpt/serve/core/service.py b/dbgpt/serve/core/service.py index 401b93517..bf17a45e4 100644 --- a/dbgpt/serve/core/service.py +++ b/dbgpt/serve/core/service.py @@ -1,8 +1,9 @@ from abc import ABC, abstractmethod from typing import Generic + from dbgpt.component import BaseComponent -from dbgpt.storage.metadata._base_dao import BaseDao, T, REQ, RES from dbgpt.serve.core.config import BaseServeConfig +from dbgpt.storage.metadata._base_dao import REQ, RES, BaseDao, T class BaseService(BaseComponent, Generic[T, REQ, RES], ABC): diff --git a/dbgpt/serve/core/tests/conftest.py b/dbgpt/serve/core/tests/conftest.py index 090f3bf95..5091d2ed2 100644 --- a/dbgpt/serve/core/tests/conftest.py +++ b/dbgpt/serve/core/tests/conftest.py @@ -1,6 +1,7 @@ +from typing import Dict + import pytest import pytest_asyncio -from typing import Dict from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from httpx import AsyncClient diff --git a/dbgpt/serve/prompt/api/endpoints.py b/dbgpt/serve/prompt/api/endpoints.py index c043858fc..f78d0843c 100644 --- a/dbgpt/serve/prompt/api/endpoints.py +++ b/dbgpt/serve/prompt/api/endpoints.py @@ -55,8 +55,9 @@ async def check_api_key( .. code-block:: python import requests + client_api_key = "your_api_key" - headers = {"Authorization": "Bearer " + client_api_key } + headers = {"Authorization": "Bearer " + client_api_key} res = requests.get("http://test/hello", headers=headers) assert res.status_code == 200 diff --git a/dbgpt/serve/prompt/serve.py b/dbgpt/serve/prompt/serve.py index ed50a353f..0fc6dd3eb 100644 --- a/dbgpt/serve/prompt/serve.py +++ b/dbgpt/serve/prompt/serve.py @@ -62,7 +62,12 @@ class Serve(BaseServe): app = FastAPI() system_app = SystemApp(app) - system_app.register(Serve, api_prefix="/api/v1/prompt", db_url_or_db="sqlite:///:memory:", try_create_tables=True) + system_app.register( + Serve, + api_prefix="/api/v1/prompt", + db_url_or_db="sqlite:///:memory:", + try_create_tables=True, + ) system_app.on_init() # Run before start hook system_app.before_start() diff --git a/dbgpt/serve/utils/_template_files/default_serve_template/api/endpoints.py b/dbgpt/serve/utils/_template_files/default_serve_template/api/endpoints.py index 198798fa5..cc8caf86c 100644 --- a/dbgpt/serve/utils/_template_files/default_serve_template/api/endpoints.py +++ b/dbgpt/serve/utils/_template_files/default_serve_template/api/endpoints.py @@ -1,15 +1,16 @@ -from typing import Optional, List from functools import cache -from fastapi import APIRouter, Depends, Query, HTTPException -from fastapi.security.http import HTTPAuthorizationCredentials, HTTPBearer +from typing import List, Optional +from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi.security.http import HTTPAuthorizationCredentials, HTTPBearer from dbgpt.component import SystemApp from dbgpt.serve.core import Result from dbgpt.util import PaginationResult -from .schemas import ServeRequest, ServerResponse + +from ..config import APP_NAME, SERVE_APP_NAME, SERVE_SERVICE_COMPONENT_NAME, ServeConfig from ..service.service import Service -from ..config import APP_NAME, SERVE_APP_NAME, ServeConfig, SERVE_SERVICE_COMPONENT_NAME +from .schemas import ServeRequest, ServerResponse router = APIRouter() @@ -54,8 +55,9 @@ async def check_api_key( .. code-block:: python import requests + client_api_key = "your_api_key" - headers = {"Authorization": "Bearer " + client_api_key } + headers = {"Authorization": "Bearer " + client_api_key} res = requests.get("http://test/hello", headers=headers) assert res.status_code == 200 diff --git a/dbgpt/serve/utils/_template_files/default_serve_template/api/schemas.py b/dbgpt/serve/utils/_template_files/default_serve_template/api/schemas.py index 548e78764..d84369475 100644 --- a/dbgpt/serve/utils/_template_files/default_serve_template/api/schemas.py +++ b/dbgpt/serve/utils/_template_files/default_serve_template/api/schemas.py @@ -1,5 +1,6 @@ # Define your Pydantic schemas here from dbgpt._private.pydantic import BaseModel, Field + from ..config import SERVE_APP_NAME_HUMP diff --git a/dbgpt/serve/utils/_template_files/default_serve_template/config.py b/dbgpt/serve/utils/_template_files/default_serve_template/config.py index 271546963..a2714be9e 100644 --- a/dbgpt/serve/utils/_template_files/default_serve_template/config.py +++ b/dbgpt/serve/utils/_template_files/default_serve_template/config.py @@ -1,9 +1,8 @@ -from typing import Optional from dataclasses import dataclass, field +from typing import Optional from dbgpt.serve.core import BaseServeConfig - APP_NAME = "{__template_app_name__all_lower__}" SERVE_APP_NAME = "dbgpt_serve_{__template_app_name__all_lower__}" SERVE_APP_NAME_HUMP = "dbgpt_serve_{__template_app_name__hump__}" diff --git a/dbgpt/serve/utils/_template_files/default_serve_template/models/models.py b/dbgpt/serve/utils/_template_files/default_serve_template/models/models.py index 516635df5..039ed289d 100644 --- a/dbgpt/serve/utils/_template_files/default_serve_template/models/models.py +++ b/dbgpt/serve/utils/_template_files/default_serve_template/models/models.py @@ -1,12 +1,15 @@ """This is an auto-generated model file You can define your own models and DAOs here """ -from typing import Union, Any, Dict from datetime import datetime -from sqlalchemy import Column, Integer, String, Index, Text, DateTime -from dbgpt.storage.metadata import Model, BaseDao, db +from typing import Any, Dict, Union + +from sqlalchemy import Column, DateTime, Index, Integer, String, Text + +from dbgpt.storage.metadata import BaseDao, Model, db + from ..api.schemas import ServeRequest, ServerResponse -from ..config import ServeConfig, SERVER_APP_TABLE_NAME +from ..config import SERVER_APP_TABLE_NAME, ServeConfig class ServeEntity(Model): diff --git a/dbgpt/serve/utils/_template_files/default_serve_template/serve.py b/dbgpt/serve/utils/_template_files/default_serve_template/serve.py index 77b9e477b..8c6564d3a 100644 --- a/dbgpt/serve/utils/_template_files/default_serve_template/serve.py +++ b/dbgpt/serve/utils/_template_files/default_serve_template/serve.py @@ -1,14 +1,17 @@ -from typing import List, Optional, Union import logging -from dbgpt.component import SystemApp +from typing import List, Optional, Union + from sqlalchemy import URL -from dbgpt.storage.metadata import DatabaseManager + +from dbgpt.component import SystemApp from dbgpt.serve.core import BaseServe -from .api.endpoints import router, init_endpoints +from dbgpt.storage.metadata import DatabaseManager + +from .api.endpoints import init_endpoints, router from .config import ( + APP_NAME, SERVE_APP_NAME, SERVE_APP_NAME_HUMP, - APP_NAME, SERVE_CONFIG_KEY_PREFIX, ServeConfig, ) diff --git a/dbgpt/serve/utils/_template_files/default_serve_template/service/service.py b/dbgpt/serve/utils/_template_files/default_serve_template/service/service.py index 0e2767023..f6dfd6a02 100644 --- a/dbgpt/serve/utils/_template_files/default_serve_template/service/service.py +++ b/dbgpt/serve/utils/_template_files/default_serve_template/service/service.py @@ -1,11 +1,13 @@ -from typing import Optional, List +from typing import List, Optional + from dbgpt.component import BaseComponent, SystemApp +from dbgpt.serve.core import BaseService from dbgpt.storage.metadata import BaseDao from dbgpt.util.pagination_utils import PaginationResult -from dbgpt.serve.core import BaseService -from ..models.models import ServeDao, ServeEntity + from ..api.schemas import ServeRequest, ServerResponse -from ..config import SERVE_SERVICE_COMPONENT_NAME, SERVE_CONFIG_KEY_PREFIX, ServeConfig +from ..config import SERVE_CONFIG_KEY_PREFIX, SERVE_SERVICE_COMPONENT_NAME, ServeConfig +from ..models.models import ServeDao, ServeEntity class Service(BaseService[ServeEntity, ServeRequest, ServerResponse]): diff --git a/dbgpt/serve/utils/_template_files/default_serve_template/tests/test_endpoints.py b/dbgpt/serve/utils/_template_files/default_serve_template/tests/test_endpoints.py index 79f35c7a0..ba7b4f0cd 100644 --- a/dbgpt/serve/utils/_template_files/default_serve_template/tests/test_endpoints.py +++ b/dbgpt/serve/utils/_template_files/default_serve_template/tests/test_endpoints.py @@ -1,15 +1,15 @@ import pytest +from fastapi import FastAPI from httpx import AsyncClient -from fastapi import FastAPI from dbgpt.component import SystemApp +from dbgpt.serve.core.tests.conftest import asystem_app, client from dbgpt.storage.metadata import db from dbgpt.util import PaginationResult -from ..config import SERVE_CONFIG_KEY_PREFIX -from ..api.endpoints import router, init_endpoints -from ..api.schemas import ServeRequest, ServerResponse -from dbgpt.serve.core.tests.conftest import client, asystem_app +from ..api.endpoints import init_endpoints, router +from ..api.schemas import ServeRequest, ServerResponse +from ..config import SERVE_CONFIG_KEY_PREFIX @pytest.fixture(autouse=True) diff --git a/dbgpt/serve/utils/_template_files/default_serve_template/tests/test_models.py b/dbgpt/serve/utils/_template_files/default_serve_template/tests/test_models.py index c065909b2..218307f1a 100644 --- a/dbgpt/serve/utils/_template_files/default_serve_template/tests/test_models.py +++ b/dbgpt/serve/utils/_template_files/default_serve_template/tests/test_models.py @@ -1,9 +1,12 @@ from typing import List + import pytest + from dbgpt.storage.metadata import db -from ..config import ServeConfig + from ..api.schemas import ServeRequest, ServerResponse -from ..models.models import ServeEntity, ServeDao +from ..config import ServeConfig +from ..models.models import ServeDao, ServeEntity @pytest.fixture(autouse=True) diff --git a/dbgpt/serve/utils/_template_files/default_serve_template/tests/test_service.py b/dbgpt/serve/utils/_template_files/default_serve_template/tests/test_service.py index 003286c82..00177924d 100644 --- a/dbgpt/serve/utils/_template_files/default_serve_template/tests/test_service.py +++ b/dbgpt/serve/utils/_template_files/default_serve_template/tests/test_service.py @@ -1,11 +1,13 @@ from typing import List + import pytest + from dbgpt.component import SystemApp -from dbgpt.storage.metadata import db from dbgpt.serve.core.tests.conftest import system_app +from dbgpt.storage.metadata import db -from ..models.models import ServeEntity from ..api.schemas import ServeRequest, ServerResponse +from ..models.models import ServeEntity from ..service.service import Service From a71e120937df4fffa5ac063a6bc744bff315d87f Mon Sep 17 00:00:00 2001 From: Fangyin Cheng Date: Thu, 28 Dec 2023 03:32:53 +0800 Subject: [PATCH 4/6] chore: Format a part of agent serve code --- Makefile | 1 + dbgpt/serve/agent/agents/controller.py | 56 ++++++------------- dbgpt/serve/agent/agents/db_gpts_memory.py | 9 +-- dbgpt/serve/agent/agents/dbgpts.py | 7 +-- dbgpt/serve/agent/db/gpts_conversations_db.py | 14 ++++- dbgpt/serve/agent/db/gpts_mange_db.py | 5 +- dbgpt/serve/agent/db/gpts_messages_db.py | 10 ++-- dbgpt/serve/agent/db/gpts_plans_db.py | 7 +-- dbgpt/serve/agent/db/my_plugin_db.py | 4 +- dbgpt/serve/agent/db/plugin_hub_db.py | 13 ++++- dbgpt/serve/agent/hub/agent_hub.py | 11 ++-- dbgpt/serve/agent/hub/controller.py | 27 +++------ dbgpt/serve/agent/model.py | 4 +- 13 files changed, 77 insertions(+), 91 deletions(-) diff --git a/Makefile b/Makefile index c66210930..5d0e52ae7 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ fmt: setup ## Format Python code $(VENV_BIN)/isort dbgpt/agent/ $(VENV_BIN)/isort dbgpt/core/ $(VENV_BIN)/isort dbgpt/serve/core/ + $(VENV_BIN)/isort dbgpt/serve/agent/ $(VENV_BIN)/isort dbgpt/serve/conversation/ $(VENV_BIN)/isort dbgpt/serve/utils/_template_files # https://github.com/psf/black diff --git a/dbgpt/serve/agent/agents/controller.py b/dbgpt/serve/agent/agents/controller.py index a6bc7d706..c0fe44d47 100644 --- a/dbgpt/serve/agent/agents/controller.py +++ b/dbgpt/serve/agent/agents/controller.py @@ -1,52 +1,30 @@ -import logging import json -import asyncio +import logging import uuid +from abc import ABC from collections import defaultdict -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union -from fastapi import ( - APIRouter, - Body, - UploadFile, - File, -) -from fastapi.responses import StreamingResponse -from abc import ABC -from typing import List -from dbgpt.core.awel import BaseOperator, SimpleCallDataInputSource, InputOperator, DAG -from dbgpt.model.operator.model_operator import ModelOperator, ModelStreamOperator -from dbgpt.app.openapi.api_view_model import Result, ConversationVo -from dbgpt.util.json_utils import EnhancedJSONEncoder -from dbgpt.serve.agent.model import ( - PluginHubParam, - PagenationFilter, - PagenationResult, - PluginHubFilter, -) +from fastapi import APIRouter, Body +from fastapi.responses import StreamingResponse -from dbgpt.agent.common.schema import Status -from dbgpt.agent.agents.agents_mange import AgentsMange +from dbgpt._private.config import Config +from dbgpt.agent.agents.agent import AgentContext +from dbgpt.agent.agents.agents_mange import agent_mange +from dbgpt.agent.agents.plan_group_chat import PlanChat, PlanChatManager from dbgpt.agent.agents.planner_agent import PlannerAgent from dbgpt.agent.agents.user_proxy_agent import UserProxyAgent -from dbgpt.agent.agents.plan_group_chat import PlanChat, PlanChatManager -from dbgpt.agent.agents.agent import AgentContext +from dbgpt.agent.common.schema import Status from dbgpt.agent.memory.gpts_memory import GptsMemory - -from .db_gpts_memory import MetaDbGptsPlansMemory, MetaDbGptsMessageMemory - -from ..db.gpts_mange_db import GptsInstanceDao, GptsInstanceEntity -from ..db.gpts_conversations_db import GptsConversationsDao, GptsConversationsEntity - -from .dbgpts import DbGptsCompletion, DbGptsTaskStep, DbGptsMessage, DbGptsInstance +from dbgpt.app.openapi.api_view_model import Result from dbgpt.component import BaseComponent, ComponentType, SystemApp -from dbgpt.agent.agents.agents_mange import agent_mange -from dbgpt._private.config import Config -from dbgpt.model.cluster.controller.controller import BaseModelController -from dbgpt.agent.memory.gpts_memory import GptsMessage -from dbgpt.model.cluster import WorkerManager, WorkerManagerFactory - +from dbgpt.model.cluster import WorkerManagerFactory from dbgpt.model.cluster.client import DefaultLLMClient +from dbgpt.serve.agent.model import PagenationFilter, PluginHubFilter + +from ..db.gpts_conversations_db import GptsConversationsDao, GptsConversationsEntity +from ..db.gpts_mange_db import GptsInstanceDao, GptsInstanceEntity +from .db_gpts_memory import MetaDbGptsMessageMemory, MetaDbGptsPlansMemory +from .dbgpts import DbGptsInstance CFG = Config() diff --git a/dbgpt/serve/agent/agents/db_gpts_memory.py b/dbgpt/serve/agent/agents/db_gpts_memory.py index ba6d9a6a8..385439920 100644 --- a/dbgpt/serve/agent/agents/db_gpts_memory.py +++ b/dbgpt/serve/agent/agents/db_gpts_memory.py @@ -1,13 +1,14 @@ from typing import List, Optional + +from dbgpt.agent.memory.base import GptsPlan from dbgpt.agent.memory.gpts_memory import ( - GptsPlansMemory, - GptsPlan, - GptsMessageMemory, GptsMessage, + GptsMessageMemory, + GptsPlansMemory, ) -from ..db.gpts_plans_db import GptsPlansEntity, GptsPlansDao from ..db.gpts_messages_db import GptsMessagesDao, GptsMessagesEntity +from ..db.gpts_plans_db import GptsPlansDao, GptsPlansEntity class MetaDbGptsPlansMemory(GptsPlansMemory): diff --git a/dbgpt/serve/agent/agents/dbgpts.py b/dbgpt/serve/agent/agents/dbgpts.py index 37173fb73..f0fdbb638 100644 --- a/dbgpt/serve/agent/agents/dbgpts.py +++ b/dbgpt/serve/agent/agents/dbgpts.py @@ -1,10 +1,9 @@ from __future__ import annotations -from enum import Enum -from abc import ABC, abstractmethod -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, Union -from dataclasses import dataclass, asdict, fields import dataclasses +from dataclasses import dataclass +from enum import Enum +from typing import Any, Dict, List, Optional from dbgpt.agent.agents.agent import AgentResource diff --git a/dbgpt/serve/agent/db/gpts_conversations_db.py b/dbgpt/serve/agent/db/gpts_conversations_db.py index 92ecb76c2..69b8d0c5d 100644 --- a/dbgpt/serve/agent/db/gpts_conversations_db.py +++ b/dbgpt/serve/agent/db/gpts_conversations_db.py @@ -1,8 +1,16 @@ from datetime import datetime -from typing import List -from sqlalchemy import Column, Integer, String, Index, DateTime, func, Text, desc -from sqlalchemy import UniqueConstraint +from sqlalchemy import ( + Column, + DateTime, + Index, + Integer, + String, + Text, + UniqueConstraint, + desc, + func, +) from dbgpt.storage.metadata import BaseDao, Model diff --git a/dbgpt/serve/agent/db/gpts_mange_db.py b/dbgpt/serve/agent/db/gpts_mange_db.py index 3db062b94..81a474ac0 100644 --- a/dbgpt/serve/agent/db/gpts_mange_db.py +++ b/dbgpt/serve/agent/db/gpts_mange_db.py @@ -1,7 +1,6 @@ from datetime import datetime -from typing import List -from sqlalchemy import Column, Integer, String, Index, DateTime, func, Text, Boolean -from sqlalchemy import UniqueConstraint + +from sqlalchemy import Column, DateTime, Integer, String, Text, UniqueConstraint from dbgpt.storage.metadata import BaseDao, Model diff --git a/dbgpt/serve/agent/db/gpts_messages_db.py b/dbgpt/serve/agent/db/gpts_messages_db.py index 26e87abfc..54f1bddef 100644 --- a/dbgpt/serve/agent/db/gpts_messages_db.py +++ b/dbgpt/serve/agent/db/gpts_messages_db.py @@ -1,18 +1,18 @@ from datetime import datetime from typing import List, Optional + from sqlalchemy import ( Column, + DateTime, + Index, Integer, String, - Index, - DateTime, - func, Text, - or_, and_, desc, + func, + or_, ) -from sqlalchemy import UniqueConstraint from dbgpt.storage.metadata import BaseDao, Model diff --git a/dbgpt/serve/agent/db/gpts_plans_db.py b/dbgpt/serve/agent/db/gpts_plans_db.py index d0c2bce9d..57fe94197 100644 --- a/dbgpt/serve/agent/db/gpts_plans_db.py +++ b/dbgpt/serve/agent/db/gpts_plans_db.py @@ -1,10 +1,9 @@ from datetime import datetime -from typing import List -from sqlalchemy import Column, Integer, String, Index, DateTime, func, Text -from sqlalchemy import UniqueConstraint -from dbgpt.storage.metadata import BaseDao, Model +from sqlalchemy import Column, DateTime, Integer, String, Text, UniqueConstraint + from dbgpt.agent.common.schema import Status +from dbgpt.storage.metadata import BaseDao, Model class GptsPlansEntity(Model): diff --git a/dbgpt/serve/agent/db/my_plugin_db.py b/dbgpt/serve/agent/db/my_plugin_db.py index 0927f4283..bd202dab0 100644 --- a/dbgpt/serve/agent/db/my_plugin_db.py +++ b/dbgpt/serve/agent/db/my_plugin_db.py @@ -1,6 +1,6 @@ from datetime import datetime -from sqlalchemy import Column, Integer, String, DateTime, func -from sqlalchemy import UniqueConstraint + +from sqlalchemy import Column, DateTime, Integer, String, UniqueConstraint, func from dbgpt.storage.metadata import BaseDao, Model diff --git a/dbgpt/serve/agent/db/plugin_hub_db.py b/dbgpt/serve/agent/db/plugin_hub_db.py index e1bcfffcf..8230682ab 100644 --- a/dbgpt/serve/agent/db/plugin_hub_db.py +++ b/dbgpt/serve/agent/db/plugin_hub_db.py @@ -1,7 +1,16 @@ from datetime import datetime + import pytz -from sqlalchemy import Column, Integer, String, Index, DateTime, func, DDL -from sqlalchemy import UniqueConstraint +from sqlalchemy import ( + DDL, + Column, + DateTime, + Index, + Integer, + String, + UniqueConstraint, + func, +) from dbgpt.storage.metadata import BaseDao, Model diff --git a/dbgpt/serve/agent/hub/agent_hub.py b/dbgpt/serve/agent/hub/agent_hub.py index a075d6625..689c9ff66 100644 --- a/dbgpt/serve/agent/hub/agent_hub.py +++ b/dbgpt/serve/agent/hub/agent_hub.py @@ -1,18 +1,19 @@ +import glob import json import logging import os -import glob import shutil -from fastapi import UploadFile -from typing import Any import tempfile +from typing import Any -from ..db.plugin_hub_db import PluginHubEntity, PluginHubDao -from ..db.my_plugin_db import MyPluginDao, MyPluginEntity +from fastapi import UploadFile from dbgpt.agent.common.schema import PluginStorageType from dbgpt.agent.plugin.plugins_util import scan_plugins, update_from_git +from ..db.my_plugin_db import MyPluginDao, MyPluginEntity +from ..db.plugin_hub_db import PluginHubDao, PluginHubEntity + logger = logging.getLogger(__name__) Default_User = "default" DEFAULT_PLUGIN_REPO = "https://github.com/eosphoros-ai/DB-GPT-Plugins.git" diff --git a/dbgpt/serve/agent/hub/controller.py b/dbgpt/serve/agent/hub/controller.py index ed5d2ed05..b4a9a5dad 100644 --- a/dbgpt/serve/agent/hub/controller.py +++ b/dbgpt/serve/agent/hub/controller.py @@ -1,31 +1,22 @@ import logging -from fastapi import ( - APIRouter, - Body, - UploadFile, - File, -) from abc import ABC from typing import List -from dbgpt.app.openapi.api_view_model import ( - Result, -) +from fastapi import APIRouter, Body, File, UploadFile +from dbgpt.agent.plugin.generator import PluginPromptGenerator +from dbgpt.agent.plugin.plugins_util import scan_plugins +from dbgpt.app.openapi.api_view_model import Result +from dbgpt.component import BaseComponent, ComponentType, SystemApp +from dbgpt.configs.model_config import PLUGINS_DIR +from dbgpt.serve.agent.db.plugin_hub_db import PluginHubEntity +from dbgpt.serve.agent.hub.agent_hub import AgentHub from dbgpt.serve.agent.model import ( - PluginHubParam, PagenationFilter, PagenationResult, PluginHubFilter, + PluginHubParam, ) -from dbgpt.serve.agent.hub.agent_hub import AgentHub -from dbgpt.serve.agent.db.plugin_hub_db import PluginHubEntity - -from dbgpt.agent.plugin.plugins_util import scan_plugins -from dbgpt.agent.plugin.generator import PluginPromptGenerator - -from dbgpt.configs.model_config import PLUGINS_DIR -from dbgpt.component import BaseComponent, ComponentType, SystemApp router = APIRouter() logger = logging.getLogger(__name__) diff --git a/dbgpt/serve/agent/model.py b/dbgpt/serve/agent/model.py index 3f8ff7c20..0e86aa0a0 100644 --- a/dbgpt/serve/agent/model.py +++ b/dbgpt/serve/agent/model.py @@ -1,6 +1,6 @@ -from typing import TypedDict, Optional, Dict, List from dataclasses import dataclass -from typing import TypeVar, Generic, Any +from typing import Any, Dict, Generic, List, Optional, TypedDict, TypeVar + from dbgpt._private.pydantic import BaseModel, Field T = TypeVar("T") From 44d71090391605b4465b06f8d3629ae1dceae01c Mon Sep 17 00:00:00 2001 From: Fangyin Cheng Date: Thu, 28 Dec 2023 09:40:38 +0800 Subject: [PATCH 5/6] chore: Format a part of example --- Makefile | 2 ++ .../auto_plan_agent_dialogue_example.py | 16 ++++++-------- .../agents/single_agent_dialogue_example.py | 8 +++---- examples/awel/simple_chat_dag_example.py | 3 ++- examples/awel/simple_chat_history_example.py | 22 ++++++++----------- examples/awel/simple_dag_example.py | 1 - examples/awel/simple_llm_client_example.py | 15 +++++-------- examples/awel/simple_rag_example.py | 12 +++++----- examples/sdk/simple_sdk_llm_example.py | 11 +++------- examples/sdk/simple_sdk_llm_sql_example.py | 19 ++++++---------- 10 files changed, 45 insertions(+), 64 deletions(-) diff --git a/Makefile b/Makefile index 5d0e52ae7..420eafc0d 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ fmt: setup ## Format Python code $(VENV_BIN)/isort dbgpt/serve/agent/ $(VENV_BIN)/isort dbgpt/serve/conversation/ $(VENV_BIN)/isort dbgpt/serve/utils/_template_files + $(VENV_BIN)/isort --extend-skip="examples/notebook" examples # https://github.com/psf/black $(VENV_BIN)/black --extend-exclude="examples/notebook" . # TODO: Use blackdoc to format Python doctests. @@ -39,6 +40,7 @@ fmt: setup ## Format Python code $(VENV_BIN)/blackdoc dbgpt/agent/ $(VENV_BIN)/blackdoc dbgpt/model/ $(VENV_BIN)/blackdoc dbgpt/serve/ + $(VENV_BIN)/blackdoc examples # TODO: Type checking of Python code. # https://github.com/python/mypy # $(VENV_BIN)/mypy dbgpt diff --git a/examples/agents/auto_plan_agent_dialogue_example.py b/examples/agents/auto_plan_agent_dialogue_example.py index 1f968db45..574f1fb02 100644 --- a/examples/agents/auto_plan_agent_dialogue_example.py +++ b/examples/agents/auto_plan_agent_dialogue_example.py @@ -15,21 +15,19 @@ """ -from dbgpt.agent.agents.planner_agent import PlannerAgent -from dbgpt.agent.agents.user_proxy_agent import UserProxyAgent -from dbgpt.agent.agents.plan_group_chat import PlanChat, PlanChatManager +import asyncio +import os +from dbgpt.agent.agents.agent import AgentContext +from dbgpt.agent.agents.agents_mange import agent_mange from dbgpt.agent.agents.expand.code_assistant_agent import CodeAssistantAgent from dbgpt.agent.agents.expand.plugin_assistant_agent import PluginAgent -from dbgpt.agent.agents.agents_mange import agent_mange - -from dbgpt.agent.agents.agent import AgentContext +from dbgpt.agent.agents.plan_group_chat import PlanChat, PlanChatManager +from dbgpt.agent.agents.planner_agent import PlannerAgent +from dbgpt.agent.agents.user_proxy_agent import UserProxyAgent from dbgpt.agent.memory.gpts_memory import GptsMemory from dbgpt.core.interface.llm import ModelMetadata -import asyncio -import os - if __name__ == "__main__": from dbgpt.model import OpenAILLMClient diff --git a/examples/agents/single_agent_dialogue_example.py b/examples/agents/single_agent_dialogue_example.py index 5fb32da1e..3b4d68ebe 100644 --- a/examples/agents/single_agent_dialogue_example.py +++ b/examples/agents/single_agent_dialogue_example.py @@ -14,14 +14,14 @@ python examples/agents/single_agent_dialogue_example.py """ +import asyncio +import os + +from dbgpt.agent.agents.agent import AgentContext from dbgpt.agent.agents.expand.code_assistant_agent import CodeAssistantAgent from dbgpt.agent.agents.user_proxy_agent import UserProxyAgent from dbgpt.agent.memory.gpts_memory import GptsMemory -from dbgpt.agent.agents.agent import AgentContext from dbgpt.core.interface.llm import ModelMetadata -import asyncio -import os - if __name__ == "__main__": from dbgpt.model import OpenAILLMClient diff --git a/examples/awel/simple_chat_dag_example.py b/examples/awel/simple_chat_dag_example.py index 50a26fb0e..62d30756d 100644 --- a/examples/awel/simple_chat_dag_example.py +++ b/examples/awel/simple_chat_dag_example.py @@ -14,9 +14,10 @@ }' """ from typing import Dict + from dbgpt._private.pydantic import BaseModel, Field -from dbgpt.core.awel import DAG, HttpTrigger, MapOperator from dbgpt.core import ModelMessage +from dbgpt.core.awel import DAG, HttpTrigger, MapOperator from dbgpt.model.operator.model_operator import ModelOperator diff --git a/examples/awel/simple_chat_history_example.py b/examples/awel/simple_chat_history_example.py index f0c1a6aa5..d1121aa09 100644 --- a/examples/awel/simple_chat_history_example.py +++ b/examples/awel/simple_chat_history_example.py @@ -55,27 +55,23 @@ """ -from typing import Dict, Any, Optional, Union, List import logging +from typing import Dict, List, Optional, Union + from dbgpt._private.pydantic import BaseModel, Field -from dbgpt.core.awel import ( - DAG, - HttpTrigger, - MapOperator, - JoinOperator, -) -from dbgpt.core import LLMClient, InMemoryStorage +from dbgpt.core import InMemoryStorage, LLMClient +from dbgpt.core.awel import DAG, HttpTrigger, JoinOperator, MapOperator from dbgpt.core.operator import ( + BufferedConversationMapperOperator, LLMBranchOperator, LLMOperator, - StreamingLLMOperator, - RequestBuildOperator, - PreConversationOperator, PostConversationOperator, PostStreamingConversationOperator, - BufferedConversationMapperOperator, + PreConversationOperator, + RequestBuildOperator, + StreamingLLMOperator, ) -from dbgpt.model import OpenAIStreamingOperator, MixinLLMOperator +from dbgpt.model import MixinLLMOperator, OpenAIStreamingOperator logger = logging.getLogger(__name__) diff --git a/examples/awel/simple_dag_example.py b/examples/awel/simple_dag_example.py index 93548388d..cfe7cd078 100644 --- a/examples/awel/simple_dag_example.py +++ b/examples/awel/simple_dag_example.py @@ -10,7 +10,6 @@ """ from dbgpt._private.pydantic import BaseModel, Field - from dbgpt.core.awel import DAG, HttpTrigger, MapOperator diff --git a/examples/awel/simple_llm_client_example.py b/examples/awel/simple_llm_client_example.py index 6394c41ce..5751221c9 100644 --- a/examples/awel/simple_llm_client_example.py +++ b/examples/awel/simple_llm_client_example.py @@ -34,24 +34,19 @@ }' """ -from typing import Dict, Any, Optional, Union, List import logging +from typing import Any, Dict, List, Optional, Union + from dbgpt._private.pydantic import BaseModel, Field -from dbgpt.core.awel import ( - DAG, - HttpTrigger, - MapOperator, - JoinOperator, -) from dbgpt.core import LLMClient - +from dbgpt.core.awel import DAG, HttpTrigger, JoinOperator, MapOperator from dbgpt.core.operator import ( LLMBranchOperator, LLMOperator, - StreamingLLMOperator, RequestBuildOperator, + StreamingLLMOperator, ) -from dbgpt.model import OpenAIStreamingOperator, MixinLLMOperator +from dbgpt.model import MixinLLMOperator, OpenAIStreamingOperator logger = logging.getLogger(__name__) diff --git a/examples/awel/simple_rag_example.py b/examples/awel/simple_rag_example.py index a59ab6b3c..d6347cc1d 100644 --- a/examples/awel/simple_rag_example.py +++ b/examples/awel/simple_rag_example.py @@ -17,17 +17,17 @@ """ -from dbgpt.core.awel import HttpTrigger, DAG, MapOperator +from dbgpt.app.openapi.api_view_model import ConversationVo +from dbgpt.app.scene import ChatScene from dbgpt.app.scene.operator._experimental import ( + BaseChatOperator, ChatContext, - PromptManagerOperator, - ChatHistoryStorageOperator, ChatHistoryOperator, + ChatHistoryStorageOperator, EmbeddingEngingOperator, - BaseChatOperator, + PromptManagerOperator, ) -from dbgpt.app.scene import ChatScene -from dbgpt.app.openapi.api_view_model import ConversationVo +from dbgpt.core.awel import DAG, HttpTrigger, MapOperator from dbgpt.model.operator.model_operator import ModelOperator diff --git a/examples/sdk/simple_sdk_llm_example.py b/examples/sdk/simple_sdk_llm_example.py index 40f1321cf..ea479f8aa 100644 --- a/examples/sdk/simple_sdk_llm_example.py +++ b/examples/sdk/simple_sdk_llm_example.py @@ -1,13 +1,8 @@ import asyncio + +from dbgpt.core import BaseOutputParser, PromptTemplate from dbgpt.core.awel import DAG -from dbgpt.core import ( - BaseOutputParser, - PromptTemplate, -) -from dbgpt.core.operator import ( - LLMOperator, - RequestBuildOperator, -) +from dbgpt.core.operator import LLMOperator, RequestBuildOperator from dbgpt.model import OpenAILLMClient with DAG("simple_sdk_llm_example_dag") as dag: diff --git a/examples/sdk/simple_sdk_llm_sql_example.py b/examples/sdk/simple_sdk_llm_sql_example.py index fcf0f9206..7a12ec8b2 100644 --- a/examples/sdk/simple_sdk_llm_sql_example.py +++ b/examples/sdk/simple_sdk_llm_sql_example.py @@ -1,25 +1,20 @@ import asyncio -from typing import Dict, List import json +from typing import Dict, List + +from dbgpt.core import PromptTemplate, SQLOutputParser from dbgpt.core.awel import ( DAG, InputOperator, - SimpleCallDataInputSource, JoinOperator, MapOperator, + SimpleCallDataInputSource, ) -from dbgpt.core import ( - SQLOutputParser, - PromptTemplate, -) -from dbgpt.core.operator import ( - LLMOperator, - RequestBuildOperator, -) -from dbgpt.datasource.rdbms.conn_sqlite import SQLiteTempConnect +from dbgpt.core.operator import LLMOperator, RequestBuildOperator from dbgpt.datasource.operator.datasource_operator import DatasourceOperator -from dbgpt.rag.operator.datasource import DatasourceRetrieverOperator +from dbgpt.datasource.rdbms.conn_sqlite import SQLiteTempConnect from dbgpt.model import OpenAILLMClient +from dbgpt.rag.operator.datasource import DatasourceRetrieverOperator def _create_temporary_connection(): From b85862b759ab9a9f18b70e23832644bc3d4cdb7f Mon Sep 17 00:00:00 2001 From: Fangyin Cheng Date: Thu, 28 Dec 2023 10:40:52 +0800 Subject: [PATCH 6/6] build: Automatically run unit tests and code formatting --- .pre-commit-config.yaml | 23 +++++++++++++++++++++++ CONTRIBUTING.md | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..0a4e1ad34 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,23 @@ +# Please run command `pre-commit install` to install pre-commit hook +repos: + - repo: local + hooks: + - id: python-fmt + name: Python Format + entry: make fmt + language: system + exclude: '^dbgpt/app/static/|^web/' + types: [python] + stages: [commit] + pass_filenames: false + args: [] + - id: python-test + name: Python Unit Test + entry: make test + language: system + exclude: '^dbgpt/app/static/|^web/' + types: [python] + stages: [commit] + pass_filenames: false + args: [] + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 264f69df1..2c9fcfa57 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ pip install -e ".[default]" ``` 4. Install pre-commit hooks ``` -pre-commit install --allow-missing-config +pre-commit install ``` 5. Create a new branch for your changes using the following command: