refactor: The first refactored version for sdk release (#907)

Co-authored-by: chengfangyin2 <[email protected]>

.gitignore

@@ -26,9 +26,9 @@ sdist/
 
 var/
 wheels/
-models/
+/models/
 # Soft link
-models
+/models
 plugins/
 
 pip-wheel-metadata/

dbgpt/__init__.py

@@ -0,0 +1,17 @@
+from dbgpt.component import SystemApp, BaseComponent
+
+
+__ALL__ = ["SystemApp", "BaseComponent"]
+
+_CORE_LIBS = ["core", "rag", "model", "agent", "datasource", "vis", "storage", "train"]
+_SERVE_LIBS = ["serve"]
+_LIBS = _CORE_LIBS + _SERVE_LIBS
+
+
+def __getattr__(name: str):
+    # Lazy load
+    import importlib
+
+    if name in _LIBS:
+        return importlib.import_module("." + name, __name__)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

dbgpt/_private/__init__.py

@@ -0,0 +1,4 @@
+"""This is a private module.
+
+You should not import anything from this module.
+"""

pilot/common/chat_util.py → dbgpt/_private/chat_util.py

@@ -1,10 +1,8 @@
 import asyncio
 from typing import Coroutine, List, Any
 
-from starlette.responses import StreamingResponse
 
-from pilot.scene.base_chat import BaseChat
-from pilot.scene.chat_factory import ChatFactory
+from dbgpt.app.scene import BaseChat, ChatFactory
 
 chat_factory = ChatFactory()
 

pilot/configs/config.py → dbgpt/_private/config.py

@@ -5,11 +5,11 @@
 import os
 from typing import List, Optional, TYPE_CHECKING
 
-from pilot.singleton import Singleton
+from dbgpt.util.singleton import Singleton
 
 if TYPE_CHECKING:
     from auto_gpt_plugin_template import AutoGPTPluginTemplate
-    from pilot.component import SystemApp
+    from dbgpt.component import SystemApp
 
 
 class Config(metaclass=Singleton):
@@ -120,7 +120,7 @@ def __init__(self) -> None:
         )
         self.speak_mode = False
 
-        from pilot.prompts.prompt_registry import PromptTemplateRegistry
+        from dbgpt.core._private.prompt_registry import PromptTemplateRegistry
 
         self.prompt_template_registry = PromptTemplateRegistry()
         ### Related configuration of built-in commands

pilot/common/llm_metadata.py → dbgpt/_private/llm_metadata.py

@@ -1,4 +1,4 @@
-from pydantic import Field, BaseModel
+from dbgpt._private.pydantic import Field, BaseModel
 
 DEFAULT_CONTEXT_WINDOW = 3900
 DEFAULT_NUM_OUTPUTS = 256

dbgpt/_private/pydantic.py

@@ -0,0 +1,33 @@
+import pydantic
+
+if pydantic.VERSION.startswith("1."):
+    PYDANTIC_VERSION = 1
+    from pydantic import (
+        BaseModel,
+        Extra,
+        Field,
+        NonNegativeFloat,
+        NonNegativeInt,
+        PositiveFloat,
+        PositiveInt,
+        ValidationError,
+        root_validator,
+        validator,
+        PrivateAttr,
+    )
+else:
+    PYDANTIC_VERSION = 2
+    # pydantic 2.x
+    from pydantic.v1 import (
+        BaseModel,
+        Extra,
+        Field,
+        NonNegativeFloat,
+        NonNegativeInt,
+        PositiveFloat,
+        PositiveInt,
+        ValidationError,
+        root_validator,
+        validator,
+        PrivateAttr,
+    )

pilot/base_modules/agent/commands/built_in/audio_text.py → dbgpt/agent/commands/built_in/audio_text.py

@@ -3,8 +3,8 @@
 
 import requests
 
-from pilot.base_modules.agent.commands.command_mange import command
-from pilot.configs.config import Config
+from dbgpt.agent.commands.command_mange import command
+from dbgpt._private.config import Config
 
 CFG = Config()
 

pilot/base_modules/agent/commands/built_in/image_gen.py → dbgpt/agent/commands/built_in/image_gen.py

@@ -7,8 +7,8 @@
 import requests
 from PIL import Image
 
-from pilot.base_modules.agent.commands.command_mange import command
-from pilot.configs.config import Config
+from dbgpt.agent.commands.command_mange import command
+from dbgpt._private.config import Config
 
 logger = logging.getLogger(__name__)
 CFG = Config()

pilot/base_modules/agent/commands/command.py → dbgpt/agent/commands/command.py

@@ -7,7 +7,7 @@
 from .exception_not_commands import NotCommands
 from .generator import PluginPromptGenerator
 
-from pilot.configs.config import Config
+from dbgpt._private.config import Config
 
 
 def _resolve_pathlike_command_args(command_args):
@@ -36,7 +36,7 @@ def execute_ai_response_json(
     Returns:
 
     """
-    from pilot.speech.say import say_text
+    from dbgpt.util.speech.say import say_text
 
     cfg = Config()
 

pilot/base_modules/agent/commands/command_mange.py → dbgpt/agent/commands/command_mange.py

@@ -1,20 +1,17 @@
 import functools
 import importlib
 import inspect
-import time
 import json
 import logging
 import xml.etree.ElementTree as ET
-import pandas as pd
 
-from pilot.common.json_utils import serialize
+from dbgpt.util.json_utils import serialize
 from datetime import datetime
 from typing import Any, Callable, Optional, List
-from pydantic import BaseModel
-from pilot.base_modules.agent.common.schema import Status, ApiTagType
-from pilot.base_modules.agent.commands.command import execute_command
-from pilot.base_modules.agent.commands.generator import PluginPromptGenerator
-from pilot.common.string_utils import extract_content_open_ending, extract_content
+from dbgpt._private.pydantic import BaseModel
+from dbgpt.agent.common.schema import Status
+from dbgpt.agent.commands.command import execute_command
+from dbgpt.util.string_utils import extract_content_open_ending, extract_content
 
 # Unique identifier for auto-gpt commands
 AUTO_GPT_COMMAND_IDENTIFIER = "auto_gpt_command"

pilot/base_modules/agent/commands/disply_type/show_chart_gen.py → dbgpt/agent/commands/disply_type/show_chart_gen.py

@@ -1,6 +1,6 @@
 from pandas import DataFrame
 
-from pilot.base_modules.agent.commands.command_mange import command
+from dbgpt.agent.commands.command_mange import command
 import pandas as pd
 import uuid
 import os
@@ -11,14 +11,15 @@
 import matplotlib.pyplot as plt
 import matplotlib.ticker as mtick
 from matplotlib.font_manager import FontManager
-from pilot.common.string_utils import is_scientific_notation
+from dbgpt.util.string_utils import is_scientific_notation
+from dbgpt.configs.model_config import PILOT_PATH
 
 import logging
 
 logger = logging.getLogger(__name__)
 
 
-static_message_img_path = os.path.join(os.getcwd(), "message/img")
+static_message_img_path = os.path.join(PILOT_PATH, "message/img")
 
 
 def data_pre_classification(df: DataFrame):

pilot/base_modules/agent/commands/disply_type/show_table_gen.py → dbgpt/agent/commands/disply_type/show_table_gen.py

@@ -1,6 +1,6 @@
 from pandas import DataFrame
 
-from pilot.base_modules.agent.commands.command_mange import command
+from dbgpt.agent.commands.command_mange import command
 
 import logging
 

pilot/base_modules/agent/commands/disply_type/show_text_gen.py → dbgpt/agent/commands/disply_type/show_text_gen.py

@@ -1,6 +1,6 @@
 from pandas import DataFrame
 
-from pilot.base_modules.agent.commands.command_mange import command
+from dbgpt.agent.commands.command_mange import command
 
 import logging
 

pilot/base_modules/agent/commands/generator.py → dbgpt/agent/commands/generator.py

@@ -1,5 +1,4 @@
 """ A module for generating custom prompt strings."""
-import json
 from typing import Any, Callable, Dict, List, Optional
 
 

pilot/base_modules/agent/controller.py → dbgpt/agent/controller.py

@@ -1,18 +1,14 @@
-import json
-import time
 import logging
 from fastapi import (
     APIRouter,
     Body,
     UploadFile,
     File,
 )
-from abc import ABC, abstractmethod
+from abc import ABC
 from typing import List
-from pilot.configs.model_config import LOGDIR
 
-
-from pilot.openapi.api_view_model import (
+from dbgpt.app.openapi.api_view_model import (
     Result,
 )
 
@@ -21,15 +17,14 @@
     PagenationFilter,
     PagenationResult,
     PluginHubFilter,
-    MyPluginFilter,
 )
 from .hub.agent_hub import AgentHub
 from .db.plugin_hub_db import PluginHubEntity
 from .plugins_util import scan_plugins
 from .commands.generator import PluginPromptGenerator
 
-from pilot.configs.model_config import PLUGINS_DIR
-from pilot.component import BaseComponent, ComponentType, SystemApp
+from dbgpt.configs.model_config import PLUGINS_DIR
+from dbgpt.component import BaseComponent, ComponentType, SystemApp
 
 router = APIRouter()
 logger = logging.getLogger(__name__)
@@ -79,6 +74,7 @@ async def agent_hub_update(update_param: PluginHubParam = Body()):
             if update_param.branch is not None and len(update_param.branch) > 0
             else None
         )
+        # TODO change it to async
         agent_hub.refresh_hub_from_git(update_param.url, branch, authorization)
         return Result.succ(None)
     except Exception as e:

pilot/base_modules/agent/db/my_plugin_db.py → dbgpt/agent/db/my_plugin_db.py

@@ -1,10 +1,9 @@
 from datetime import datetime
-from typing import List
-from sqlalchemy import Column, Integer, String, Index, DateTime, func
+from sqlalchemy import Column, Integer, String, DateTime, func
 from sqlalchemy import UniqueConstraint
 
-from pilot.base_modules.meta_data.base_dao import BaseDao
-from pilot.base_modules.meta_data.meta_data import (
+from dbgpt.storage.metadata import BaseDao
+from dbgpt.storage.metadata.meta_data import (
     Base,
     engine,
     session,

pilot/base_modules/agent/db/plugin_hub_db.py → dbgpt/agent/db/plugin_hub_db.py

@@ -1,12 +1,10 @@
 from datetime import datetime
 import pytz
-from typing import List
-from sqlalchemy import Column, Integer, String, Index, DateTime, func, Boolean, DDL
+from sqlalchemy import Column, Integer, String, Index, DateTime, func, DDL
 from sqlalchemy import UniqueConstraint
-from pilot.base_modules.meta_data.meta_data import Base
 
-from pilot.base_modules.meta_data.base_dao import BaseDao
-from pilot.base_modules.meta_data.meta_data import (
+from dbgpt.storage.metadata import BaseDao
+from dbgpt.storage.metadata.meta_data import (
     Base,
     engine,
     session,

pilot/base_modules/agent/model.py → dbgpt/agent/model.py

@@ -1,7 +1,7 @@
 from typing import TypedDict, Optional, Dict, List
 from dataclasses import dataclass
-from pydantic import BaseModel, Field
 from typing import TypeVar, Generic, Any
+from dbgpt._private.pydantic import BaseModel, Field
 
 T = TypeVar("T")
 

pilot/base_modules/agent/plugins_util.py → dbgpt/agent/plugins_util.py

@@ -4,22 +4,19 @@
 import os
 import glob
 import zipfile
-import fnmatch
-import requests
 import git
 import threading
 import datetime
 import logging
 from pathlib import Path
 from typing import List
-from urllib.parse import urlparse
 from zipimport import zipimporter
 
 import requests
 from auto_gpt_plugin_template import AutoGPTPluginTemplate
 
-from pilot.configs.config import Config
-from pilot.configs.model_config import PLUGINS_DIR
+from dbgpt._private.config import Config
+from dbgpt.configs.model_config import PLUGINS_DIR
 
 logger = logging.getLogger(__name__)
 

pilot/server/__init__.py → dbgpt/app/__init__.py

@@ -1,10 +1,12 @@
+"""The app package.
+
+This package will not be uploaded to PyPI. So, your can't import it if some other package depends on it.
+"""
 import os
 import random
 import sys
 
 from dotenv import load_dotenv
-from pilot.base_modules.agent import PluginPromptGenerator
-
 
 if "pytest" in sys.argv or "pytest" in sys.modules or os.getenv("CI"):
     print("Setting random seed to 42")

pilot/server/_cli.py → dbgpt/app/_cli.py

@@ -1,22 +1,22 @@
 import click
 import os
-from pilot.server.base import WebWerverParameters
-from pilot.configs.model_config import LOGDIR
-from pilot.utils.parameter_utils import EnvArgumentParser
-from pilot.utils.command_utils import _run_current_with_daemon, _stop_service
+from dbgpt.app.base import WebServerParameters
+from dbgpt.configs.model_config import LOGDIR
+from dbgpt.util.parameter_utils import EnvArgumentParser
+from dbgpt.util.command_utils import _run_current_with_daemon, _stop_service
 
 
 @click.command(name="webserver")
-@EnvArgumentParser.create_click_option(WebWerverParameters)
+@EnvArgumentParser.create_click_option(WebServerParameters)
 def start_webserver(**kwargs):
     """Start webserver(dbgpt_server.py)"""
     if kwargs["daemon"]:
         log_file = os.path.join(LOGDIR, "webserver_uvicorn.log")
         _run_current_with_daemon("WebServer", log_file)
     else:
-        from pilot.server.dbgpt_server import run_webserver
+        from dbgpt.app.dbgpt_server import run_webserver
 
-        run_webserver(WebWerverParameters(**kwargs))
+        run_webserver(WebServerParameters(**kwargs))
 
 
 @click.command(name="webserver")