chore: Add pylint for storage

.mypy.ini

@@ -8,8 +8,8 @@ follow_imports = skip
 [mypy-dbgpt.datasource.*]
 follow_imports = skip
 
-[mypy-dbgpt.storage.*]
-follow_imports = skip
+# [mypy-dbgpt.storage.*]
+# follow_imports = skip
 
 [mypy-dbgpt.serve.*]
 follow_imports = skip
@@ -57,4 +57,17 @@ ignore_missing_imports = True
 
 [mypy-spacy.*]
 ignore_missing_imports = True
-follow_imports = skip
+follow_imports = skip
+
+# Storage
+[mypy-msgpack.*]
+ignore_missing_imports = True
+
+[mypy-rocksdict.*]
+ignore_missing_imports = True
+
+[mypy-weaviate.*]
+ignore_missing_imports = True
+
+[mypy-pymilvus.*]
+ignore_missing_imports = True

Makefile

@@ -50,6 +50,7 @@ fmt: setup ## Format Python code
 	# https://flake8.pycqa.org/en/latest/
 	$(VENV_BIN)/flake8 dbgpt/core/
 	$(VENV_BIN)/flake8 dbgpt/rag/
+	$(VENV_BIN)/flake8 dbgpt/storage/
 	# TODO: More package checks with flake8.
 
 .PHONY: fmt-check
@@ -60,8 +61,7 @@ fmt-check: setup ## Check Python code formatting and style without making change
 	$(VENV_BIN)/blackdoc --check dbgpt examples
 	$(VENV_BIN)/flake8 dbgpt/core/
 	$(VENV_BIN)/flake8 dbgpt/rag/
-    # $(VENV_BIN)/blackdoc --check dbgpt examples
-    # $(VENV_BIN)/flake8 dbgpt/core/
+	$(VENV_BIN)/flake8 dbgpt/storage/
 
 .PHONY: pre-commit
 pre-commit: fmt-check test test-doc mypy ## Run formatting and unit tests before committing
@@ -77,8 +77,10 @@ test-doc: $(VENV)/.testenv ## Run doctests
 .PHONY: mypy
 mypy: $(VENV)/.testenv ## Run mypy checks
 	# https://github.com/python/mypy
-	$(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/core/
 	$(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/rag/
+	# rag depends on core and storage, so we not need to check it again.
+	# $(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/storage/
+	# $(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/core/
 	# TODO: More package checks with mypy.
 
 .PHONY: coverage

assets/schema/dbgpt.sql

@@ -83,7 +83,7 @@ CREATE TABLE IF NOT EXISTS `chat_history`
     `id`        int                                     NOT NULL AUTO_INCREMENT COMMENT 'autoincrement id',
     `conv_uid`  varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'Conversation record unique id',
     `chat_mode` varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'Conversation scene mode',
-    `summary`   varchar(255) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'Conversation record summary',
+    `summary`   longtext COLLATE utf8mb4_unicode_ci NOT NULL COMMENT 'Conversation record summary',
     `user_name` varchar(255) COLLATE utf8mb4_unicode_ci DEFAULT NULL COMMENT 'interlocutor',
     `messages`  text COLLATE utf8mb4_unicode_ci COMMENT 'Conversation details',
     `message_ids` text COLLATE utf8mb4_unicode_ci COMMENT 'Message id list, split by comma',

dbgpt/app/openapi/api_v1/editor/_chat_history/__init__.py

@@ -0,0 +1,4 @@
+"""Old chat history module.
+
+Just used by editor.
+"""

dbgpt/storage/chat_history/base.py → dbgpt/app/openapi/api_v1/editor/_chat_history/base.py

@@ -8,10 +8,10 @@
 
 
 class MemoryStoreType(Enum):
-    File = "file"
-    Memory = "memory"
+    # File = "file"
+    # Memory = "memory"
     DB = "db"
-    DuckDb = "duckdb"
+    # DuckDb = "duckdb"
 
 
 class BaseChatHistoryMemory(ABC):
@@ -24,18 +24,14 @@ def __init__(self):
     def messages(self) -> List[OnceConversation]:  # type: ignore
         """Retrieve the messages from the local file"""
 
-    @abstractmethod
-    def create(self, user_name: str) -> None:
-        """Append the message to the record in the local file"""
+    # @abstractmethod
+    # def create(self, user_name: str) -> None:
+    #     """Append the message to the record in the local file"""
 
     @abstractmethod
     def append(self, message: OnceConversation) -> None:
         """Append the message to the record in the local file"""
 
-    # @abstractmethod
-    # def clear(self) -> None:
-    #     """Clear session memory from the local file"""
-
     @abstractmethod
     def update(self, messages: List[OnceConversation]) -> None:
         pass
@@ -45,14 +41,11 @@ def delete(self) -> bool:
         pass
 
     @abstractmethod
-    def conv_info(self, conv_uid: Optional[str] = None) -> None:
-        pass
-
-    @abstractmethod
-    def get_messages(self) -> List[OnceConversation]:
+    def get_messages(self) -> List[Dict]:
         pass
 
     @staticmethod
+    @abstractmethod
     def conv_list(
         user_name: Optional[str] = None, sys_code: Optional[str] = None
     ) -> List[Dict]:

dbgpt/storage/chat_history/chat_hisotry_factory.py → dbgpt/app/openapi/api_v1/editor/_chat_history/chat_hisotry_factory.py

@@ -1,13 +1,16 @@
+"""Module for chat history factory.
+
+It will remove in the future, just support db store type now.
+"""
 import logging
 from typing import Type
 
 from dbgpt._private.config import Config
-from dbgpt.storage.chat_history.base import BaseChatHistoryMemory
 
-from .base import MemoryStoreType
+from .base import BaseChatHistoryMemory, MemoryStoreType
 
 # Import first for auto create table
-from .store_type.meta_db_history import DbHistoryMemory
+from .meta_db_history import DbHistoryMemory
 
 # TODO remove global variable
 CFG = Config()
@@ -20,13 +23,6 @@ def __init__(self):
         self.mem_store_class_map = {}
 
         # Just support db store type after v0.4.6
-        # from .store_type.duckdb_history import DuckdbHistoryMemory
-        # from .store_type.file_history import FileHistoryMemory
-        # from .store_type.mem_history import MemHistoryMemory
-        # self.mem_store_class_map[DuckdbHistoryMemory.store_type] = DuckdbHistoryMemory
-        # self.mem_store_class_map[FileHistoryMemory.store_type] = FileHistoryMemory
-        # self.mem_store_class_map[MemHistoryMemory.store_type] = MemHistoryMemory
-
         self.mem_store_class_map[DbHistoryMemory.store_type] = DbHistoryMemory
 
     def get_store_instance(self, chat_session_id: str) -> BaseChatHistoryMemory:
@@ -53,24 +49,26 @@ def _check_store_type(self, store_type: str):
         Raises:
             ValueError: Invalid store type
         """
-        from .store_type.duckdb_history import DuckdbHistoryMemory
-        from .store_type.file_history import FileHistoryMemory
-        from .store_type.mem_history import MemHistoryMemory
-
-        if store_type == MemHistoryMemory.store_type:
+        if store_type == "memory":
             logger.error(
                 "Not support memory store type, just support db store type now"
             )
             raise ValueError(f"Invalid store type: {store_type}")
 
-        if store_type == FileHistoryMemory.store_type:
+        if store_type == "file":
             logger.error("Not support file store type, just support db store type now")
             raise ValueError(f"Invalid store type: {store_type}")
-        if store_type == DuckdbHistoryMemory.store_type:
-            link1 = "https://docs.dbgpt.site/docs/faq/install#q6-how-to-migrate-meta-table-chat_history-and-connect_config-from-duckdb-to-sqlitel"
-            link2 = "https://docs.dbgpt.site/docs/faq/install#q7-how-to-migrate-meta-table-chat_history-and-connect_config-from-duckdb-to-mysql"
+        if store_type == "duckdb":
+            link1 = (
+                "https://docs.dbgpt.site/docs/latest/faq/install#q6-how-to-migrate-meta"
+                "-table-chat_history-and-connect_config-from-duckdb-to-sqlite"
+            )
+            link2 = (
+                "https://docs.dbgpt.site/docs/latest/faq/install/#q7-how-to-migrate-"
+                "meta-table-chat_history-and-connect_config-from-duckdb-to-mysql"
+            )
             logger.error(
-                "Not support duckdb store type after v0.4.6, just support db store type now, "
-                f"you can migrate your message according to {link1} or {link2}"
+                "Not support duckdb store type after v0.4.6, just support db store "
+                f"type now, you can migrate your message according to {link1} or {link2}"
             )
             raise ValueError(f"Invalid store type: {store_type}")

dbgpt/storage/chat_history/store_type/meta_db_history.py → dbgpt/app/openapi/api_v1/editor/_chat_history/meta_db_history.py

@@ -4,64 +4,76 @@
 
 from dbgpt._private.config import Config
 from dbgpt.core.interface.message import OnceConversation, _conversation_to_dict
-from dbgpt.storage.chat_history.base import BaseChatHistoryMemory, MemoryStoreType
 from dbgpt.storage.chat_history.chat_history_db import ChatHistoryDao, ChatHistoryEntity
 
+from .base import BaseChatHistoryMemory, MemoryStoreType
+
 CFG = Config()
 logger = logging.getLogger(__name__)
 
 
 class DbHistoryMemory(BaseChatHistoryMemory):
-    store_type: str = MemoryStoreType.DB.value
+    """Db history memory storage.
+
+    It is deprecated.
+    """
+
+    store_type: str = MemoryStoreType.DB.value  # type: ignore
 
     def __init__(self, chat_session_id: str):
         self.chat_seesion_id = chat_session_id
         self.chat_history_dao = ChatHistoryDao()
 
     def messages(self) -> List[OnceConversation]:
-        chat_history: ChatHistoryEntity = self.chat_history_dao.get_by_uid(
+        chat_history: Optional[ChatHistoryEntity] = self.chat_history_dao.get_by_uid(
             self.chat_seesion_id
         )
         if chat_history:
             context = chat_history.messages
             if context:
-                conversations: List[OnceConversation] = json.loads(context)
+                conversations: List[OnceConversation] = json.loads(
+                    context  # type: ignore
+                )
                 return conversations
         return []
 
-    def create(self, chat_mode, summary: str, user_name: str) -> None:
-        try:
-            chat_history: ChatHistoryEntity = ChatHistoryEntity()
-            chat_history.chat_mode = chat_mode
-            chat_history.summary = summary
-            chat_history.user_name = user_name
-
-            self.chat_history_dao.raw_update(chat_history)
-        except Exception as e:
-            logger.error("init create conversation log error！" + str(e))
-
+    # def create(self, chat_mode, summary: str, user_name: str) -> None:
+    #     try:
+    #         chat_history: ChatHistoryEntity = ChatHistoryEntity()
+    #         chat_history.chat_mode = chat_mode
+    #         chat_history.summary = summary
+    #         chat_history.user_name = user_name
+    #
+    #         self.chat_history_dao.raw_update(chat_history)
+    #     except Exception as e:
+    #         logger.error("init create conversation log error！" + str(e))
+    #
     def append(self, once_message: OnceConversation) -> None:
         logger.debug(f"db history append: {once_message}")
-        chat_history: ChatHistoryEntity = self.chat_history_dao.get_by_uid(
+        chat_history: Optional[ChatHistoryEntity] = self.chat_history_dao.get_by_uid(
             self.chat_seesion_id
         )
-        conversations: List[OnceConversation] = []
+        conversations: List[Dict] = []
+        latest_user_message = once_message.get_latest_user_message()
+        summary = latest_user_message.content if latest_user_message else ""
         if chat_history:
             context = chat_history.messages
             if context:
-                conversations = json.loads(context)
+                conversations = json.loads(context)  # type: ignore
             else:
-                chat_history.summary = once_message.get_latest_user_message().content
+                chat_history.summary = summary  # type: ignore
         else:
-            chat_history: ChatHistoryEntity = ChatHistoryEntity()
-            chat_history.conv_uid = self.chat_seesion_id
-            chat_history.chat_mode = once_message.chat_mode
-            chat_history.user_name = once_message.user_name
-            chat_history.sys_code = once_message.sys_code
-            chat_history.summary = once_message.get_latest_user_message().content
+            chat_history = ChatHistoryEntity()
+            chat_history.conv_uid = self.chat_seesion_id  # type: ignore
+            chat_history.chat_mode = once_message.chat_mode  # type: ignore
+            chat_history.user_name = once_message.user_name  # type: ignore
+            chat_history.sys_code = once_message.sys_code  # type: ignore
+            chat_history.summary = summary  # type: ignore
 
         conversations.append(_conversation_to_dict(once_message))
-        chat_history.messages = json.dumps(conversations, ensure_ascii=False)
+        chat_history.messages = json.dumps(  # type: ignore
+            conversations, ensure_ascii=False
+        )
 
         self.chat_history_dao.raw_update(chat_history)
 
@@ -72,18 +84,13 @@ def update(self, messages: List[OnceConversation]) -> None:
 
     def delete(self) -> bool:
         self.chat_history_dao.raw_delete(self.chat_seesion_id)
+        return True
 
-    def conv_info(self, conv_uid: str = None) -> None:
-        logger.info("conv_info:{}", conv_uid)
-        chat_history = self.chat_history_dao.get_by_uid(conv_uid)
-        return chat_history.__dict__
-
-    def get_messages(self) -> List[OnceConversation]:
-        # logger.info("get_messages:{}", self.chat_seesion_id)
+    def get_messages(self) -> List[Dict]:
         chat_history = self.chat_history_dao.get_by_uid(self.chat_seesion_id)
         if chat_history:
             context = chat_history.messages
-            return json.loads(context)
+            return json.loads(context)  # type: ignore
         return []
 
     @staticmethod

dbgpt/app/openapi/api_v1/editor/api_editor_v1.py

@@ -1,7 +1,7 @@
 import json
 import logging
 import time
-from typing import List
+from typing import Dict, List
 
 from fastapi import APIRouter, Body, Depends
 
@@ -23,9 +23,9 @@
 )
 from dbgpt.app.scene import ChatFactory
 from dbgpt.app.scene.chat_dashboard.data_loader import DashboardDataLoader
-from dbgpt.core.interface.message import OnceConversation
 from dbgpt.serve.conversation.serve import Serve as ConversationServe
-from dbgpt.storage.chat_history.chat_hisotry_factory import ChatHistory
+
+from ._chat_history.chat_hisotry_factory import ChatHistory
 
 router = APIRouter()
 CFG = Config()
@@ -201,7 +201,7 @@ async def chart_editor_submit(chart_edit_context: ChatChartEditContext = Body())
 
     chat_history_fac = ChatHistory()
     history_mem = chat_history_fac.get_store_instance(chart_edit_context.con_uid)
-    history_messages: List[OnceConversation] = history_mem.get_messages()
+    history_messages: List[Dict] = history_mem.get_messages()
     if history_messages:
         dashboard_data_loader: DashboardDataLoader = DashboardDataLoader()
         db_conn = CFG.LOCAL_DB_MANAGE.get_connect(chart_edit_context.db_name)

dbgpt/app/scene/operators/app_operator.py

@@ -21,10 +21,9 @@
 from dbgpt.core.operators import (
     BufferedConversationMapperOperator,
     HistoryPromptBuilderOperator,
-    LLMBranchOperator,
 )
 from dbgpt.model.operators import LLMOperator, StreamingLLMOperator
-from dbgpt.storage.cache.operator import (
+from dbgpt.storage.cache.operators import (
     CachedModelOperator,
     CachedModelStreamOperator,
     CacheManager,