chore: Add pylint for DB-GPT core lib (#1076)

.flake8

@@ -0,0 +1,40 @@
+[flake8]
+exclude =
+  .eggs/
+  build/
+  */tests/*
+  *_private
+max-line-length = 88
+inline-quotes = "
+ignore =
+  C408
+  C417
+  E121
+  E123
+  E126
+  E203
+  E226
+  E24
+  E704
+  W503
+  W504
+  W605
+  I
+  N
+  B001
+  B002
+  B003
+  B004
+  B005
+  B007
+  B008
+  B009
+  B010
+  B011
+  B012
+  B013
+  B014
+  B015
+  B016
+  B017
+avoid-escape = no

.isort.cfg

@@ -0,0 +1,13 @@
+[settings]
+# This is to make isort compatible with Black. See
+# https://black.readthedocs.io/en/stable/the_black_code_style.html#how-black-wraps-lines.
+line_length=88
+profile=black
+multi_line_output=3
+include_trailing_comma=True
+use_parentheses=True
+float_to_top=True
+filter_files=True
+
+skip_glob=examples/notebook/*
+sections=FUTURE,STDLIB,THIRDPARTY,FIRSTPARTY,LOCALFOLDER,AFTERRAY

.mypy.ini

@@ -0,0 +1,20 @@
+[mypy]
+exclude = /tests/
+# plugins = pydantic.mypy
+
+[mypy-graphviz.*]
+ignore_missing_imports = True
+
+[mypy-cachetools.*]
+ignore_missing_imports = True
+
+[mypy-coloredlogs.*]
+ignore_missing_imports = True
+
+[mypy-termcolor.*]
+ignore_missing_imports = True
+
+[mypy-pydantic.*]
+strict_optional = False
+ignore_missing_imports = True
+follow_imports = skip

.pre-commit-config.yaml

@@ -20,4 +20,22 @@ repos:
         stages: [commit]
         pass_filenames: false
         args: []
+      - id: python-test-doc
+        name: Python Doc Test
+        entry: make test-doc
+        language: system
+        exclude: '^dbgpt/app/static/|^web/'
+        types: [python]
+        stages: [commit]
+        pass_filenames: false
+        args: []
+      - id: python-lint-mypy
+        name: Python Lint mypy
+        entry: make mypy
+        language: system
+        exclude: '^dbgpt/app/static/|^web/'
+        types: [python]
+        stages: [commit]
+        pass_filenames: false
+        args: []
 

Makefile

@@ -14,7 +14,7 @@ setup: $(VENV)/bin/activate
 
 $(VENV)/bin/activate: $(VENV)/.venv-timestamp
 
-$(VENV)/.venv-timestamp: setup.py
+$(VENV)/.venv-timestamp: setup.py requirements
 	# Create new virtual environment if setup.py has changed
 	python3 -m venv $(VENV)
 	$(VENV_BIN)/pip install --upgrade pip
@@ -46,15 +46,14 @@ fmt: setup ## Format Python code
 	# $(VENV_BIN)/blackdoc .
 	$(VENV_BIN)/blackdoc dbgpt
 	$(VENV_BIN)/blackdoc examples
-	# TODO: Type checking of Python code.
-	# https://github.com/python/mypy
-	# $(VENV_BIN)/mypy dbgpt
-	# TODO: uUse flake8 to enforce Python style guide.
+	# TODO: Use flake8 to enforce Python style guide.
 	# https://flake8.pycqa.org/en/latest/
-	# $(VENV_BIN)/flake8 dbgpt
+	$(VENV_BIN)/flake8 dbgpt/core/
+	# TODO: More package checks with flake8.
+
 
 .PHONY: pre-commit
-pre-commit: fmt test ## Run formatting and unit tests before committing
+pre-commit: fmt test test-doc mypy ## Run formatting and unit tests before committing
 
 test: $(VENV)/.testenv ## Run unit tests
 	$(VENV_BIN)/pytest dbgpt
@@ -64,6 +63,12 @@ test-doc: $(VENV)/.testenv ## Run doctests
 	# -k "not test_" skips tests that are not doctests.
 	$(VENV_BIN)/pytest --doctest-modules -k "not test_" dbgpt/core
 
+.PHONY: mypy
+mypy: $(VENV)/.testenv ## Run mypy checks
+	# https://github.com/python/mypy
+	$(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/core/
+	# TODO: More package checks with mypy.
+
 .PHONY: coverage
 coverage: setup ## Run tests and report coverage
 	$(VENV_BIN)/pytest dbgpt --cov=dbgpt

dbgpt/agent/plugin/commands/built_in/disply_type/show_chart_gen.py

@@ -1,22 +1,22 @@
+import logging
 import os
 import uuid
 
 import matplotlib
+import matplotlib.pyplot as plt
+import matplotlib.ticker as mtick
 import pandas as pd
 import seaborn as sns
+from matplotlib.font_manager import FontManager
 from pandas import DataFrame
 
+from dbgpt.configs.model_config import PILOT_PATH
+from dbgpt.util.string_utils import is_scientific_notation
+
 from ...command_mange import command
 
 matplotlib.use("Agg")
-import logging
 
-import matplotlib.pyplot as plt
-import matplotlib.ticker as mtick
-from matplotlib.font_manager import FontManager
-
-from dbgpt.configs.model_config import PILOT_PATH
-from dbgpt.util.string_utils import is_scientific_notation
 
 logger = logging.getLogger(__name__)
 

dbgpt/app/base.py

@@ -192,7 +192,8 @@ def _create_mysql_database(db_name: str, db_url: str, try_to_create_db: bool = F
                 with engine_no_db.connect() as conn:
                     conn.execute(
                         DDL(
-                            f"CREATE DATABASE {db_name} CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci"
+                            f"CREATE DATABASE {db_name} CHARACTER SET utf8mb4 COLLATE "
+                            f"utf8mb4_unicode_ci"
                         )
                     )
                     logger.info(f"Database {db_name} successfully created")
@@ -218,26 +219,31 @@ class WebServerParameters(BaseParameters):
     controller_addr: Optional[str] = field(
         default=None,
         metadata={
-            "help": "The Model controller address to connect. If None, read model controller address from environment key `MODEL_SERVER`."
+            "help": "The Model controller address to connect. If None, read model "
+            "controller address from environment key `MODEL_SERVER`."
         },
     )
     model_name: str = field(
         default=None,
         metadata={
-            "help": "The default model name to use. If None, read model name from environment key `LLM_MODEL`.",
+            "help": "The default model name to use. If None, read model name from "
+            "environment key `LLM_MODEL`.",
             "tags": "fixed",
         },
     )
     share: Optional[bool] = field(
         default=False,
         metadata={
-            "help": "Whether to create a publicly shareable link for the interface. Creates an SSH tunnel to make your UI accessible from anywhere. "
+            "help": "Whether to create a publicly shareable link for the interface. "
+            "Creates an SSH tunnel to make your UI accessible from anywhere. "
         },
     )
     remote_embedding: Optional[bool] = field(
         default=False,
         metadata={
-            "help": "Whether to enable remote embedding models. If it is True, you need to start a embedding model through `dbgpt start worker --worker_type text2vec --model_name xxx --model_path xxx`"
+            "help": "Whether to enable remote embedding models. If it is True, you need"
+            " to start a embedding model through `dbgpt start worker --worker_type "
+            "text2vec --model_name xxx --model_path xxx`"
         },
     )
     log_level: Optional[str] = field(
@@ -286,3 +292,10 @@ class WebServerParameters(BaseParameters):
             "help": "The directories to search awel files, split by `,`",
         },
     )
+    default_thread_pool_size: Optional[int] = field(
+        default=None,
+        metadata={
+            "help": "The default thread pool size, If None, "
+            "use default config of python thread pool",
+        },
+    )

dbgpt/app/component_configs.py

@@ -25,7 +25,9 @@ def initialize_components(
     from dbgpt.model.cluster.controller.controller import controller
 
     # Register global default executor factory first
-    system_app.register(DefaultExecutorFactory)
+    system_app.register(
+        DefaultExecutorFactory, max_workers=param.default_thread_pool_size
+    )
     system_app.register_instance(controller)
 
     from dbgpt.serve.agent.hub.controller import module_agent

dbgpt/app/dbgpt_server.py

@@ -3,8 +3,6 @@
 import sys
 from typing import List
 
-ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-sys.path.append(ROOT_PATH)
 from fastapi import FastAPI
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
@@ -41,6 +39,10 @@
     setup_logging,
 )
 
+ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(ROOT_PATH)
+
+
 static_file_path = os.path.join(ROOT_PATH, "dbgpt", "app/static")
 
 CFG = Config()

dbgpt/app/knowledge/_cli/knowledge_client.py

@@ -5,6 +5,7 @@
 from urllib.parse import urljoin
 
 import requests
+from prettytable import PrettyTable
 
 from dbgpt.app.knowledge.request.request import (
     ChunkQueryRequest,
@@ -193,9 +194,6 @@ def upload(filename: str):
             return
 
 
-from prettytable import PrettyTable
-
-
 class _KnowledgeVisualizer:
     def __init__(self, api_address: str, out_format: str):
         self.client = KnowledgeApiClient(api_address)

dbgpt/app/llmserver.py

@@ -4,13 +4,14 @@
 import os
 import sys
 
-ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-sys.path.append(ROOT_PATH)
-
 from dbgpt._private.config import Config
 from dbgpt.configs.model_config import EMBEDDING_MODEL_CONFIG, LLM_MODEL_CONFIG
 from dbgpt.model.cluster import run_worker_manager
 
+ROOT_PATH = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+sys.path.append(ROOT_PATH)
+
+
 CFG = Config()
 
 model_path = LLM_MODEL_CONFIG.get(CFG.LLM_MODEL)

dbgpt/app/scene/base_chat.py

@@ -313,8 +313,9 @@ async def stream_call(self):
             )
             ### store current conversation
             span.end(metadata={"error": str(e)})
-        # self.memory.append(self.current_message)
-        self.current_message.end_current_round()
+        await blocking_func_to_async(
+            self._executor, self.current_message.end_current_round
+        )
 
     async def nostream_call(self):
         payload = await self._build_model_request()
@@ -381,8 +382,9 @@ async def nostream_call(self):
             )
             span.end(metadata={"error": str(e)})
         ### store dialogue
-        # self.memory.append(self.current_message)
-        self.current_message.end_current_round()
+        await blocking_func_to_async(
+            self._executor, self.current_message.end_current_round
+        )
         return self.current_ai_response()
 
     async def get_llm_response(self):

dbgpt/component.py

@@ -104,28 +104,37 @@ def init_app(self, system_app: SystemApp):
 
     @classmethod
     def get_instance(
-        cls,
+        cls: Type[T],
         system_app: SystemApp,
         default_component=_EMPTY_DEFAULT_COMPONENT,
-        or_register_component: Type[BaseComponent] = None,
+        or_register_component: Optional[Type[T]] = None,
         *args,
         **kwargs,
-    ) -> BaseComponent:
+    ) -> T:
         """Get the current component instance.
 
         Args:
             system_app (SystemApp): The system app
             default_component : The default component instance if not retrieve by name
-            or_register_component (Type[BaseComponent]): The new component to register if not retrieve by name
+            or_register_component (Type[T]): The new component to register if not retrieve by name
 
         Returns:
-            BaseComponent: The component instance
+            T: The component instance
         """
+        # Check for keyword argument conflicts
+        if "default_component" in kwargs:
+            raise ValueError(
+                "default_component argument given in both fixed and **kwargs"
+            )
+        if "or_register_component" in kwargs:
+            raise ValueError(
+                "or_register_component argument given in both fixed and **kwargs"
+            )
+        kwargs["default_component"] = default_component
+        kwargs["or_register_component"] = or_register_component
         return system_app.get_component(
             cls.name,
             cls,
-            default_component=default_component,
-            or_register_component=or_register_component,
             *args,
             **kwargs,
         )
@@ -159,11 +168,11 @@ def config(self) -> AppConfig:
         """Returns the internal AppConfig."""
         return self._app_config
 
-    def register(self, component: Type[BaseComponent], *args, **kwargs) -> T:
+    def register(self, component: Type[T], *args, **kwargs) -> T:
         """Register a new component by its type.
 
         Args:
-            component (Type[BaseComponent]): The component class to register
+            component (Type[T]): The component class to register
 
         Returns:
             T: The instance of registered component
@@ -198,7 +207,7 @@ def get_component(
         name: Union[str, ComponentType],
         component_type: Type[T],
         default_component=_EMPTY_DEFAULT_COMPONENT,
-        or_register_component: Type[BaseComponent] = None,
+        or_register_component: Optional[Type[T]] = None,
         *args,
         **kwargs,
     ) -> T:
@@ -208,7 +217,7 @@ def get_component(
             name (Union[str, ComponentType]): Component name
             component_type (Type[T]): The type of current retrieve component
             default_component : The default component instance if not retrieve by name
-            or_register_component (Type[BaseComponent]): The new component to register if not retrieve by name
+            or_register_component (Type[T]): The new component to register if not retrieve by name
 
         Returns:
             T: The instance retrieved by component name