Merge branch 'main' into feat/mui-antd

.env.template

@@ -166,6 +166,7 @@ VECTOR_STORE_TYPE=Chroma
 #*******************************************************************#
 #**                  WebServer Language Support                   **#
 #*******************************************************************#
+# en, zh, fr, ja, ko, ru
 LANGUAGE=en
 #LANGUAGE=zh
 
@@ -235,4 +236,11 @@ SUMMARY_CONFIG=FAST
 # FATAL, ERROR, WARNING, WARNING, INFO, DEBUG, NOTSET
 DBGPT_LOG_LEVEL=INFO
 # LOG dir, default: ./logs
-#DBGPT_LOG_DIR=
+#DBGPT_LOG_DIR=
+
+
+#*******************************************************************#
+#**                         API_KEYS                              **#
+#*******************************************************************#
+# API_KEYS - The list of API keys that are allowed to access the API. Each of the below are an option, separated by commas.
+# API_KEYS=dbgpt

.gitignore

@@ -63,7 +63,7 @@ coverage.xml
 .pytest_cache/
 
 # Translations
-*.mo
+# *.mo
 *.pot
 
 # Django stuff:
@@ -182,4 +182,6 @@ thirdparty
 
 # Ignore awel DAG visualization files
 /examples/**/*.gv
-/examples/**/*.gv.pdf
+/examples/**/*.gv.pdf
+/i18n/locales/**/**/*_ai_translated.po
+/i18n/locales/**/**/*~

.mypy.ini

@@ -5,9 +5,15 @@ exclude = /tests/
 [mypy-dbgpt.app.*]
 follow_imports = skip
 
+[mypy-dbgpt.agent.*]
+follow_imports = skip
+
 [mypy-dbgpt.serve.*]
 follow_imports = skip
 
+[mypy-dbgpt.model.*]
+follow_imports = skip
+
 [mypy-dbgpt.util.*]
 follow_imports = skip
 
@@ -80,4 +86,7 @@ ignore_missing_imports = True
 ignore_missing_imports = True
 
 [mypy-clickhouse_connect.*]
+ignore_missing_imports = True
+
+[mypy-fastchat.protocol.api_protocol]
 ignore_missing_imports = True

CONTRIBUTING.md

@@ -1,5 +1,11 @@
+# Contribution 
 
-To contribute to this GitHub project, you can follow these steps:
+First of all, thank you for considering contributing to this project. 
+It's people like you that make it a reality for the community. There are many ways to contribute, and we appreciate all of them.
+
+This guide will help you get started with contributing to this project.
+
+## Fork The Repository
 
 1. Fork the repository you want to contribute to by clicking the "Fork" button on the project page.
 
@@ -8,71 +14,107 @@ To contribute to this GitHub project, you can follow these steps:
 ```
 git clone https://github.com/<YOUR-GITHUB-USERNAME>/DB-GPT
 ```
+Please replace `<YOUR-GITHUB-USERNAME>` with your GitHub username.
+
+
+## Create A New Development Environment
+
+1. Create a new virtual environment using the following command:
+```
+# Make sure python >= 3.10
+conda create -n dbgpt_env python=3.10
+conda activate dbgpt_env
+```
 
-3. Install the project requirements
+2. Change to the project directory using the following command:
 ```
+cd DB-GPT
+```
+
+3. Install the project from the local source using the following command:
+```
+# it will take some minutes
 pip install -e ".[default]"
 ```
 
-4. Install pre-commit hooks
+4. Install development requirements
+```
+pip install -r requirements/dev-requirements.txt
+pip install -r requirements/lint-requirements.txt
+```
+
+5. Install pre-commit hooks
 ```
 pre-commit install
 ```
 
-5. Create a new branch for your changes using the following command:
+6. Install `make` command
+The `make` command has been installed by default on most Unix-based systems. If you not 
+have it, you can install it by searching on the internet.
+
+## New Branch And Make Changes
+
+1. Create a new branch for your changes using the following command:
 ```
-git checkout -b "branch-name"
+git checkout -b <branch-name>
 ```
+Please replace `<branch-name>` with a descriptive name for your branch.
 
-6. Make your changes to the code or documentation.
+2. Make your changes to the code or documentation.
 
-- Example: Improve User Interface or Add Documentation.
+3. Add tests for your changes if necessary.
 
-7. Format the code using the following command:
+4. Format your code using the following command:
 ```
 make fmt
 ```
 
-8. Add the changes to the staging area using the following command:
+5. Run the tests using the following command:
 ```
-git add .
+make test
 ```
 
-9. Make sure the tests pass and your code lints using the following command:
+6. Check types using the following command:
 ```
-make pre-commit
+make mypy
 ```
 
-10. Commit the changes with a meaningful commit message using the following command:
+7. Check lint using the following command:
 ```
-git commit -m "your commit message"
+make fmt-check
+```
+
+8. If all checks pass, you can add and commit your changes using the following commands:
 ```
-11. Push the changes to your forked repository using the following command:
+git add xxxx
 ```
-git push origin branch-name
+make sure to replace `xxxx` with the files you want to commit.
+
+then commit your changes using the following command:
+```
+git commit -m "your commit message"
 ```
-12. Go to the GitHub website and navigate to your forked repository.
+Please replace `your commit message` with a meaningful commit message.
 
-13. Click the "New pull request" button.
+It will take some time to get used to the process, but it's worth it. And it will run 
+all git hooks and checks before you commit. If it fails, you need to fix the issues 
+then re-commit it.
 
-14. Select the branch you just pushed to and the branch you want to merge into on the original repository.
+9. Push the changes to your forked repository using the following command:
+```
+git push origin <branch-name>
+```
 
-15. Add a description of your changes and click the "Create pull request" button.
+## Create A Pull Request
 
-16. Wait for the project maintainer to review your changes and provide feedback.
+1. Go to the GitHub website and navigate to your forked repository.
 
-17. Make any necessary changes based on feedback and repeat steps 5-12 until your changes are accepted and merged into the main project.
+2. Click the "New pull request" button.
 
-18. Once your changes are merged, you can update your forked repository and local copy of the repository with the following commands:
+3. Select the branch you just pushed to and the branch you want to merge into on the original repository.
+Write necessary information about your changes and click "Create pull request".
+
+4. Wait for the project maintainer to review your changes and provide feedback.
 
-```
-git fetch upstream
-git checkout master
-git merge upstream/master
-```
-Finally, delete the branch you created with the following command:
-```
-git branch -d branch-name
-```
 That's it you made it 🐣⭐⭐
 

Makefile

@@ -26,8 +26,7 @@ testenv: $(VENV)/.testenv
 
 $(VENV)/.testenv: $(VENV)/bin/activate
 	# $(VENV_BIN)/pip install -e ".[framework]"
-	# $(VENV_BIN)/pip install -e ".[knowledge]"
-	# the openai optional dependency is include framework and knowledge dependencies
+	# the openai optional dependency is include framework and rag dependencies
 	$(VENV_BIN)/pip install -e ".[openai]"
 	touch $(VENV)/.testenv
 
@@ -48,7 +47,7 @@ fmt: setup ## Format Python code
 	$(VENV_BIN)/blackdoc examples
 	# TODO: Use flake8 to enforce Python style guide.
 	# https://flake8.pycqa.org/en/latest/
-	$(VENV_BIN)/flake8 dbgpt/core/ dbgpt/rag/ dbgpt/storage/ dbgpt/datasource/
+	$(VENV_BIN)/flake8 dbgpt/core/ dbgpt/rag/ dbgpt/storage/ dbgpt/datasource/ dbgpt/client/
 	# TODO: More package checks with flake8.
 
 .PHONY: fmt-check
@@ -57,7 +56,7 @@ fmt-check: setup ## Check Python code formatting and style without making change
 	$(VENV_BIN)/isort --check-only --extend-skip="examples/notebook" examples
 	$(VENV_BIN)/black --check --extend-exclude="examples/notebook" .
 	$(VENV_BIN)/blackdoc --check dbgpt examples
-	$(VENV_BIN)/flake8 dbgpt/core/ dbgpt/rag/ dbgpt/storage/ dbgpt/datasource/
+	$(VENV_BIN)/flake8 dbgpt/core/ dbgpt/rag/ dbgpt/storage/ dbgpt/datasource/ dbgpt/client/
 
 .PHONY: pre-commit
 pre-commit: fmt-check test test-doc mypy ## Run formatting and unit tests before committing
@@ -73,7 +72,7 @@ test-doc: $(VENV)/.testenv ## Run doctests
 .PHONY: mypy
 mypy: $(VENV)/.testenv ## Run mypy checks
 	# https://github.com/python/mypy
-	$(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/rag/ dbgpt/datasource/
+	$(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/rag/ dbgpt/datasource/ dbgpt/client/
 	# rag depends on core and storage, so we not need to check it again.
 	# $(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/storage/
 	# $(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/core/
@@ -100,11 +99,11 @@ package: clean-dist ## Package the project for distribution
 	IS_DEV_MODE=false python setup.py sdist bdist_wheel
 
 .PHONY: upload
-upload: package ## Upload the package to PyPI
+upload: ## Upload the package to PyPI
 	# upload to testpypi: twine upload --repository testpypi dist/*
 	twine upload dist/*
 
 .PHONY: help
 help:  ## Display this help screen
 	@echo "Available commands:"
-	@grep -E '^[a-z.A-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-18s\033[0m %s\n", $$1, $$2}' | sort
+	@grep -E '^[a-z.A-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-18s\033[0m %s\n", $$1, $$2}' | sort

README.md

@@ -184,7 +184,6 @@ At present, we have introduced several key features to showcase our current capa
 
 ## Contribution
 
-- Please run `black .` before submitting the code.
 - To check detailed guidelines for new contributions, please refer [how to contribute](https://github.com/eosphoros-ai/DB-GPT/blob/main/CONTRIBUTING.md)
 
 ### Contributors Wall

README.zh.md

@@ -199,7 +199,8 @@
 - [数据Agents](https://www.yuque.com/eosphoros/dbgpt-docs/gwz4rayfuwz78fbq)
 
 ## 贡献
-> 提交代码前请先执行 `black .`
+
+更加详细的贡献指南请参考[如何贡献](https://github.com/eosphoros-ai/DB-GPT/blob/main/CONTRIBUTING.md)。
 
 这是一个用于数据库的复杂且创新的工具, 我们的项目也在紧急的开发当中, 会陆续发布一些新的feature。如在使用当中有任何具体问题, 优先在项目下提issue, 如有需要, 请联系如下微信，我会尽力提供帮助，同时也非常欢迎大家参与到项目建设中。
 

dbgpt/_private/config.py

@@ -286,6 +286,8 @@ def __init__(self) -> None:
         self.MODEL_CACHE_STORAGE_DISK_DIR: Optional[str] = os.getenv(
             "MODEL_CACHE_STORAGE_DISK_DIR"
         )
+        # global dbgpt api key
+        self.API_KEYS = os.getenv("API_KEYS", None)
 
     @property
     def local_db_manager(self) -> "ConnectorManager":

dbgpt/_version.py

@@ -1 +1 @@
-version = "0.5.1"
+version = "0.5.2"

dbgpt/app/dbgpt_server.py

@@ -22,14 +22,10 @@
 # initialize_components import time cost about 0.1s
 from dbgpt.app.component_configs import initialize_components
 from dbgpt.component import SystemApp
-from dbgpt.configs.model_config import (
-    EMBEDDING_MODEL_CONFIG,
-    LLM_MODEL_CONFIG,
-    LOGDIR,
-    ROOT_PATH,
-)
+from dbgpt.configs.model_config import EMBEDDING_MODEL_CONFIG, LLM_MODEL_CONFIG, LOGDIR
 from dbgpt.serve.core import add_exception_handler
 from dbgpt.util.fastapi import PriorityAPIRouter
+from dbgpt.util.i18n_utils import _, set_default_language
 from dbgpt.util.parameter_utils import _get_dict_from_obj
 from dbgpt.util.system_utils import get_system_info
 from dbgpt.util.tracer import SpanType, SpanTypeRunName, initialize_tracer, root_tracer
@@ -47,10 +43,12 @@
 static_file_path = os.path.join(ROOT_PATH, "dbgpt", "app/static")
 
 CFG = Config()
+set_default_language(CFG.LANGUAGE)
+
 
 app = FastAPI(
-    title="DBGPT OPEN API",
-    description="This is dbgpt, with auto docs for the API and everything",
+    title=_("DB-GPT Open API"),
+    description=_("DB-GPT Open API"),
     version=version,
     openapi_tags=[],
 )
@@ -65,18 +63,6 @@
 )
 
 
-@app.get("/doc", include_in_schema=False)
-async def custom_swagger_ui_html():
-    return get_swagger_ui_html(
-        openapi_url=app.openapi_url,
-        title="Custom Swagger UI",
-        swagger_js_url="/swagger_static/swagger-ui-bundle.js",
-        swagger_css_url="/swagger_static/swagger-ui.css",
-    )
-
-
-# applications.get_swagger_ui_html = swagger_monkey_patch
-
 system_app = SystemApp(app)
 
 
@@ -89,13 +75,17 @@ def mount_routers(app: FastAPI):
         router as api_editor_route_v1,
     )
     from dbgpt.app.openapi.api_v1.feedback.api_fb_v1 import router as api_fb_v1
+    from dbgpt.app.openapi.api_v2 import router as api_v2
     from dbgpt.serve.agent.app.controller import router as gpts_v1
+    from dbgpt.serve.agent.app.endpoints import router as app_v2
 
     app.include_router(api_v1, prefix="/api", tags=["Chat"])
+    app.include_router(api_v2, prefix="/api", tags=["ChatV2"])
     app.include_router(api_editor_route_v1, prefix="/api", tags=["Editor"])
     app.include_router(llm_manage_api, prefix="/api", tags=["LLM Manage"])
     app.include_router(api_fb_v1, prefix="/api", tags=["FeedBack"])
     app.include_router(gpts_v1, prefix="/api", tags=["GptsApp"])
+    app.include_router(app_v2, prefix="/api", tags=["App"])
 
     app.include_router(knowledge_router, tags=["Knowledge"])
 

dbgpt/app/initialization/db_model_initialization.py

@@ -2,12 +2,12 @@
 """
 from dbgpt.app.knowledge.chunk_db import DocumentChunkEntity
 from dbgpt.app.knowledge.document_db import KnowledgeDocumentEntity
-from dbgpt.app.knowledge.space_db import KnowledgeSpaceEntity
 from dbgpt.app.openapi.api_v1.feedback.feed_back_db import ChatFeedBackEntity
 from dbgpt.datasource.manages.connect_config_db import ConnectConfigEntity
 from dbgpt.serve.agent.db.my_plugin_db import MyPluginEntity
 from dbgpt.serve.agent.db.plugin_hub_db import PluginHubEntity
 from dbgpt.serve.prompt.models.models import ServeEntity as PromptManageEntity
+from dbgpt.serve.rag.models.models import KnowledgeSpaceEntity
 from dbgpt.storage.chat_history.chat_history_db import (
     ChatHistoryEntity,
     ChatHistoryMessageEntity,

dbgpt/app/initialization/embedding_component.py

@@ -82,8 +82,10 @@ def create(
         return self._model
 
     def _load_model(self) -> "Embeddings":
-        from dbgpt.model.cluster.embedding.loader import EmbeddingLoader
-        from dbgpt.model.cluster.worker.embedding_worker import _parse_embedding_params
+        from dbgpt.model.adapter.embeddings_loader import (
+            EmbeddingLoader,
+            _parse_embedding_params,
+        )
         from dbgpt.model.parameter import (
             EMBEDDING_NAME_TO_PARAMETER_CLASS_CONFIG,
             BaseEmbeddingModelParameters,