feat: add Client and API v2 (#1316)

# Description
1. Provide /api/v2 for DB-GPT
2.  Add DBGPT Python Client for Chat, Flow, App, Knowledge, Including:
   - Chat
   - Create
   - Update
   - Delete
   - Get
   - List
 3. Add examples in `examples/client/`
 4. Add API Reference document

# How Has This Been Tested?
## Test Chat Normal
### Curl
1. set `API_KEYS=dbgpt` in `.env`
2. `python dbgpt/app/dbgpt_server.py`
3. test with curl 
```
DBGPT_API_KEY=dbgpt
curl -X POST "http://localhost:5000/api/v2/chat/completions" \
    -H "Authorization: Bearer $DBGPT_API_KEY" \
    -H "accept: application/json" \
    -H "Content-Type: application/json" \
    -d "{\"messages\":\"Hello\",\"model\":\"chatgpt_proxyllm\"}"
```

```
data: {"id": "chatcmpl-ab5fd180-e699-11ee-8388-acde48001122", "model": "chatgpt_proxyllm", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "Hello"}}]}

data: {"id": "chatcmpl-ab5fd180-e699-11ee-8388-acde48001122", "model": "chatgpt_proxyllm", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "!"}}]}

data: {"id": "chatcmpl-ab5fd180-e699-11ee-8388-acde48001122", "model": "chatgpt_proxyllm", "choices": [{"index": 0, "delta": {"role": "assistant", "content": " How"}}]}

data: {"id": "chatcmpl-ab5fd180-e699-11ee-8388-acde48001122", "model": "chatgpt_proxyllm", "choices": [{"index": 0, "delta": {"role": "assistant", "content": " can"}}]}

data: {"id": "chatcmpl-ab5fd180-e699-11ee-8388-acde48001122", "model": "chatgpt_proxyllm", "choices": [{"index": 0, "delta": {"role": "assistant", "content": " I"}}]}

data: {"id": "chatcmpl-ab5fd180-e699-11ee-8388-acde48001122", "model": "chatgpt_proxyllm", "choices": [{"index": 0, "delta": {"role": "assistant", "content": " assist"}}]}

data: {"id": "chatcmpl-ab5fd180-e699-11ee-8388-acde48001122", "model": "chatgpt_proxyllm", "choices": [{"index": 0, "delta": {"role": "assistant", "content": " you"}}]}

data: {"id": "chatcmpl-ab5fd180-e699-11ee-8388-acde48001122", "model": "chatgpt_proxyllm", "choices": [{"index": 0, "delta": {"role": "assistant", "content": " today"}}]}

data: {"id": "chatcmpl-ab5fd180-e699-11ee-8388-acde48001122", "model": "chatgpt_proxyllm", "choices": [{"index": 0, "delta": {"role": "assistant", "content": "?"}}]}

data: [DONE]
```

### Python
```python
from dbgpt.client import Client

DBGPT_API_KEY = "dbgpt"
client = Client(api_key=DBGPT_API_KEY)
# stream
async for data in client.chat_stream(
    model="chatgpt_proxyllm",
    messages="hello",
):
   print(data)

# no stream
await client.chat(model="chatgpt_proxyllm", messages="hello")

```
## Test Chat App
### Curl
 test with curl 
```
 DBGPT_API_KEY=dbgpt
 APP_CODE={YOUR_APP_CODE}

curl -X POST "http://localhost:5000/api/v2/chat/completions" \
    -H "Authorization: Bearer $DBGPT_API_KEY" \
    -H "accept: application/json" \
    -H "Content-Type: application/json" \
    -d "{\"messages\":\"Hello\",\"model\":\"chatgpt_proxyllm\", \"chat_mode\": \"chat_app\", \"chat_param\": \"$APP_CODE\"}"

```
### Python
```python
from dbgpt.client import Client

DBGPT_API_KEY = "dbgpt"
client = Client(api_key=DBGPT_API_KEY)
APP_CODE="{YOUR_APP_CODE}"

async for data in client.chat_stream(
     model="chatgpt_proxyllm",
     messages="hello",
     chat_mode="chat_app", 
     chat_param=APP_CODE
    ):
        print(data)

```


# Snapshots:

Include snapshots for easier review.

# Checklist:

- [x] My code follows the style guidelines of this project
- [x] I have already rebased the commits and make the commit message
conform to the project standard.
- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I have made corresponding changes to the documentation
- [x] Any dependent changes have been merged and published in downstream
modules

.env.template

@@ -235,4 +235,11 @@ SUMMARY_CONFIG=FAST
 # FATAL, ERROR, WARNING, WARNING, INFO, DEBUG, NOTSET
 DBGPT_LOG_LEVEL=INFO
 # LOG dir, default: ./logs
-#DBGPT_LOG_DIR=
+#DBGPT_LOG_DIR=
+
+
+#*******************************************************************#
+#**                         API_KEYS                              **#
+#*******************************************************************#
+# API_KEYS - The list of API keys that are allowed to access the API. Each of the below are an option, separated by commas.
+# API_KEYS=dbgpt

.mypy.ini

@@ -5,6 +5,9 @@ exclude = /tests/
 [mypy-dbgpt.app.*]
 follow_imports = skip
 
+[mypy-dbgpt.agent.*]
+follow_imports = skip
+
 [mypy-dbgpt.serve.*]
 follow_imports = skip
 
@@ -80,4 +83,7 @@ ignore_missing_imports = True
 ignore_missing_imports = True
 
 [mypy-clickhouse_connect.*]
+ignore_missing_imports = True
+
+[mypy-fastchat.protocol.api_protocol]
 ignore_missing_imports = True

Makefile

@@ -48,7 +48,7 @@ fmt: setup ## Format Python code
 	$(VENV_BIN)/blackdoc examples
 	# TODO: Use flake8 to enforce Python style guide.
 	# https://flake8.pycqa.org/en/latest/
-	$(VENV_BIN)/flake8 dbgpt/core/ dbgpt/rag/ dbgpt/storage/ dbgpt/datasource/
+	$(VENV_BIN)/flake8 dbgpt/core/ dbgpt/rag/ dbgpt/storage/ dbgpt/datasource/ dbgpt/client/
 	# TODO: More package checks with flake8.
 
 .PHONY: fmt-check
@@ -57,7 +57,7 @@ fmt-check: setup ## Check Python code formatting and style without making change
 	$(VENV_BIN)/isort --check-only --extend-skip="examples/notebook" examples
 	$(VENV_BIN)/black --check --extend-exclude="examples/notebook" .
 	$(VENV_BIN)/blackdoc --check dbgpt examples
-	$(VENV_BIN)/flake8 dbgpt/core/ dbgpt/rag/ dbgpt/storage/ dbgpt/datasource/
+	$(VENV_BIN)/flake8 dbgpt/core/ dbgpt/rag/ dbgpt/storage/ dbgpt/datasource/ dbgpt/client/
 
 .PHONY: pre-commit
 pre-commit: fmt-check test test-doc mypy ## Run formatting and unit tests before committing
@@ -73,7 +73,7 @@ test-doc: $(VENV)/.testenv ## Run doctests
 .PHONY: mypy
 mypy: $(VENV)/.testenv ## Run mypy checks
 	# https://github.com/python/mypy
-	$(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/rag/ dbgpt/datasource/
+	$(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/rag/ dbgpt/datasource/ dbgpt/client/
 	# rag depends on core and storage, so we not need to check it again.
 	# $(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/storage/
 	# $(VENV_BIN)/mypy --config-file .mypy.ini dbgpt/core/
@@ -107,4 +107,4 @@ upload: package ## Upload the package to PyPI
 .PHONY: help
 help:  ## Display this help screen
 	@echo "Available commands:"
-	@grep -E '^[a-z.A-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-18s\033[0m %s\n", $$1, $$2}' | sort
+	@grep -E '^[a-z.A-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-18s\033[0m %s\n", $$1, $$2}' | sort

dbgpt/_private/config.py

@@ -286,6 +286,8 @@ def __init__(self) -> None:
         self.MODEL_CACHE_STORAGE_DISK_DIR: Optional[str] = os.getenv(
             "MODEL_CACHE_STORAGE_DISK_DIR"
         )
+        # global dbgpt api key
+        self.API_KEYS = os.getenv("API_KEYS", None)
 
     @property
     def local_db_manager(self) -> "ConnectorManager":

dbgpt/app/dbgpt_server.py

@@ -89,13 +89,17 @@ def mount_routers(app: FastAPI):
         router as api_editor_route_v1,
     )
     from dbgpt.app.openapi.api_v1.feedback.api_fb_v1 import router as api_fb_v1
+    from dbgpt.app.openapi.api_v2 import router as api_v2
     from dbgpt.serve.agent.app.controller import router as gpts_v1
+    from dbgpt.serve.agent.app.endpoints import router as app_v2
 
     app.include_router(api_v1, prefix="/api", tags=["Chat"])
+    app.include_router(api_v2, prefix="/api", tags=["ChatV2"])
     app.include_router(api_editor_route_v1, prefix="/api", tags=["Editor"])
     app.include_router(llm_manage_api, prefix="/api", tags=["LLM Manage"])
     app.include_router(api_fb_v1, prefix="/api", tags=["FeedBack"])
     app.include_router(gpts_v1, prefix="/api", tags=["GptsApp"])
+    app.include_router(app_v2, prefix="/api", tags=["App"])
 
     app.include_router(knowledge_router, tags=["Knowledge"])
 

dbgpt/app/initialization/db_model_initialization.py

@@ -2,12 +2,12 @@
 """
 from dbgpt.app.knowledge.chunk_db import DocumentChunkEntity
 from dbgpt.app.knowledge.document_db import KnowledgeDocumentEntity
-from dbgpt.app.knowledge.space_db import KnowledgeSpaceEntity
 from dbgpt.app.openapi.api_v1.feedback.feed_back_db import ChatFeedBackEntity
 from dbgpt.datasource.manages.connect_config_db import ConnectConfigEntity
 from dbgpt.serve.agent.db.my_plugin_db import MyPluginEntity
 from dbgpt.serve.agent.db.plugin_hub_db import PluginHubEntity
 from dbgpt.serve.prompt.models.models import ServeEntity as PromptManageEntity
+from dbgpt.serve.rag.models.models import KnowledgeSpaceEntity
 from dbgpt.storage.chat_history.chat_history_db import (
     ChatHistoryEntity,
     ChatHistoryMessageEntity,

dbgpt/app/initialization/serve_initialization.py

@@ -5,6 +5,8 @@
 def register_serve_apps(system_app: SystemApp, cfg: Config):
     """Register serve apps"""
     system_app.config.set("dbgpt.app.global.language", cfg.LANGUAGE)
+    if cfg.API_KEYS:
+        system_app.config.set("dbgpt.app.global.api_keys", cfg.API_KEYS)
 
     # ################################ Prompt Serve Register Begin ######################################
     from dbgpt.serve.prompt.serve import (
@@ -42,4 +44,12 @@ def register_serve_apps(system_app: SystemApp, cfg: Config):
 
     # Register serve app
     system_app.register(FlowServe)
+
+    from dbgpt.serve.rag.serve import (
+        SERVE_CONFIG_KEY_PREFIX as RAG_SERVE_CONFIG_KEY_PREFIX,
+    )
+    from dbgpt.serve.rag.serve import Serve as RagServe
+
+    # Register serve app
+    system_app.register(RagServe)
     # ################################ AWEL Flow Serve Register End ########################################

dbgpt/app/knowledge/api.py

@@ -4,7 +4,7 @@
 import tempfile
 from typing import List
 
-from fastapi import APIRouter, File, Form, UploadFile
+from fastapi import APIRouter, Depends, File, Form, UploadFile
 
 from dbgpt._private.config import Config
 from dbgpt.app.knowledge.request.request import (
@@ -16,7 +16,6 @@
     KnowledgeDocumentRequest,
     KnowledgeQueryRequest,
     KnowledgeSpaceRequest,
-    KnowledgeSyncRequest,
     SpaceArgumentRequest,
 )
 from dbgpt.app.knowledge.request.response import KnowledgeQueryResponse
@@ -31,6 +30,8 @@
 from dbgpt.rag.knowledge.base import ChunkStrategy
 from dbgpt.rag.knowledge.factory import KnowledgeFactory
 from dbgpt.rag.retriever.embedding import EmbeddingRetriever
+from dbgpt.serve.rag.api.schemas import KnowledgeSyncRequest
+from dbgpt.serve.rag.service.service import Service
 from dbgpt.storage.vector_store.base import VectorStoreConfig
 from dbgpt.storage.vector_store.connector import VectorStoreConnector
 from dbgpt.util.tracer import SpanType, root_tracer
@@ -44,6 +45,11 @@
 knowledge_space_service = KnowledgeService()
 
 
+def get_rag_service() -> Service:
+    """Get Rag Service."""
+    return Service.get_instance(CFG.SYSTEM_APP)
+
+
 @router.post("/knowledge/space/add")
 def space_add(request: KnowledgeSpaceRequest):
     print(f"/space/add params: {request}")
@@ -226,12 +232,20 @@ def document_sync(space_name: str, request: DocumentSyncRequest):
 
 
 @router.post("/knowledge/{space_name}/document/sync_batch")
-def batch_document_sync(space_name: str, request: List[KnowledgeSyncRequest]):
+def batch_document_sync(
+    space_name: str,
+    request: List[KnowledgeSyncRequest],
+    service: Service = Depends(get_rag_service),
+):
     logger.info(f"Received params: {space_name}, {request}")
     try:
-        doc_ids = knowledge_space_service.batch_document_sync(
-            space_name=space_name, sync_requests=request
-        )
+        space = service.get({"name": space_name})
+        for sync_request in request:
+            sync_request.space_id = space.id
+        doc_ids = service.sync_document(requests=request)
+        # doc_ids = service.sync_document(
+        #     space_name=space_name, sync_requests=request
+        # )
         return Result.succ({"tasks": doc_ids})
     except Exception as e:
         return Result.failed(code="E000X", msg=f"document sync error {e}")

dbgpt/app/knowledge/document_db.py

@@ -1,9 +1,11 @@
 from datetime import datetime
-from typing import List
+from typing import Any, Dict, List, Union
 
 from sqlalchemy import Column, DateTime, Integer, String, Text, func
 
 from dbgpt._private.config import Config
+from dbgpt.serve.conversation.api.schemas import ServeRequest
+from dbgpt.serve.rag.api.schemas import DocumentServeRequest, DocumentServeResponse
 from dbgpt.storage.metadata import BaseDao, Model
 
 CFG = Config()
@@ -218,3 +220,70 @@ def raw_delete(self, query: KnowledgeDocumentEntity):
         knowledge_documents.delete()
         session.commit()
         session.close()
+
+    def from_request(
+        self, request: Union[ServeRequest, Dict[str, Any]]
+    ) -> KnowledgeDocumentEntity:
+        """Convert the request to an entity
+
+        Args:
+            request (Union[ServeRequest, Dict[str, Any]]): The request
+
+        Returns:
+            T: The entity
+        """
+        request_dict = (
+            request.dict() if isinstance(request, DocumentServeRequest) else request
+        )
+        entity = KnowledgeDocumentEntity(**request_dict)
+        return entity
+
+    def to_request(self, entity: KnowledgeDocumentEntity) -> DocumentServeResponse:
+        """Convert the entity to a request
+
+        Args:
+            entity (T): The entity
+
+        Returns:
+            REQ: The request
+        """
+        return DocumentServeResponse(
+            id=entity.id,
+            doc_name=entity.doc_name,
+            doc_type=entity.doc_type,
+            space=entity.space,
+            chunk_size=entity.chunk_size,
+            status=entity.status,
+            last_sync=entity.last_sync,
+            content=entity.content,
+            result=entity.result,
+            vector_ids=entity.vector_ids,
+            summary=entity.summary,
+            gmt_created=entity.gmt_created,
+            gmt_modified=entity.gmt_modified,
+        )
+
+    def to_response(self, entity: KnowledgeDocumentEntity) -> DocumentServeResponse:
+        """Convert the entity to a response
+
+        Args:
+            entity (T): The entity
+
+        Returns:
+            REQ: The request
+        """
+        return DocumentServeResponse(
+            id=entity.id,
+            doc_name=entity.doc_name,
+            doc_type=entity.doc_type,
+            space=entity.space,
+            chunk_size=entity.chunk_size,
+            status=entity.status,
+            last_sync=entity.last_sync,
+            content=entity.content,
+            result=entity.result,
+            vector_ids=entity.vector_ids,
+            summary=entity.summary,
+            gmt_created=entity.gmt_created,
+            gmt_modified=entity.gmt_modified,
+        )

dbgpt/app/knowledge/request/request.py

@@ -17,6 +17,8 @@ class KnowledgeQueryRequest(BaseModel):
 class KnowledgeSpaceRequest(BaseModel):
     """name: knowledge space name"""
 
+    """vector_type: vector type"""
+    id: int = None
     name: str = None
     """vector_type: vector type"""
     vector_type: str = None
@@ -37,9 +39,6 @@ class KnowledgeDocumentRequest(BaseModel):
     """content: content"""
     source: str = None
 
-    """text_chunk_size: text_chunk_size"""
-    # text_chunk_size: int
-
 
 class DocumentQueryRequest(BaseModel):
     """doc_name: doc path"""
@@ -80,20 +79,6 @@ class DocumentSyncRequest(BaseModel):
     chunk_overlap: Optional[int] = None
 
 
-class KnowledgeSyncRequest(BaseModel):
-    """Sync request"""
-
-    """doc_ids: doc ids"""
-    doc_id: int
-
-    """model_name: model name"""
-    model_name: Optional[str] = None
-
-    """chunk_parameters: chunk parameters 
-    """
-    chunk_parameters: ChunkParameters
-
-
 class ChunkQueryRequest(BaseModel):
     """id: id"""
 

dbgpt/app/knowledge/service.py

@@ -1,7 +1,6 @@
 import json
 import logging
 from datetime import datetime
-from enum import Enum
 from typing import List
 
 from dbgpt._private.config import Config
@@ -17,15 +16,13 @@
     DocumentSyncRequest,
     KnowledgeDocumentRequest,
     KnowledgeSpaceRequest,
-    KnowledgeSyncRequest,
     SpaceArgumentRequest,
 )
 from dbgpt.app.knowledge.request.response import (
     ChunkQueryResponse,
     DocumentQueryResponse,
     SpaceQueryResponse,
 )
-from dbgpt.app.knowledge.space_db import KnowledgeSpaceDao, KnowledgeSpaceEntity
 from dbgpt.component import ComponentType
 from dbgpt.configs.model_config import EMBEDDING_MODEL_CONFIG
 from dbgpt.core import Chunk
@@ -38,8 +35,11 @@
     RecursiveCharacterTextSplitter,
     SpacyTextSplitter,
 )
+from dbgpt.serve.rag.api.schemas import KnowledgeSyncRequest
 from dbgpt.serve.rag.assembler.embedding import EmbeddingAssembler
 from dbgpt.serve.rag.assembler.summary import SummaryAssembler
+from dbgpt.serve.rag.models.models import KnowledgeSpaceDao, KnowledgeSpaceEntity
+from dbgpt.serve.rag.service.service import Service, SyncStatus
 from dbgpt.storage.vector_store.base import VectorStoreConfig
 from dbgpt.storage.vector_store.connector import VectorStoreConnector
 from dbgpt.util.executor_utils import ExecutorFactory, blocking_func_to_async
@@ -53,13 +53,6 @@
 CFG = Config()
 
 
-class SyncStatus(Enum):
-    TODO = "TODO"
-    FAILED = "FAILED"
-    RUNNING = "RUNNING"
-    FINISHED = "FINISHED"
-
-
 # default summary max iteration call with llm.
 DEFAULT_SUMMARY_MAX_ITERATION = 5
 # default summary concurrency call with llm.
@@ -88,8 +81,8 @@ def create_knowledge_space(self, request: KnowledgeSpaceRequest):
         spaces = knowledge_space_dao.get_knowledge_space(query)
         if len(spaces) > 0:
             raise Exception(f"space name:{request.name} have already named")
-        knowledge_space_dao.create_knowledge_space(request)
-        return True
+        space_id = knowledge_space_dao.create_knowledge_space(request)
+        return space_id
 
     def create_knowledge_document(self, space, request: KnowledgeDocumentRequest):
         """create knowledge document
@@ -199,7 +192,9 @@ def get_knowledge_documents(self, space, request: DocumentQueryRequest):
         return res
 
     def batch_document_sync(
-        self, space_name, sync_requests: List[KnowledgeSyncRequest]
+        self,
+        space_name,
+        sync_requests: List[KnowledgeSyncRequest],
     ) -> List[int]:
         """batch sync knowledge document chunk into vector store
         Args: