diff --git a/dbgpt/app/operators/rag.py b/dbgpt/app/operators/rag.py index d7fa75b24..abf10736e 100644 --- a/dbgpt/app/operators/rag.py +++ b/dbgpt/app/operators/rag.py @@ -104,13 +104,12 @@ async def map(self, context: HOContextBody) -> List[Chunk]: return chunks metadata = ViewMetadata( - label=_("Knowledge Operator"), + label=_("Knowledge Space Operator"), name="higher_order_knowledge_operator", category=OperatorCategory.RAG, description=_( _( - "Knowledge Operator, retrieve your knowledge(documents) from knowledge" - " space" + "Knowledge Space Operator, retrieve your knowledge from knowledge space" ) ), parameters=[ diff --git a/dbgpt/core/awel/flow/base.py b/dbgpt/core/awel/flow/base.py index 336b3a32b..a98f55276 100644 --- a/dbgpt/core/awel/flow/base.py +++ b/dbgpt/core/awel/flow/base.py @@ -213,6 +213,9 @@ class OperatorType(str, Enum): "embeddings": _CategoryDetail("Embeddings", "The embeddings resource"), "rag": _CategoryDetail("RAG", "The resource"), "vector_store": _CategoryDetail("Vector Store", "The vector store resource"), + "knowledge_graph": _CategoryDetail( + "Knowledge Graph", "The knowledge graph resource" + ), "database": _CategoryDetail("Database", "Interact with the database"), "example": _CategoryDetail("Example", "The example resource"), } @@ -231,6 +234,8 @@ class ResourceCategory(str, Enum): EMBEDDINGS = "embeddings" RAG = "rag" VECTOR_STORE = "vector_store" + KNOWLEDGE_GRAPH = "knowledge_graph" + FULL_TEXT = "full_text" DATABASE = "database" EXAMPLE = "example" diff --git a/dbgpt/rag/operators/__init__.py b/dbgpt/rag/operators/__init__.py index ceaef2745..e239b966d 100644 --- a/dbgpt/rag/operators/__init__.py +++ b/dbgpt/rag/operators/__init__.py @@ -1,5 +1,5 @@ """Module for RAG operators.""" - +from .chunk_manager import ChunkManagerOperator # noqa: F401 from .datasource import DatasourceRetrieverOperator # noqa: F401 from .db_schema import ( # noqa: F401 DBSchemaAssemblerOperator, @@ -10,21 +10,32 @@ EmbeddingRetrieverOperator, ) from .evaluation import RetrieverEvaluatorOperator # noqa: F401 +from .full_text import FullTextStorageOperator # noqa: F401 from .knowledge import ChunksToStringOperator, KnowledgeOperator # noqa: F401 +from .knowledge_graph import KnowledgeGraphOperator # noqa: F401 +from .process_branch import KnowledgeProcessBranchOperator # noqa: F401 +from .process_branch import KnowledgeProcessJoinOperator from .rerank import RerankOperator # noqa: F401 from .rewrite import QueryRewriteOperator # noqa: F401 from .summary import SummaryAssemblerOperator # noqa: F401 +from .vector_store import VectorStorageOperator # noqa: F401 __all__ = [ + "ChunkManagerOperator", "DatasourceRetrieverOperator", "DBSchemaRetrieverOperator", "DBSchemaAssemblerOperator", "EmbeddingRetrieverOperator", "EmbeddingAssemblerOperator", + "FullTextStorageOperator", "KnowledgeOperator", + "KnowledgeGraphOperator", + "KnowledgeProcessBranchOperator", + "KnowledgeProcessJoinOperator", "ChunksToStringOperator", "RerankOperator", "QueryRewriteOperator", "SummaryAssemblerOperator", "RetrieverEvaluatorOperator", + "VectorStorageOperator", ] diff --git a/dbgpt/rag/operators/chunk_manager.py b/dbgpt/rag/operators/chunk_manager.py new file mode 100644 index 000000000..f2c48230f --- /dev/null +++ b/dbgpt/rag/operators/chunk_manager.py @@ -0,0 +1,68 @@ +"""Chunk Manager Operator.""" +from typing import List, Optional + +from dbgpt.core import Chunk +from dbgpt.core.awel import MapOperator +from dbgpt.core.awel.flow import IOField, OperatorCategory, Parameter, ViewMetadata +from dbgpt.rag import ChunkParameters +from dbgpt.rag.chunk_manager import ChunkManager +from dbgpt.rag.knowledge.base import Knowledge +from dbgpt.util.i18n_utils import _ + + +class ChunkManagerOperator(MapOperator[Knowledge, List[Chunk]]): + """Chunk Manager Operator.""" + + metadata = ViewMetadata( + label=_("Chunk Manager Operator"), + name="chunk_manager_operator", + description=_(" Split Knowledge Documents into chunks."), + category=OperatorCategory.RAG, + parameters=[ + Parameter.build_from( + _("Chunk Split Parameters"), + "chunk_parameters", + ChunkParameters, + description=_("Chunk Split Parameters."), + optional=True, + default=None, + alias=["chunk_parameters"], + ), + ], + inputs=[ + IOField.build_from( + _("Knowledge"), + "knowledge", + Knowledge, + description=_("The knowledge to be loaded."), + ) + ], + outputs=[ + IOField.build_from( + _("Chunks"), + "chunks", + List[Chunk], + description=_("The split chunks by chunk manager."), + is_list=True, + ) + ], + ) + + def __init__( + self, + chunk_parameters: Optional[ChunkParameters] = None, + **kwargs, + ): + """Init the datasource operator.""" + MapOperator.__init__(self, **kwargs) + self._chunk_parameters = chunk_parameters or ChunkParameters( + chunk_strategy="Automatic" + ) + + async def map(self, knowledge: Knowledge) -> List[Chunk]: + """Persist chunks in vector db.""" + documents = knowledge.load() + chunk_manager = ChunkManager( + knowledge=knowledge, chunk_parameter=self._chunk_parameters + ) + return chunk_manager.split(documents) diff --git a/dbgpt/rag/operators/full_text.py b/dbgpt/rag/operators/full_text.py new file mode 100644 index 000000000..fad7d2cad --- /dev/null +++ b/dbgpt/rag/operators/full_text.py @@ -0,0 +1,74 @@ +"""Full Text Operator.""" +import os +from typing import List, Optional + +from dbgpt.core import Chunk +from dbgpt.core.awel import MapOperator +from dbgpt.core.awel.flow import IOField, OperatorCategory, Parameter, ViewMetadata +from dbgpt.storage.full_text.base import FullTextStoreBase +from dbgpt.util.i18n_utils import _ + + +class FullTextStorageOperator(MapOperator[List[Chunk], List[Chunk]]): + """Full Text Operator.""" + + metadata = ViewMetadata( + label=_("Full Text Storage Operator"), + name="full text_storage_operator", + description=_("Persist embeddings into full text storage."), + category=OperatorCategory.RAG, + parameters=[ + Parameter.build_from( + _("Full Text Connector"), + "full_text_store", + FullTextStoreBase, + description=_("The full text store."), + alias=["full_text_store"], + ), + ], + inputs=[ + IOField.build_from( + _("Chunks"), + "chunks", + List[Chunk], + description=_("The text split chunks by chunk manager."), + is_list=True, + ) + ], + outputs=[ + IOField.build_from( + _("Chunks"), + "chunks", + List[Chunk], + description=_( + "The assembled chunks, it has been persisted to full text " "store." + ), + is_list=True, + ) + ], + ) + + def __init__( + self, + full_text_store: Optional[FullTextStoreBase] = None, + max_chunks_once_load: Optional[int] = None, + **kwargs, + ): + """Init the datasource operator.""" + MapOperator.__init__(self, **kwargs) + self._full_text_store = full_text_store + self._embeddings = full_text_store.get_config().embedding_fn + self._max_chunks_once_load = max_chunks_once_load + self.full_text_store = full_text_store + + async def map(self, chunks: List[Chunk]) -> List[Chunk]: + """Persist chunks in full text db.""" + max_chunks_once_load = self._max_chunks_once_load or int( + os.getenv("KNOWLEDGE_MAX_CHUNKS_ONCE_LOAD", 10) + ) + full_text_ids = await self._full_text_store.aload_document_with_limit( + chunks, max_chunks_once_load + ) + for chunk, full_text_id in zip(chunks, full_text_ids): + chunk.chunk_id = str(full_text_id) + return chunks diff --git a/dbgpt/rag/operators/knowledge_graph.py b/dbgpt/rag/operators/knowledge_graph.py new file mode 100644 index 000000000..05a7366e6 --- /dev/null +++ b/dbgpt/rag/operators/knowledge_graph.py @@ -0,0 +1,74 @@ +"""Knowledge Graph Operator.""" +import os +from typing import List, Optional + +from dbgpt.core import Chunk +from dbgpt.core.awel import MapOperator +from dbgpt.core.awel.flow import IOField, OperatorCategory, Parameter, ViewMetadata +from dbgpt.storage.knowledge_graph.base import KnowledgeGraphBase +from dbgpt.util.i18n_utils import _ + + +class KnowledgeGraphOperator(MapOperator[List[Chunk], List[Chunk]]): + """Knowledge Graph Operator.""" + + metadata = ViewMetadata( + label=_("Knowledge Graph Operator"), + name="knowledge_graph_operator", + description=_("Extract Documents and persist into graph database."), + category=OperatorCategory.RAG, + parameters=[ + Parameter.build_from( + _("Knowledge Graph Connector"), + "graph_store", + KnowledgeGraphBase, + description=_("The knowledge graph."), + alias=["graph_store"], + ), + ], + inputs=[ + IOField.build_from( + _("Chunks"), + "chunks", + List[Chunk], + description=_("The text split chunks by chunk manager."), + is_list=True, + ) + ], + outputs=[ + IOField.build_from( + _("Chunks"), + "chunks", + List[Chunk], + description=_( + "The assembled chunks, it has been persisted to graph store." + ), + is_list=True, + ) + ], + ) + + def __init__( + self, + graph_store: Optional[KnowledgeGraphBase] = None, + max_chunks_once_load: Optional[int] = None, + **kwargs, + ): + """Init the Knowledge Graph operator.""" + MapOperator.__init__(self, **kwargs) + self._graph_store = graph_store + self._embeddings = graph_store.get_config().embedding_fn + self._max_chunks_once_load = max_chunks_once_load + self.graph_store = graph_store + + async def map(self, chunks: List[Chunk]) -> List[Chunk]: + """Persist chunks in graph db.""" + max_chunks_once_load = self._max_chunks_once_load or int( + os.getenv("KNOWLEDGE_MAX_CHUNKS_ONCE_LOAD", 10) + ) + graph_ids = await self._graph_store.aload_document_with_limit( + chunks, max_chunks_once_load + ) + for chunk, graph_id in zip(chunks, graph_ids): + chunk.chunk_id = str(graph_id) + return chunks diff --git a/dbgpt/rag/operators/process_branch.py b/dbgpt/rag/operators/process_branch.py new file mode 100644 index 000000000..c0d9541ef --- /dev/null +++ b/dbgpt/rag/operators/process_branch.py @@ -0,0 +1,193 @@ +"""Knowledge Process Branch Operator.""" +from typing import Dict, List, Optional + +from dbgpt.core import Chunk +from dbgpt.core.awel import ( + BranchFunc, + BranchOperator, + BranchTaskType, + JoinOperator, + logger, +) +from dbgpt.core.awel.flow import IOField, OperatorCategory, OperatorType, ViewMetadata +from dbgpt.rag.knowledge.base import Knowledge +from dbgpt.util.i18n_utils import _ + + +class KnowledgeProcessBranchOperator(BranchOperator[Knowledge, Knowledge]): + """Knowledge Process branch operator. + + This operator will branch the workflow based on + the stream flag of the request. + """ + + metadata = ViewMetadata( + label=_("Knowledge Process Branch Operator"), + name="knowledge_process_operator", + category=OperatorCategory.RAG, + operator_type=OperatorType.BRANCH, + description=_("Branch the workflow based on the stream flag of the request."), + parameters=[], + inputs=[ + IOField.build_from( + _("Document Chunks"), + "input_value", + List[Chunk], + description=_("The input value of the operator."), + is_list=True, + ), + ], + outputs=[ + IOField.build_from( + _("Chunks"), + "chunks", + List[Chunk], + description=_("Chunks for Vector Storage Connector."), + is_list=True, + ), + IOField.build_from( + _("Chunks"), + "chunks", + List[Chunk], + description=_("Chunks for Knowledge Graph Connector."), + is_list=True, + ), + IOField.build_from( + _("Chunks"), + "chunks", + List[Chunk], + description=_("Chunks for Full Text Connector."), + is_list=True, + ), + ], + ) + + def __init__( + self, + graph_task_name: Optional[str] = None, + embedding_task_name: Optional[str] = None, + **kwargs, + ): + """Create the intent detection branch operator.""" + super().__init__(**kwargs) + self._graph_task_name = graph_task_name + self._embedding_task_name = embedding_task_name + self._full_text_task_name = embedding_task_name + + async def branches( + self, + ) -> Dict[BranchFunc[Knowledge], BranchTaskType]: + """Branch the intent detection result to different tasks.""" + download_cls_list = set(task.__class__ for task in self.downstream) # noqa + branch_func_map = {} + + async def check_graph_process(r: Knowledge) -> bool: + # If check graph is true, we will run extract knowledge graph triplets. + from dbgpt.rag.operators import KnowledgeGraphOperator + + if KnowledgeGraphOperator in download_cls_list: + return True + return False + + async def check_embedding_process(r: Knowledge) -> bool: + # If check embedding is true, we will run extract document embedding. + from dbgpt.rag.operators import VectorStorageOperator + + if VectorStorageOperator in download_cls_list: + return True + return False + + async def check_full_text_process(r: Knowledge) -> bool: + # If check full text is true, we will run extract document keywords. + from dbgpt.rag.operators.full_text import FullTextStorageOperator + + if FullTextStorageOperator in download_cls_list: + return True + return False + + branch_func_map[check_graph_process] = self._graph_task_name + branch_func_map[check_embedding_process] = self._embedding_task_name + branch_func_map[check_full_text_process] = self._full_text_task_name + return branch_func_map # type: ignore + + +class KnowledgeProcessJoinOperator(JoinOperator[List[str]]): + """Knowledge Process Join Operator.""" + + metadata = ViewMetadata( + label=_("Knowledge Process Join Operator"), + name="knowledge_process_join_operator", + category=OperatorCategory.RAG, + operator_type=OperatorType.JOIN, + description=_( + "Join Branch the workflow based on the Knowledge Process Results." + ), + parameters=[], + inputs=[ + IOField.build_from( + _("Vector Storage Results"), + "input_value", + List[Chunk], + description=_("vector storage results."), + is_list=True, + ), + IOField.build_from( + _("Knowledge Graph Storage Results"), + "input_value", + List[Chunk], + description=_("knowledge graph storage results."), + is_list=True, + ), + ], + outputs=[ + IOField.build_from( + _("Chunks"), + "chunks", + List[Chunk], + description=_("Knowledge Process Results."), + is_list=True, + ), + ], + ) + + def __init__( + self, + **kwargs, + ): + """Knowledge Process Join Operator.""" + super().__init__(combine_function=self._join, **kwargs) + + async def _join( + self, + vector_chunks: Optional[List[Chunk]] = None, + graph_chunks: Optional[List[Chunk]] = None, + full_text_chunks: Optional[List[Chunk]] = None, + ) -> List[str]: + """Join results. + + Args: + vector_chunks: The list of vector chunks. + graph_chunks: The list of graph chunks. + full_text_chunks: The list of full text chunks. + """ + results = [] + if vector_chunks: + result_msg = ( + f"async persist vector store success {len(vector_chunks)} chunks." + ) + logger.info(result_msg) + results.append(result_msg) + if graph_chunks: + result_msg = ( + f"async persist graph store success {len(graph_chunks)} chunks." + ) + logger.info(result_msg) + results.append(result_msg) + if full_text_chunks: + result_msg = ( + f"async persist full text store success {len(full_text_chunks)} " + f"chunks." + ) + logger.info(result_msg) + results.append(result_msg) + return results diff --git a/dbgpt/rag/operators/vector_store.py b/dbgpt/rag/operators/vector_store.py new file mode 100644 index 000000000..a3add17e4 --- /dev/null +++ b/dbgpt/rag/operators/vector_store.py @@ -0,0 +1,74 @@ +"""Vector Storage Operator.""" +import os +from typing import List, Optional + +from dbgpt.core import Chunk +from dbgpt.core.awel import MapOperator +from dbgpt.core.awel.flow import IOField, OperatorCategory, Parameter, ViewMetadata +from dbgpt.storage.vector_store.base import VectorStoreBase +from dbgpt.util.i18n_utils import _ + + +class VectorStorageOperator(MapOperator[List[Chunk], List[Chunk]]): + """Vector Storage Operator.""" + + metadata = ViewMetadata( + label=_("Vector Storage Operator"), + name="vector_storage_operator", + description=_("Persist embeddings into vector storage."), + category=OperatorCategory.RAG, + parameters=[ + Parameter.build_from( + _("Vector Store Connector"), + "vector_store", + VectorStoreBase, + description=_("The vector store."), + alias=["vector_store"], + ), + ], + inputs=[ + IOField.build_from( + _("Chunks"), + "chunks", + List[Chunk], + description=_("The text split chunks by chunk manager."), + is_list=True, + ) + ], + outputs=[ + IOField.build_from( + _("Chunks"), + "chunks", + List[Chunk], + description=_( + "The assembled chunks, it has been persisted to vector " "store." + ), + is_list=True, + ) + ], + ) + + def __init__( + self, + vector_store: Optional[VectorStoreBase] = None, + max_chunks_once_load: Optional[int] = None, + **kwargs, + ): + """Init the datasource operator.""" + MapOperator.__init__(self, **kwargs) + self._vector_store = vector_store + self._embeddings = vector_store.get_config().embedding_fn + self._max_chunks_once_load = max_chunks_once_load + self.vector_store = vector_store + + async def map(self, chunks: List[Chunk]) -> List[Chunk]: + """Persist chunks in vector db.""" + max_chunks_once_load = self._max_chunks_once_load or int( + os.getenv("KNOWLEDGE_MAX_CHUNKS_ONCE_LOAD", 10) + ) + vector_ids = await self._vector_store.aload_document_with_limit( + chunks, max_chunks_once_load + ) + for chunk, vector_id in zip(chunks, vector_ids): + chunk.chunk_id = str(vector_id) + return chunks diff --git a/dbgpt/serve/agent/resource/knowledge.py b/dbgpt/serve/agent/resource/knowledge.py index 65c062415..f74133e80 100644 --- a/dbgpt/serve/agent/resource/knowledge.py +++ b/dbgpt/serve/agent/resource/knowledge.py @@ -67,7 +67,9 @@ def __init__(self, name: str, space_name: str, context: Optional[dict] = None): # TODO: Build the retriever in a thread pool, it will block the event loop retriever = KnowledgeSpaceRetriever( space_id=space_name, - top_k=context.get("top_k", None) if context else 4, + top_k=context.get("top_k", None) + if context + else CFG.KNOWLEDGE_SEARCH_TOP_SIZE, ) super().__init__(name, retriever=retriever) diff --git a/dbgpt/serve/flow/templates/en/embedded-knowledge-process-flow-template.json b/dbgpt/serve/flow/templates/en/embedded-knowledge-process-flow-template.json new file mode 100644 index 000000000..a555188e3 --- /dev/null +++ b/dbgpt/serve/flow/templates/en/embedded-knowledge-process-flow-template.json @@ -0,0 +1,852 @@ +{ + "flow": { + "uid": "0a2bb656-4538-45bf-963a-c2101127a767", + "label": "Embedding Knowledge Process Workflow", + "name": "embedding_process_workflow", + "flow_category": null, + "flow_data": { + "nodes": [ + { + "width": 320, + "height": 323, + "id": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "position": { + "x": -25.997695320590083, + "y": -90.04159277333981, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "Vector Storage Operator", + "custom_label": null, + "name": "vector_storage_operator", + "description": "Persist embeddings into vector storage.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "VectorStoreBase", + "type_cls": "dbgpt.storage.vector_store.base.VectorStoreBase", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Vector Storage连接器", + "name": "vector_store", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "The vector store.", + "options": null, + "value": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunk", + "custom_label": null, + "name": "chunks", + "description": "The text split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunk", + "custom_label": null, + "name": "chunks", + "description": "已组装的Chunk，已持久化到Vector Storage中.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "VectorStorageOperator", + "type_cls": "dbgpt.rag.operators.vector_store.VectorStorageOperator" + }, + "position_absolute": { + "x": -25.997695320590083, + "y": -90.04159277333981, + "zoom": 0 + } + }, + { + "width": 320, + "height": 321, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "position": { + "x": -913.571872386726, + "y": -61.6367538649408, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "Chunk Manager Operator", + "custom_label": null, + "name": "chunk_manager_operator", + "description": " Split Knowledge Documents into chunks.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "ChunkParameters", + "type_cls": "dbgpt.rag.chunk_manager.ChunkParameters", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Chunk Split Parameters", + "name": "chunk_parameters", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Chunk Split Parameters.", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "Knowledge", + "type_cls": "dbgpt.rag.knowledge.base.Knowledge", + "label": "Knowledge", + "custom_label": null, + "name": "knowledge", + "description": "The knowledge to be loaded.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunk", + "custom_label": null, + "name": "chunks", + "description": "The split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "ChunkManagerOperator", + "type_cls": "dbgpt.rag.operators.chunk_manager.ChunkManagerOperator" + }, + "position_absolute": { + "x": -913.571872386726, + "y": -61.6367538649408, + "zoom": 0 + } + }, + { + "width": 320, + "height": 234, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "position": { + "x": -256.96257013540503, + "y": -509.98997877383584, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "ChromaStore", + "type_cls": "dbgpt.storage.vector_store.chroma_store.ChromaStore", + "label": "Chroma Vector Store", + "custom_label": null, + "name": "chroma_vector_store", + "description": "Chroma Vector Storage.", + "category": "vector_store", + "category_label": "Vector Store", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "ChromaVectorConfig", + "type_cls": "dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Chroma Config", + "name": "vector_store_config", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "the chroma config of vector store.", + "options": null, + "value": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.vector_store.chroma_store.ChromaStore", + "dbgpt.storage.vector_store.base.VectorStoreBase", + "dbgpt.rag.index.base.IndexStoreBase" + ] + }, + "position_absolute": { + "x": -256.96257013540503, + "y": -509.98997877383584, + "zoom": 0 + } + }, + { + "width": 320, + "height": 674, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "position": { + "x": -731.2095474673597, + "y": -879.5845342539665, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "ChromaVectorConfig", + "type_cls": "dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig", + "label": "Chroma Config", + "custom_label": null, + "name": "chroma_vector_config", + "description": "Chroma vector store config.", + "category": "vector_store", + "category_label": "Vector Store", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "集合名称", + "name": "name", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "dbgpt_collection", + "placeholder": null, + "description": "Vector Storage的名称，如果未设置，将使用默认名称.", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "用户", + "name": "user", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Vector Storage的用户，如果未设置，将使用默认用户.", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "密码", + "name": "password", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Vector Storage的密码，如果未设置，将使用默认密码.", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "Embeddings", + "type_cls": "dbgpt.core.interface.embeddings.Embeddings", + "dynamic": false, + "dynamic_minimum": 0, + "label": "嵌入函数", + "name": "embedding_fn", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Vector Storage的嵌入函数，如果未设置，将使用默认的嵌入函数.", + "options": null, + "value": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "max_chunks_once_load", + "name": "max_chunks_once_load", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 10, + "placeholder": null, + "description": "max_chunks_once_load default 10.", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "max_threads", + "name": "max_threads", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 1, + "placeholder": null, + "description": "max_threads.", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Persist Path", + "name": "persist_path", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Vector Storage Path", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig", + "dbgpt.storage.vector_store.base.VectorStoreConfig", + "dbgpt.rag.index.base.IndexStoreConfig", + "pydantic.main.BaseModel" + ] + }, + "position_absolute": { + "x": -731.2095474673597, + "y": -879.5845342539665, + "zoom": 0 + } + }, + { + "width": 320, + "height": 431, + "id": "operator_knowledge_operator___$$___rag___$$___v1_0", + "position": { + "x": -1517.087378905087, + "y": -191.2030717055229, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "Knowledge Loader Operator", + "custom_label": null, + "name": "knowledge_operator", + "description": "Knowledge Loader Operator.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": "https://github.com/openai/openai-python", + "id": "operator_knowledge_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "默认数据源", + "name": "datasource", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "默认数据源.", + "options": null, + "value": "../../docs/docs/awel/awel.md", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Knowledge Type", + "name": "knowledge_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "DOCUMENT", + "placeholder": null, + "description": "Knowledge Type.", + "options": [ + { + "label": "DOCUMENT", + "name": "DOCUMENT", + "value": "DOCUMENT", + "children": null + }, + { + "label": "URL", + "name": "URL", + "value": "URL", + "children": null + }, + { + "label": "TEXT", + "name": "TEXT", + "value": "TEXT", + "children": null + } + ], + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "dict", + "type_cls": "builtins.dict", + "label": "Knowledge Datasource", + "custom_label": null, + "name": "knowledge datasource", + "description": "Knowledge data source.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "Knowledge", + "type_cls": "dbgpt.rag.knowledge.base.Knowledge", + "label": "Knowledge", + "custom_label": null, + "name": "Knowledge", + "description": "Knowledge.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeOperator", + "type_cls": "dbgpt.rag.operators.knowledge.KnowledgeOperator" + }, + "position_absolute": { + "x": -1517.087378905087, + "y": -191.2030717055229, + "zoom": 0 + } + }, + { + "width": 320, + "height": 602, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "position": { + "x": -2015.3280350941911, + "y": -603.9181210010445, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "Dict HTTP Trigger", + "custom_label": null, + "name": "dict_http_trigger", + "description": "通过 HTTP 请求触发您的工作流，并将请求主体解析为Dict", + "category": "trigger", + "category_label": "Trigger", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "API Endpoint", + "name": "endpoint", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "API Endpoint", + "options": null, + "value": "/rag/knowledge/embedding/process", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP Method", + "name": "methods", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "POST", + "placeholder": null, + "description": "API Method", + "options": [ + { + "label": "HTTP PUT Method", + "name": "http_put", + "value": "PUT", + "children": null + }, + { + "label": "HTTP POST Method", + "name": "http_post", + "value": "POST", + "children": null + } + ], + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "bool", + "type_cls": "builtins.bool", + "dynamic": false, + "dynamic_minimum": 0, + "label": "流式响应", + "name": "streaming_response", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": false, + "placeholder": null, + "description": "响应是否为流式传输", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "BaseHttpBody", + "type_cls": "dbgpt.core.awel.trigger.http_trigger.BaseHttpBody", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP 响应主体", + "name": "http_response_body", + "is_list": false, + "category": "resource", + "resource_type": "class", + "optional": true, + "default": null, + "placeholder": null, + "description": "API 响应主体", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Media Type", + "name": "response_media_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Media Type", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP Status", + "name": "status_code", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 200, + "placeholder": null, + "description": "HTTP Status", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "input", + "inputs": [], + "outputs": [ + { + "type_name": "dict", + "type_cls": "builtins.dict", + "label": "请求体", + "custom_label": null, + "name": "request_body", + "description": "API Body", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "version": "v1", + "type_name": "DictHttpTrigger", + "type_cls": "dbgpt.core.awel.trigger.http_trigger.DictHttpTrigger" + }, + "position_absolute": { + "x": -2015.3280350941911, + "y": -603.9181210010445, + "zoom": 0 + } + }, + { + "width": 320, + "height": 148, + "id": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "position": { + "x": -1297.0596621977236, + "y": -756.4644248292581, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "DefaultEmbeddings", + "type_cls": "dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings", + "label": "DefaultEmbeddings", + "custom_label": null, + "name": "default_embeddings", + "description": "DefaultEmbeddings", + "category": "embeddings", + "category_label": "Embeddings", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings", + "dbgpt.core.interface.embeddings.Embeddings" + ] + }, + "position_absolute": { + "x": -1297.0596621977236, + "y": -756.4644248292581, + "zoom": 0 + } + } + ], + "edges": [ + { + "source": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "source_order": 0, + "target": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0|operator_vector_storage_operator___$$___rag___$$___v1_0", + "source_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0|outputs|0", + "target_handle": "operator_vector_storage_operator___$$___rag___$$___v1_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "source_order": 0, + "target": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "target_order": 0, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0|resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "source_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0|outputs|0", + "target_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "operator_knowledge_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_knowledge_operator___$$___rag___$$___v1_0|operator_chunk_manager_operator___$$___rag___$$___v1_0", + "source_handle": "operator_knowledge_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_chunk_manager_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0|operator_knowledge_operator___$$___rag___$$___v1_0", + "source_handle": "operator_dict_http_trigger___$$___trigger___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "source_order": 0, + "target": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "target_order": 3, + "id": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0|resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "source_handle": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0|outputs|0", + "target_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0|parameters|3", + "type": "buttonedge" + }, + { + "source": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0|operator_vector_storage_operator___$$___rag___$$___v1_0", + "source_handle": "operator_chunk_manager_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_vector_storage_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + } + ], + "viewport": { + "x": 1043.477794539488, + "y": 580.8652141334148, + "zoom": 0.4897130009645833 + } + }, + "description": "Embedding Knowledge Process Workflow", + "state": "running", + "error_message": "", + "source": "DBGPT-WEB", + "source_url": null, + "version": "0.1.1", + "define_type": "json", + "editable": true, + "user_name": null, + "sys_code": null, + "dag_id": "flow_dag_embedding_process_workflow_0a2bb656-4538-45bf-963a-c2101127a767", + "gmt_created": "2024-12-16 18:04:00", + "gmt_modified": "2024-12-16 18:04:00", + "metadata": { + "sse_output": false, + "streaming_output": false, + "tags": {}, + "triggers": [ + { + "trigger_type": "http", + "path": "/api/v1/awel/trigger/rag/embdding/process", + "methods": [ + "POST" + ], + "trigger_mode": "command" + } + ] + }, + "variables": null, + "authors": null +} +} \ No newline at end of file diff --git a/dbgpt/serve/flow/templates/en/hybrid-knowledge-process-template.json b/dbgpt/serve/flow/templates/en/hybrid-knowledge-process-template.json new file mode 100644 index 000000000..2429dba43 --- /dev/null +++ b/dbgpt/serve/flow/templates/en/hybrid-knowledge-process-template.json @@ -0,0 +1,1394 @@ +{ + "flow": { + "uid": "fa8f01ab-fc7b-4e35-9533-f8e4cf045cce", + "label": "Hybrid Knowledge Process Workflow", + "name": "hybrid_knowledge_process_workflow", + "flow_category": null, + "description": "hybrid_knowledge_process_workflow", + "state": "running", + "error_message": "", + "source": "DBGPT-WEB", + "source_url": null, + "version": "0.1.1", + "define_type": "json", + "editable": true, + "user_name": null, + "sys_code": null, + "dag_id": "flow_dag_knowledge_factory_workflow_fa8f01ab-fc7b-4e35-9533-f8e4cf045cce", + "gmt_created": "2024-12-14 15:57:44", + "gmt_modified": "2024-12-14 15:57:44", + "metadata": null, + "variables": null, + "authors": null, + "flow_data": { + "edges": [ + { + "source": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_process_join_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_vector_storage_operator___$$___rag___$$___v1_0|operator_knowledge_process_join_operator___$$___rag___$$___v1_0", + "source_handle": "operator_vector_storage_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_process_join_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_process_join_operator___$$___rag___$$___v1_0", + "target_order": 1, + "id": "operator_knowledge_graph_operator___$$___rag___$$___v1_0|operator_knowledge_process_join_operator___$$___rag___$$___v1_0", + "source_handle": "operator_knowledge_graph_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_process_join_operator___$$___rag___$$___v1_0|inputs|1", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "source_order": 0, + "target": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0|operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "source_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0|outputs|0", + "target_handle": "operator_knowledge_graph_operator___$$___rag___$$___v1_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "source_order": 0, + "target": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "target_order": 0, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0|resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "source_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0|outputs|0", + "target_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "source_order": 0, + "target": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0|operator_vector_storage_operator___$$___rag___$$___v1_0", + "source_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0|outputs|0", + "target_handle": "operator_vector_storage_operator___$$___rag___$$___v1_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "source_order": 0, + "target": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "target_order": 0, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0|resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "source_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0|outputs|0", + "target_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "operator_knowledge_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_knowledge_operator___$$___rag___$$___v1_0|operator_chunk_manager_operator___$$___rag___$$___v1_0", + "source_handle": "operator_knowledge_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_chunk_manager_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0|operator_knowledge_operator___$$___rag___$$___v1_0", + "source_handle": "operator_dict_http_trigger___$$___trigger___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "source_order": 0, + "target": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "target_order": 3, + "id": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0|resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "source_handle": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0|outputs|0", + "target_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0|parameters|3", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "source_order": 0, + "target": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "target_order": 5, + "id": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0|resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "source_handle": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0|outputs|0", + "target_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0|parameters|5", + "type": "buttonedge" + }, + { + "source": "operator_knowledge_process_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_knowledge_process_operator___$$___rag___$$___v1_0|operator_vector_storage_operator___$$___rag___$$___v1_0", + "source_handle": "operator_knowledge_process_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_vector_storage_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "operator_knowledge_process_operator___$$___rag___$$___v1_0", + "source_order": 1, + "target": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_knowledge_process_operator___$$___rag___$$___v1_0|operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "source_handle": "operator_knowledge_process_operator___$$___rag___$$___v1_0|outputs|1", + "target_handle": "operator_knowledge_graph_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_process_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0|operator_knowledge_process_operator___$$___rag___$$___v1_0", + "source_handle": "operator_chunk_manager_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_process_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + } + ], + "viewport": { + "x": 862.7562630728534, + "y": 416.4952851718194, + "zoom": 0.3836649890553727 + }, + "nodes": [ + { + "width": 320, + "height": 275, + "id": "operator_knowledge_process_join_operator___$$___rag___$$___v1_0", + "position": { + "x": 604.5, + "y": 77, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": 604.5, + "y": 77, + "zoom": 0 + }, + "data": { + "label": "Knowledge Process Join Operator", + "custom_label": null, + "name": "knowledge_process_join_operator", + "description": "Join Branch the workflow based on the Knowledge Process Results.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_knowledge_process_join_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "operator_type": "join", + "inputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Knowledge Graph Storage Results", + "custom_label": null, + "name": "input_value", + "description": "knowledge graph storage results.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + }, + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Knowledge Graph Storage Results", + "custom_label": null, + "name": "input_value", + "description": "knowledge graph storage results.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunks", + "custom_label": null, + "name": "chunks", + "description": "Knowledge Process Results.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeProcessJoinOperator", + "type_cls": "dbgpt.rag.operators.process_branch.KnowledgeProcessJoinOperator", + "parameters": [] + } + }, + { + "width": 320, + "height": 323, + "id": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "position": { + "x": 174.16583729394188, + "y": -131.63401513480102, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": 174.16583729394188, + "y": -131.63401513480102, + "zoom": 0 + }, + "data": { + "label": "Vector Storage Operator", + "custom_label": null, + "name": "vector_storage_operator", + "description": "Persist embeddings into vector storage.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "operator_type": "map", + "inputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunks", + "custom_label": null, + "name": "chunks", + "description": "The text split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunks", + "custom_label": null, + "name": "chunks", + "description": "The assembled chunks, it has been persisted to vector store.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "VectorStorageOperator", + "type_cls": "dbgpt.rag.operators.vector_store.VectorStorageOperator", + "parameters": [ + { + "type_name": "VectorStoreBase", + "type_cls": "dbgpt.storage.vector_store.base.VectorStoreBase", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Vector Store Connector", + "name": "vector_store", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "The vector store.", + "value": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "options": null + } + ] + } + }, + { + "width": 320, + "height": 323, + "id": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "position": { + "x": 180.89103763027083, + "y": 341.37174185366564, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": 180.89103763027083, + "y": 341.37174185366564, + "zoom": 0 + }, + "data": { + "label": "Knowledge Graph Operator", + "custom_label": null, + "name": "knowledge_graph_operator", + "description": "Extract Documents and persist into graph database.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "operator_type": "map", + "inputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunks", + "custom_label": null, + "name": "chunks", + "description": "The text split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunks", + "custom_label": null, + "name": "chunks", + "description": "The assembled chunks, it has been persisted to graph store.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeGraphOperator", + "type_cls": "dbgpt.rag.operators.knowledge_graph.KnowledgeGraphOperator", + "parameters": [ + { + "type_name": "KnowledgeGraphBase", + "type_cls": "dbgpt.storage.knowledge_graph.base.KnowledgeGraphBase", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Knowledge Graph Connector", + "name": "graph_store", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "The knowledge graph.", + "value": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "options": null + } + ] + } + }, + { + "width": 320, + "height": 321, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "position": { + "x": -1056.545824254249, + "y": -163.01828337100258, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": -1056.545824254249, + "y": -163.01828337100258, + "zoom": 0 + }, + "data": { + "label": "Chunk Manager Operator", + "custom_label": null, + "name": "chunk_manager_operator", + "description": " Split Knowledge Documents into chunks.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "operator_type": "map", + "inputs": [ + { + "type_name": "Knowledge", + "type_cls": "dbgpt.rag.knowledge.base.Knowledge", + "label": "Knowledge", + "custom_label": null, + "name": "knowledge", + "description": "The knowledge to be loaded.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunks", + "custom_label": null, + "name": "chunks", + "description": "The split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "ChunkManagerOperator", + "type_cls": "dbgpt.rag.operators.chunk_manager.ChunkManagerOperator", + "parameters": [ + { + "type_name": "ChunkParameters", + "type_cls": "dbgpt.rag.chunk_manager.ChunkParameters", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Chunk Split Parameters", + "name": "chunk_parameters", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Chunk Split Parameters.", + "value": null, + "options": null + } + ] + } + }, + { + "width": 320, + "height": 234, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "position": { + "x": -306.391788536534, + "y": 491.0961943850906, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": -306.391788536534, + "y": 491.0961943850906, + "zoom": 0 + }, + "data": { + "type_name": "BuiltinKnowledgeGraph", + "type_cls": "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph", + "label": "Builtin Knowledge Graph", + "custom_label": null, + "name": "builtin_knowledge_graph", + "description": "Builtin Knowledge Graph.", + "category": "knowledge_graph", + "category_label": "Knowledge Graph", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "tags": { + "ui_version": "flow2.0" + }, + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph", + "dbgpt.storage.knowledge_graph.base.KnowledgeGraphBase", + "dbgpt.rag.index.base.IndexStoreBase" + ], + "parameters": [ + { + "type_name": "BuiltinKnowledgeGraphConfig", + "type_cls": "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Builtin Knowledge Graph Config.", + "name": "config", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Builtin Knowledge Graph Config.", + "value": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "options": null + } + ] + } + }, + { + "width": 320, + "height": 645, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "position": { + "x": -813.7432345424928, + "y": 328.2957752754239, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": -813.7432345424928, + "y": 328.2957752754239, + "zoom": 0 + }, + "data": { + "type_name": "BuiltinKnowledgeGraphConfig", + "type_cls": "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig", + "label": "Builtin Graph Config", + "custom_label": null, + "name": "knowledge_graph_config", + "description": "knowledge graph config.", + "category": "knowledge_graph", + "category_label": "Knowledge Graph", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "tags": { + "ui_version": "flow2.0" + }, + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig", + "dbgpt.storage.knowledge_graph.base.KnowledgeGraphConfig", + "dbgpt.rag.index.base.IndexStoreConfig", + "pydantic.main.BaseModel" + ], + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Graph Name", + "name": "name", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "dbgpt_collection", + "placeholder": null, + "description": "The name of Graph, if not set, will use the default name.", + "value": "dbgpt_collection_V1", + "options": null + }, + { + "type_name": "Embeddings", + "type_cls": "dbgpt.core.interface.embeddings.Embeddings", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Embedding Function", + "name": "embedding_fn", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "The embedding function of vector store, if not set, will use the default embedding function.", + "value": null, + "options": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Max Chunks Once Load", + "name": "max_chunks_once_load", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 10, + "placeholder": null, + "description": "The max number of chunks to load at once. If your document is large, you can set this value to a larger number to speed up the loading process. Default is 10.", + "value": null, + "options": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Max Threads", + "name": "max_threads", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 1, + "placeholder": null, + "description": "The max number of threads to use. Default is 1. If you set this bigger than 1, please make sure your vector store is thread-safe.", + "value": null, + "options": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Knowledge Graph Type", + "name": "graph_store_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "TuGraph", + "placeholder": null, + "description": "graph store type.", + "value": "TuGraph", + "options": null + }, + { + "type_name": "LLMClient", + "type_cls": "dbgpt.core.interface.llm.LLMClient", + "dynamic": false, + "dynamic_minimum": 0, + "label": "LLM Client", + "name": "llm_client", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "llm client for extract graph triplets.", + "value": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "options": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "LLM Model Name", + "name": "model_name", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "llm model name.", + "value": "zhipu_proxyllm", + "options": null + } + ] + } + }, + { + "width": 320, + "height": 234, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "position": { + "x": -251.7635173402225, + "y": -367.01602690631285, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": -251.7635173402225, + "y": -367.01602690631285, + "zoom": 0 + }, + "data": { + "type_name": "ChromaStore", + "type_cls": "dbgpt.storage.vector_store.chroma_store.ChromaStore", + "label": "Chroma Vector Store", + "custom_label": null, + "name": "chroma_vector_store", + "description": "Chroma vector store.", + "category": "vector_store", + "category_label": "Vector Store", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "tags": { + "ui_version": "flow2.0" + }, + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.vector_store.chroma_store.ChromaStore", + "dbgpt.storage.vector_store.base.VectorStoreBase", + "dbgpt.rag.index.base.IndexStoreBase" + ], + "parameters": [ + { + "type_name": "ChromaVectorConfig", + "type_cls": "dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Chroma Config", + "name": "vector_store_config", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "the chroma config of vector store.", + "value": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "options": null + } + ] + } + }, + { + "width": 320, + "height": 674, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "position": { + "x": -684.4180723107158, + "y": -934.1745886033843, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": -684.4180723107158, + "y": -934.1745886033843, + "zoom": 0 + }, + "data": { + "type_name": "ChromaVectorConfig", + "type_cls": "dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig", + "label": "Chroma Config", + "custom_label": null, + "name": "chroma_vector_config", + "description": "Chroma vector store config.", + "category": "vector_store", + "category_label": "Vector Store", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "tags": { + "ui_version": "flow2.0" + }, + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig", + "dbgpt.storage.vector_store.base.VectorStoreConfig", + "dbgpt.rag.index.base.IndexStoreConfig", + "pydantic.main.BaseModel" + ], + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Collection Name", + "name": "name", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "dbgpt_collection", + "placeholder": null, + "description": "The name of vector store, if not set, will use the default name.", + "value": null, + "options": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "User", + "name": "user", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "The user of vector store, if not set, will use the default user.", + "value": null, + "options": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Password", + "name": "password", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "The password of vector store, if not set, will use the default password.", + "value": null, + "options": null + }, + { + "type_name": "Embeddings", + "type_cls": "dbgpt.core.interface.embeddings.Embeddings", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Embedding Function", + "name": "embedding_fn", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "The embedding function of vector store, if not set, will use the default embedding function.", + "value": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "options": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Max Chunks Once Load", + "name": "max_chunks_once_load", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 10, + "placeholder": null, + "description": "The max number of chunks to load at once. If your document is large, you can set this value to a larger number to speed up the loading process. Default is 10.", + "value": null, + "options": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Max Threads", + "name": "max_threads", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 1, + "placeholder": null, + "description": "The max number of threads to use. Default is 1. If you set this bigger than 1, please make sure your vector store is thread-safe.", + "value": null, + "options": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Persist Path", + "name": "persist_path", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "the persist path of vector store.", + "value": null, + "options": null + } + ] + } + }, + { + "width": 320, + "height": 431, + "id": "operator_knowledge_operator___$$___rag___$$___v1_0", + "position": { + "x": -1558.679801266548, + "y": -149.61064934406164, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": -1558.679801266548, + "y": -149.61064934406164, + "zoom": 0 + }, + "data": { + "label": "Knowledge Loader Operator", + "custom_label": null, + "name": "knowledge_operator", + "description": "The knowledge operator, which can create knowledge from datasource.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": "https://github.com/openai/openai-python", + "id": "operator_knowledge_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "operator_type": "map", + "inputs": [ + { + "type_name": "dict", + "type_cls": "builtins.dict", + "label": "knowledge datasource", + "custom_label": null, + "name": "knowledge datasource", + "description": "knowledge datasource, which can be a document, url, or text.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "Knowledge", + "type_cls": "dbgpt.rag.knowledge.base.Knowledge", + "label": "Knowledge", + "custom_label": null, + "name": "Knowledge", + "description": "Knowledge object.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeOperator", + "type_cls": "dbgpt.rag.operators.knowledge.KnowledgeOperator", + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Default datasource", + "name": "datasource", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Default datasource.", + "value": "../../docs/docs/awel/awel.md", + "options": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Knowledge type", + "name": "knowledge_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "DOCUMENT", + "placeholder": null, + "description": "Knowledge type.", + "value": null, + "options": [ + { + "label": "DOCUMENT", + "name": "DOCUMENT", + "value": "DOCUMENT", + "children": null + }, + { + "label": "URL", + "name": "URL", + "value": "URL", + "children": null + }, + { + "label": "TEXT", + "name": "TEXT", + "value": "TEXT", + "children": null + } + ] + } + ] + } + }, + { + "width": 320, + "height": 602, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "position": { + "x": -2015.3280350941911, + "y": -603.9181210010445, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": -2015.3280350941911, + "y": -603.9181210010445, + "zoom": 0 + }, + "data": { + "label": "Dict Http Trigger", + "custom_label": null, + "name": "dict_http_trigger", + "description": "Trigger your workflow by http request, and parse the request body as a dict", + "category": "trigger", + "category_label": "Trigger", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "operator_type": "input", + "inputs": [], + "outputs": [ + { + "type_name": "dict", + "type_cls": "builtins.dict", + "label": "Request Body", + "custom_label": null, + "name": "request_body", + "description": "The request body of the API endpoint", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "version": "v1", + "type_name": "DictHttpTrigger", + "type_cls": "dbgpt.core.awel.trigger.http_trigger.DictHttpTrigger", + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "API Endpoint", + "name": "endpoint", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "The API endpoint", + "value": "/rag/knowledge/hybrid/process", + "options": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Http Methods", + "name": "methods", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "POST", + "placeholder": null, + "description": "The methods of the API endpoint", + "value": null, + "options": [ + { + "label": "HTTP Method PUT", + "name": "http_put", + "value": "PUT", + "children": null + }, + { + "label": "HTTP Method POST", + "name": "http_post", + "value": "POST", + "children": null + } + ] + }, + { + "type_name": "bool", + "type_cls": "builtins.bool", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Streaming Response", + "name": "streaming_response", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": false, + "placeholder": null, + "description": "Whether the response is streaming", + "value": null, + "options": null + }, + { + "type_name": "BaseHttpBody", + "type_cls": "dbgpt.core.awel.trigger.http_trigger.BaseHttpBody", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Http Response Body", + "name": "http_response_body", + "is_list": false, + "category": "resource", + "resource_type": "class", + "optional": true, + "default": null, + "placeholder": null, + "description": "The response body of the API endpoint", + "value": null, + "options": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Response Media Type", + "name": "response_media_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "The response media type", + "value": null, + "options": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Http Status Code", + "name": "status_code", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 200, + "placeholder": null, + "description": "The http status code", + "value": null, + "options": null + } + ] + } + }, + { + "width": 320, + "height": 148, + "id": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "position": { + "x": -1112.4932879687394, + "y": -753.8648984316667, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": -1112.4932879687394, + "y": -753.8648984316667, + "zoom": 0 + }, + "data": { + "type_name": "DefaultEmbeddings", + "type_cls": "dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings", + "label": "Default Embeddings", + "custom_label": null, + "name": "default_embeddings", + "description": "Default embeddings(using default embedding model of current system)", + "category": "embeddings", + "category_label": "Embeddings", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "tags": { + "ui_version": "flow2.0" + }, + "resource_type": "instance", + "parent_cls": [ + "dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings", + "dbgpt.core.interface.embeddings.Embeddings" + ], + "parameters": [] + } + }, + { + "width": 320, + "height": 272, + "id": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "position": { + "x": -1350.535131696419, + "y": 435.2340305150391, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": -1350.535131696419, + "y": 435.2340305150391, + "zoom": 0 + }, + "data": { + "type_name": "DefaultLLMClient", + "type_cls": "dbgpt.model.cluster.client.DefaultLLMClient", + "label": "Default LLM Client", + "custom_label": null, + "name": "default_llm_client", + "description": "Default LLM client(Connect to your DB-GPT model serving)", + "category": "llm_client", + "category_label": "LLM Client", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "tags": { + "ui_version": "flow2.0" + }, + "resource_type": "instance", + "parent_cls": [ + "dbgpt.model.cluster.client.DefaultLLMClient", + "dbgpt.core.interface.llm.LLMClient" + ], + "parameters": [ + { + "type_name": "bool", + "type_cls": "builtins.bool", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Auto Convert Message", + "name": "auto_convert_message", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": true, + "placeholder": null, + "description": "Whether to auto convert the messages that are not supported by the LLM to a compatible format", + "value": null, + "options": null + } + ] + } + }, + { + "width": 320, + "height": 295, + "id": "operator_knowledge_process_operator___$$___rag___$$___v1_0", + "position": { + "x": -614.4846931519926, + "y": -92.5163519851338, + "zoom": 0 + }, + "type": "customNode", + "position_absolute": { + "x": -614.4846931519926, + "y": -92.5163519851338, + "zoom": 0 + }, + "data": { + "label": "Knowledge Process Branch Operator", + "custom_label": null, + "name": "knowledge_process_operator", + "description": "Branch the workflow based on the stream flag of the request.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_knowledge_process_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "operator_type": "branch", + "inputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Document Chunks", + "custom_label": null, + "name": "input_value", + "description": "The input value of the operator.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunks", + "custom_label": null, + "name": "chunks", + "description": "Chunks for Full Text Connector.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + }, + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunks", + "custom_label": null, + "name": "chunks", + "description": "Chunks for Full Text Connector.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + }, + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunks", + "custom_label": null, + "name": "chunks", + "description": "Chunks for Full Text Connector.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeProcessBranchOperator", + "type_cls": "dbgpt.rag.operators.process_branch.KnowledgeProcessBranchOperator", + "parameters": [] + } + } + ] + } + } +} diff --git a/dbgpt/serve/flow/templates/en/kg-knowledge-process-flow-template.json b/dbgpt/serve/flow/templates/en/kg-knowledge-process-flow-template.json new file mode 100644 index 000000000..3d0d2fa1a --- /dev/null +++ b/dbgpt/serve/flow/templates/en/kg-knowledge-process-flow-template.json @@ -0,0 +1,872 @@ +{ + "flow": { + "uid": "e288c1c5-67f2-4dba-8288-99de289392b2", + "label": "Knowledge Graph Process Workflow", + "name": "knowledge_graph_process_workflow", + "flow_category": null, + "flow_data": { + "nodes": [ + { + "width": 320, + "height": 323, + "id": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "position": { + "x": 6.722768991652174, + "y": -225.32501282124363, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "Knowledge Graph Operator", + "custom_label": null, + "name": "knowledge_graph_operator", + "description": "Extract Documents and persist into graph database.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "KnowledgeGraphBase", + "type_cls": "dbgpt.storage.knowledge_graph.base.KnowledgeGraphBase", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Knowledge Graph Connector", + "name": "graph_store", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "The knowledge graph.", + "options": null, + "value": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunk", + "custom_label": null, + "name": "chunks", + "description": "The text split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunk", + "custom_label": null, + "name": "chunks", + "description": "已组装的块，已持久化到向量存储中。", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeGraphOperator", + "type_cls": "dbgpt.rag.operators.knowledge_graph.KnowledgeGraphOperator" + }, + "position_absolute": { + "x": 6.722768991652174, + "y": -225.32501282124363, + "zoom": 0 + } + }, + { + "width": 320, + "height": 321, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "position": { + "x": -812.1903428806644, + "y": -415.17234393736123, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "Chunk Manager Operator", + "custom_label": null, + "name": "chunk_manager_operator", + "description": " Split Knowledge Documents into chunks.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "ChunkParameters", + "type_cls": "dbgpt.rag.chunk_manager.ChunkParameters", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Chunk Split Parameters", + "name": "chunk_parameters", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Chunk Split Parameters.", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "Knowledge", + "type_cls": "dbgpt.rag.knowledge.base.Knowledge", + "label": "知识", + "custom_label": null, + "name": "knowledge", + "description": "The knowledge to be loaded.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Chunk", + "custom_label": null, + "name": "chunks", + "description": "The split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "ChunkManagerOperator", + "type_cls": "dbgpt.rag.operators.chunk_manager.ChunkManagerOperator" + }, + "position_absolute": { + "x": -812.1903428806644, + "y": -415.17234393736123, + "zoom": 0 + } + }, + { + "width": 320, + "height": 234, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "position": { + "x": -446.7662140064656, + "y": 116.76439313193941, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "BuiltinKnowledgeGraph", + "type_cls": "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph", + "label": "Builtin Knowledge Graph", + "custom_label": null, + "name": "builtin_knowledge_graph", + "description": "Builtin Knowledge Graph.", + "category": "knowledge_graph", + "category_label": "Knowledge Graph", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "BuiltinKnowledgeGraphConfig", + "type_cls": "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Builtin Knowledge Graph Config.", + "name": "config", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Builtin Knowledge Graph Config.", + "options": null, + "value": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph", + "dbgpt.storage.knowledge_graph.base.KnowledgeGraphBase", + "dbgpt.rag.index.base.IndexStoreBase" + ] + }, + "position_absolute": { + "x": -446.7662140064656, + "y": 116.76439313193941, + "zoom": 0 + } + }, + { + "width": 320, + "height": 645, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "position": { + "x": -915.1247640485547, + "y": 148.92845384162234, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "BuiltinKnowledgeGraphConfig", + "type_cls": "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig", + "label": "Builtin Graph Config", + "custom_label": null, + "name": "knowledge_graph_config", + "description": "knowledge graph config.", + "category": "knowledge_graph", + "category_label": "Knowledge Graph", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Graph Name", + "name": "name", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "dbgpt_collection", + "placeholder": null, + "description": "The name of Graph, if not set, will use the default name.", + "options": null, + "value": "dbgpt_collection_V1", + "alias": null, + "ui": null + }, + { + "type_name": "Embeddings", + "type_cls": "dbgpt.core.interface.embeddings.Embeddings", + "dynamic": false, + "dynamic_minimum": 0, + "label": "嵌入函数", + "name": "embedding_fn", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "向量存储的嵌入函数，如果未设置，将使用默认的嵌入函数。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "一次加载的最大块数", + "name": "max_chunks_once_load", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 10, + "placeholder": null, + "description": "一次加载的最大块数。如果您的文档很大，可以将此值设置为较大的数字，以加快加载过程。默认值为 10。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "最大线程数", + "name": "max_threads", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 1, + "placeholder": null, + "description": "要使用的最大线程数。默认值为 1。如果您将此值设置为大于 1，请确保您的向量存储是线程安全的。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Knowledge Graph Type", + "name": "graph_store_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "graph store type.", + "options": null, + "value": "TuGraph", + "alias": null, + "ui": null + }, + { + "type_name": "LLMClient", + "type_cls": "dbgpt.core.interface.llm.LLMClient", + "dynamic": false, + "dynamic_minimum": 0, + "label": "LLM Client", + "name": "llm_client", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "llm client for extract graph triplets.", + "options": null, + "value": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "LLM Model Name", + "name": "model_name", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "llm model name.", + "options": null, + "value": "zhipu_proxyllm", + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig", + "dbgpt.storage.knowledge_graph.base.KnowledgeGraphConfig", + "dbgpt.rag.index.base.IndexStoreConfig", + "pydantic.main.BaseModel" + ] + }, + "position_absolute": { + "x": -915.1247640485547, + "y": 148.92845384162234, + "zoom": 0 + } + }, + { + "width": 320, + "height": 431, + "id": "operator_knowledge_operator___$$___rag___$$___v1_0", + "position": { + "x": -1381.9120062303377, + "y": -370.57039313932444, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "Document Knowledge Loader Operator", + "custom_label": null, + "name": "knowledge_operator", + "description": "Load knowledge from document.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": "https://github.com/openai/openai-python", + "id": "operator_knowledge_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Data Source", + "name": "datasource", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "默认数据源。", + "options": null, + "value": "../../docs/docs/awel/awel.md", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Knowledge Type", + "name": "knowledge_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "DOCUMENT", + "placeholder": null, + "description": "Knowledge Type", + "options": [ + { + "label": "DOCUMENT", + "name": "DOCUMENT", + "value": "DOCUMENT", + "children": null + }, + { + "label": "URL", + "name": "URL", + "value": "URL", + "children": null + }, + { + "label": "TEXT", + "name": "TEXT", + "value": "TEXT", + "children": null + } + ], + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "dict", + "type_cls": "builtins.dict", + "label": "Knowledge data source.", + "custom_label": null, + "name": "knowledge datasource", + "description": "Knowledge data source.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "Knowledge", + "type_cls": "dbgpt.rag.knowledge.base.Knowledge", + "label": "Knowledge", + "custom_label": null, + "name": "Knowledge", + "description": "Knowledge", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeOperator", + "type_cls": "dbgpt.rag.operators.knowledge.KnowledgeOperator" + }, + "position_absolute": { + "x": -1381.9120062303377, + "y": -370.57039313932444, + "zoom": 0 + } + }, + { + "width": 320, + "height": 602, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "position": { + "x": -2020.527087889374, + "y": -445.3470107479735, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "Dict HTTP Trigger", + "custom_label": null, + "name": "dict_http_trigger", + "description": "Dict HTTP Trigger.", + "category": "trigger", + "category_label": "Trigger", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "API 端点", + "name": "endpoint", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "Endpoint", + "options": null, + "value": "/rag/knowledge/kg/process", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP Method", + "name": "methods", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "POST", + "placeholder": null, + "description": "API Endpoint", + "options": [ + { + "label": "HTTP PUT Method", + "name": "http_put", + "value": "PUT", + "children": null + }, + { + "label": "HTTP POST Method", + "name": "http_post", + "value": "POST", + "children": null + } + ], + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "bool", + "type_cls": "builtins.bool", + "dynamic": false, + "dynamic_minimum": 0, + "label": "流式响应", + "name": "streaming_response", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": false, + "placeholder": null, + "description": "Is Stream", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "BaseHttpBody", + "type_cls": "dbgpt.core.awel.trigger.http_trigger.BaseHttpBody", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP 响应主体", + "name": "http_response_body", + "is_list": false, + "category": "resource", + "resource_type": "class", + "optional": true, + "default": null, + "placeholder": null, + "description": "API Http Body", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "响应媒体类型", + "name": "response_media_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Media", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP Status", + "name": "status_code", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 200, + "placeholder": null, + "description": "HTTP Status", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "input", + "inputs": [], + "outputs": [ + { + "type_name": "dict", + "type_cls": "builtins.dict", + "label": "请求体", + "custom_label": null, + "name": "request_body", + "description": "API Http Body", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "version": "v1", + "type_name": "DictHttpTrigger", + "type_cls": "dbgpt.core.awel.trigger.http_trigger.DictHttpTrigger" + }, + "position_absolute": { + "x": -2020.527087889374, + "y": -445.3470107479735, + "zoom": 0 + } + }, + { + "width": 320, + "height": 272, + "id": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "position": { + "x": -1506.5067155518987, + "y": 313.0562898282468, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "DefaultLLMClient", + "type_cls": "dbgpt.model.cluster.client.DefaultLLMClient", + "label": "默认 LLM 客户端", + "custom_label": null, + "name": "default_llm_client", + "description": "默认 LLM 客户端（连接到您的 DB-GPT 模型服务）", + "category": "llm_client", + "category_label": "LLM Client", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "bool", + "type_cls": "builtins.bool", + "dynamic": false, + "dynamic_minimum": 0, + "label": "自动转换消息", + "name": "auto_convert_message", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": true, + "placeholder": null, + "description": "是否将 LLM 不支持的消息自动转换为兼容格式", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.model.cluster.client.DefaultLLMClient", + "dbgpt.core.interface.llm.LLMClient" + ] + }, + "position_absolute": { + "x": -1506.5067155518987, + "y": 313.0562898282468, + "zoom": 0 + } + } + ], + "edges": [ + { + "source": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "source_order": 0, + "target": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0|operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "source_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0|outputs|0", + "target_handle": "operator_knowledge_graph_operator___$$___rag___$$___v1_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "source_order": 0, + "target": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "target_order": 0, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0|resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "source_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0|outputs|0", + "target_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "operator_knowledge_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_knowledge_operator___$$___rag___$$___v1_0|operator_chunk_manager_operator___$$___rag___$$___v1_0", + "source_handle": "operator_knowledge_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_chunk_manager_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0|operator_knowledge_operator___$$___rag___$$___v1_0", + "source_handle": "operator_dict_http_trigger___$$___trigger___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "source_order": 0, + "target": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "target_order": 5, + "id": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0|resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "source_handle": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0|outputs|0", + "target_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0|parameters|5", + "type": "buttonedge" + }, + { + "source": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0|operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "source_handle": "operator_chunk_manager_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_graph_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + } + ], + "viewport": { + "x": 1030.0389432865575, + "y": 345.02642518836063, + "zoom": 0.4818017509272822 + } + }, + "description": "Knowledge Graph Process Workflow", + "state": "running", + "error_message": "", + "source": "DBGPT-WEB", + "source_url": null, + "version": "0.1.1", + "define_type": "json", + "editable": true, + "user_name": null, + "sys_code": null, + "dag_id": "flow_dag_knowledge_graph_process_workflow_e288c1c5-67f2-4dba-8288-99de289392b2", + "gmt_created": "2024-12-16 19:14:00", + "gmt_modified": "2024-12-16 19:14:00", + "metadata": { + "sse_output": false, + "streaming_output": false, + "tags": {}, + "triggers": [ + { + "trigger_type": "http", + "path": "/api/v1/awel/trigger/rag/graph/process", + "methods": [ + "POST" + ], + "trigger_mode": "command" + } + ] + }, + "variables": null, + "authors": null +} +} \ No newline at end of file diff --git a/dbgpt/serve/flow/templates/zh/embedded-knowledge-process-flow-template.json b/dbgpt/serve/flow/templates/zh/embedded-knowledge-process-flow-template.json new file mode 100644 index 000000000..84f83f3a5 --- /dev/null +++ b/dbgpt/serve/flow/templates/zh/embedded-knowledge-process-flow-template.json @@ -0,0 +1,852 @@ +{ + "flow": { + "uid": "04696207-4f91-4e7e-b70c-404ed6657f92", + "label": "Embedding 向量加工工作流", + "name": "embedding_process_workflow", + "flow_category": null, + "flow_data": { + "nodes": [ + { + "width": 320, + "height": 323, + "id": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "position": { + "x": -25.997695320590083, + "y": -90.04159277333981, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "向量抽取存储算子", + "custom_label": null, + "name": "vector_storage_operator", + "description": "Persist embeddings into vector storage.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "VectorStoreBase", + "type_cls": "dbgpt.storage.vector_store.base.VectorStoreBase", + "dynamic": false, + "dynamic_minimum": 0, + "label": "向量存储连接器", + "name": "vector_store", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "The vector store.", + "options": null, + "value": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "The text split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "已组装的块，已持久化到向量存储中。", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "VectorStorageOperator", + "type_cls": "dbgpt.rag.operators.vector_store.VectorStorageOperator" + }, + "position_absolute": { + "x": -25.997695320590083, + "y": -90.04159277333981, + "zoom": 0 + } + }, + { + "width": 320, + "height": 321, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "position": { + "x": -913.571872386726, + "y": -61.6367538649408, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "文档Chunk切片算子", + "custom_label": null, + "name": "chunk_manager_operator", + "description": " Split Knowledge Documents into chunks.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "ChunkParameters", + "type_cls": "dbgpt.rag.chunk_manager.ChunkParameters", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Chunk Split Parameters", + "name": "chunk_parameters", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Chunk Split Parameters.", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "Knowledge", + "type_cls": "dbgpt.rag.knowledge.base.Knowledge", + "label": "知识", + "custom_label": null, + "name": "knowledge", + "description": "The knowledge to be loaded.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "The split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "ChunkManagerOperator", + "type_cls": "dbgpt.rag.operators.chunk_manager.ChunkManagerOperator" + }, + "position_absolute": { + "x": -913.571872386726, + "y": -61.6367538649408, + "zoom": 0 + } + }, + { + "width": 320, + "height": 234, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "position": { + "x": -256.96257013540503, + "y": -509.98997877383584, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "ChromaStore", + "type_cls": "dbgpt.storage.vector_store.chroma_store.ChromaStore", + "label": "Chroma Vector Store", + "custom_label": null, + "name": "chroma_vector_store", + "description": "Chroma 向量存储。", + "category": "vector_store", + "category_label": "Vector Store", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "ChromaVectorConfig", + "type_cls": "dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Chroma Config", + "name": "vector_store_config", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "the chroma config of vector store.", + "options": null, + "value": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.vector_store.chroma_store.ChromaStore", + "dbgpt.storage.vector_store.base.VectorStoreBase", + "dbgpt.rag.index.base.IndexStoreBase" + ] + }, + "position_absolute": { + "x": -256.96257013540503, + "y": -509.98997877383584, + "zoom": 0 + } + }, + { + "width": 320, + "height": 674, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "position": { + "x": -731.2095474673597, + "y": -879.5845342539665, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "ChromaVectorConfig", + "type_cls": "dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig", + "label": "Chroma Config", + "custom_label": null, + "name": "chroma_vector_config", + "description": "Chroma vector store config.", + "category": "vector_store", + "category_label": "Vector Store", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "集合名称", + "name": "name", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "dbgpt_collection", + "placeholder": null, + "description": "向量存储的名称，如果未设置，将使用默认名称。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "用户", + "name": "user", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "向量存储的用户，如果未设置，将使用默认用户。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "密码", + "name": "password", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "向量存储的密码，如果未设置，将使用默认密码。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "Embeddings", + "type_cls": "dbgpt.core.interface.embeddings.Embeddings", + "dynamic": false, + "dynamic_minimum": 0, + "label": "嵌入函数", + "name": "embedding_fn", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "向量存储的嵌入函数，如果未设置，将使用默认的嵌入函数。", + "options": null, + "value": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "一次加载的最大块数", + "name": "max_chunks_once_load", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 10, + "placeholder": null, + "description": "一次加载的最大块数。如果您的文档很大，可以将此值设置为较大的数字，以加快加载过程。默认值为 10。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "最大线程数", + "name": "max_threads", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 1, + "placeholder": null, + "description": "要使用的最大线程数。默认值为 1。如果您将此值设置为大于 1，请确保您的向量存储是线程安全的。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Persist Path", + "name": "persist_path", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "向量存储的持久化路径。", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig", + "dbgpt.storage.vector_store.base.VectorStoreConfig", + "dbgpt.rag.index.base.IndexStoreConfig", + "pydantic.main.BaseModel" + ] + }, + "position_absolute": { + "x": -731.2095474673597, + "y": -879.5845342539665, + "zoom": 0 + } + }, + { + "width": 320, + "height": 431, + "id": "operator_knowledge_operator___$$___rag___$$___v1_0", + "position": { + "x": -1517.087378905087, + "y": -191.2030717055229, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "文档知识加载算子", + "custom_label": null, + "name": "knowledge_operator", + "description": "知识算子，可以从数据源创建知识。", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": "https://github.com/openai/openai-python", + "id": "operator_knowledge_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "默认数据源", + "name": "datasource", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "默认数据源。", + "options": null, + "value": "../../docs/docs/awel/awel.md", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "知识类型", + "name": "knowledge_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "DOCUMENT", + "placeholder": null, + "description": "知识类型。", + "options": [ + { + "label": "DOCUMENT", + "name": "DOCUMENT", + "value": "DOCUMENT", + "children": null + }, + { + "label": "URL", + "name": "URL", + "value": "URL", + "children": null + }, + { + "label": "TEXT", + "name": "TEXT", + "value": "TEXT", + "children": null + } + ], + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "dict", + "type_cls": "builtins.dict", + "label": "知识数据源", + "custom_label": null, + "name": "knowledge datasource", + "description": "知识数据源，可以是文档、网址或文本。", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "Knowledge", + "type_cls": "dbgpt.rag.knowledge.base.Knowledge", + "label": "知识", + "custom_label": null, + "name": "Knowledge", + "description": "知识对象。", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeOperator", + "type_cls": "dbgpt.rag.operators.knowledge.KnowledgeOperator" + }, + "position_absolute": { + "x": -1517.087378905087, + "y": -191.2030717055229, + "zoom": 0 + } + }, + { + "width": 320, + "height": 602, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "position": { + "x": -2015.3280350941911, + "y": -603.9181210010445, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "字典 HTTP 触发器", + "custom_label": null, + "name": "dict_http_trigger", + "description": "通过 HTTP 请求触发您的工作流，并将请求主体解析为字典", + "category": "trigger", + "category_label": "Trigger", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "API 端点", + "name": "endpoint", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "API 端点", + "options": null, + "value": "/rag/knowledge/embedding/process", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP 方法", + "name": "methods", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "POST", + "placeholder": null, + "description": "API 端点的方法", + "options": [ + { + "label": "HTTP PUT 方法", + "name": "http_put", + "value": "PUT", + "children": null + }, + { + "label": "HTTP POST 方法", + "name": "http_post", + "value": "POST", + "children": null + } + ], + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "bool", + "type_cls": "builtins.bool", + "dynamic": false, + "dynamic_minimum": 0, + "label": "流式响应", + "name": "streaming_response", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": false, + "placeholder": null, + "description": "响应是否为流式传输", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "BaseHttpBody", + "type_cls": "dbgpt.core.awel.trigger.http_trigger.BaseHttpBody", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP 响应主体", + "name": "http_response_body", + "is_list": false, + "category": "resource", + "resource_type": "class", + "optional": true, + "default": null, + "placeholder": null, + "description": "API 端点的响应主体", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "响应媒体类型", + "name": "response_media_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "响应的媒体类型", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP 状态码", + "name": "status_code", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 200, + "placeholder": null, + "description": "HTTP 状态码", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "input", + "inputs": [], + "outputs": [ + { + "type_name": "dict", + "type_cls": "builtins.dict", + "label": "请求体", + "custom_label": null, + "name": "request_body", + "description": "API 端点的请求主体", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "version": "v1", + "type_name": "DictHttpTrigger", + "type_cls": "dbgpt.core.awel.trigger.http_trigger.DictHttpTrigger" + }, + "position_absolute": { + "x": -2015.3280350941911, + "y": -603.9181210010445, + "zoom": 0 + } + }, + { + "width": 320, + "height": 148, + "id": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "position": { + "x": -1297.0596621977236, + "y": -756.4644248292581, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "DefaultEmbeddings", + "type_cls": "dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings", + "label": "默认Embedding转换", + "custom_label": null, + "name": "default_embeddings", + "description": "默认嵌入式（使用当前系统的默认嵌入式模型）", + "category": "embeddings", + "category_label": "Embeddings", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings", + "dbgpt.core.interface.embeddings.Embeddings" + ] + }, + "position_absolute": { + "x": -1297.0596621977236, + "y": -756.4644248292581, + "zoom": 0 + } + } + ], + "edges": [ + { + "source": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "source_order": 0, + "target": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0|operator_vector_storage_operator___$$___rag___$$___v1_0", + "source_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0|outputs|0", + "target_handle": "operator_vector_storage_operator___$$___rag___$$___v1_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "source_order": 0, + "target": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "target_order": 0, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0|resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "source_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0|outputs|0", + "target_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "operator_knowledge_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_knowledge_operator___$$___rag___$$___v1_0|operator_chunk_manager_operator___$$___rag___$$___v1_0", + "source_handle": "operator_knowledge_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_chunk_manager_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0|operator_knowledge_operator___$$___rag___$$___v1_0", + "source_handle": "operator_dict_http_trigger___$$___trigger___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "source_order": 0, + "target": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "target_order": 3, + "id": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0|resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "source_handle": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0|outputs|0", + "target_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0|parameters|3", + "type": "buttonedge" + }, + { + "source": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0|operator_vector_storage_operator___$$___rag___$$___v1_0", + "source_handle": "operator_chunk_manager_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_vector_storage_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + } + ], + "viewport": { + "x": 831.8128405437491, + "y": 421.4753242151554, + "zoom": 0.3846854569072972 + } + }, + "description": "Embedding知识加工工作流", + "state": "running", + "error_message": "", + "source": "DBGPT-WEB", + "source_url": null, + "version": "0.1.1", + "define_type": "json", + "editable": true, + "user_name": null, + "sys_code": null, + "dag_id": "flow_dag_embedding_process_workflow_04696207-4f91-4e7e-b70c-404ed6657f92", + "gmt_created": "2024-12-16 15:20:10", + "gmt_modified": "2024-12-16 15:20:10", + "metadata": { + "sse_output": false, + "streaming_output": false, + "tags": {}, + "triggers": [ + { + "trigger_type": "http", + "path": "/api/v1/awel/trigger/rag/embdding/process", + "methods": [ + "POST" + ], + "trigger_mode": "command" + } + ] + }, + "variables": null, + "authors": null +} +} \ No newline at end of file diff --git a/dbgpt/serve/flow/templates/zh/hybrid-knowldge-process-flow-template.json b/dbgpt/serve/flow/templates/zh/hybrid-knowldge-process-flow-template.json new file mode 100644 index 000000000..09bb0e30d --- /dev/null +++ b/dbgpt/serve/flow/templates/zh/hybrid-knowldge-process-flow-template.json @@ -0,0 +1,1464 @@ +{ + "flow": { + "uid": "fa8f01ab-fc7b-4e35-9533-f8e4cf045cce", + "label": "Knowledge Process Workflow", + "name": "hybrid_knowledge_process_workflow_zh", + "flow_category": null, + "flow_data": { + "nodes": [ + { + "width": 320, + "height": 275, + "id": "operator_knowledge_process_join_operator___$$___rag___$$___v1_0", + "position": { + "x": 604.5, + "y": 77, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "Embedded & KG 结果聚合算子", + "custom_label": null, + "name": "knowledge_process_join_operator", + "description": "Join Branch the workflow based on the Knowledge Process Results.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_knowledge_process_join_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [], + "operator_type": "join", + "inputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Knowledge Graph Storage Results", + "custom_label": null, + "name": "input_value", + "description": "knowledge graph storage results.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + }, + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Knowledge Graph Storage Results", + "custom_label": null, + "name": "input_value", + "description": "knowledge graph storage results.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "Knowledge Process Results.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeProcessJoinOperator", + "type_cls": "dbgpt.rag.operators.process_branch.KnowledgeProcessJoinOperator" + }, + "position_absolute": { + "x": 604.5, + "y": 77, + "zoom": 0 + } + }, + { + "width": 320, + "height": 323, + "id": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "position": { + "x": 174.16583729394188, + "y": -131.63401513480102, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "向量抽取存储算子", + "custom_label": null, + "name": "vector_storage_operator", + "description": "Persist embeddings into vector storage.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "VectorStoreBase", + "type_cls": "dbgpt.storage.vector_store.base.VectorStoreBase", + "dynamic": false, + "dynamic_minimum": 0, + "label": "向量存储连接器", + "name": "vector_store", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "The vector store.", + "options": null, + "value": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "The text split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "已组装的块，已持久化到向量存储中。", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "VectorStorageOperator", + "type_cls": "dbgpt.rag.operators.vector_store.VectorStorageOperator" + }, + "position_absolute": { + "x": 174.16583729394188, + "y": -131.63401513480102, + "zoom": 0 + } + }, + { + "width": 320, + "height": 323, + "id": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "position": { + "x": 180.89103763027083, + "y": 341.37174185366564, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "知识图谱抽取算子", + "custom_label": null, + "name": "knowledge_graph_operator", + "description": "Extract Documents and persist into graph database.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "KnowledgeGraphBase", + "type_cls": "dbgpt.storage.knowledge_graph.base.KnowledgeGraphBase", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Knowledge Graph Connector", + "name": "graph_store", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "The knowledge graph.", + "options": null, + "value": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "The text split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "已组装的块，已持久化到向量存储中。", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeGraphOperator", + "type_cls": "dbgpt.rag.operators.knowledge_graph.KnowledgeGraphOperator" + }, + "position_absolute": { + "x": 180.89103763027083, + "y": 341.37174185366564, + "zoom": 0 + } + }, + { + "width": 320, + "height": 321, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "position": { + "x": -1056.545824254249, + "y": -163.01828337100258, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "文档Chunk切片算子", + "custom_label": null, + "name": "chunk_manager_operator", + "description": " Split Knowledge Documents into chunks.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "ChunkParameters", + "type_cls": "dbgpt.rag.chunk_manager.ChunkParameters", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Chunk Split Parameters", + "name": "chunk_parameters", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Chunk Split Parameters.", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "Knowledge", + "type_cls": "dbgpt.rag.knowledge.base.Knowledge", + "label": "知识", + "custom_label": null, + "name": "knowledge", + "description": "The knowledge to be loaded.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "The split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "ChunkManagerOperator", + "type_cls": "dbgpt.rag.operators.chunk_manager.ChunkManagerOperator" + }, + "position_absolute": { + "x": -1056.545824254249, + "y": -163.01828337100258, + "zoom": 0 + } + }, + { + "width": 320, + "height": 234, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "position": { + "x": -306.391788536534, + "y": 491.0961943850906, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "BuiltinKnowledgeGraph", + "type_cls": "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph", + "label": "Builtin Knowledge Graph", + "custom_label": null, + "name": "builtin_knowledge_graph", + "description": "Builtin Knowledge Graph.", + "category": "knowledge_graph", + "category_label": "Knowledge Graph", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "BuiltinKnowledgeGraphConfig", + "type_cls": "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Builtin Knowledge Graph Config.", + "name": "config", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Builtin Knowledge Graph Config.", + "options": null, + "value": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph", + "dbgpt.storage.knowledge_graph.base.KnowledgeGraphBase", + "dbgpt.rag.index.base.IndexStoreBase" + ] + }, + "position_absolute": { + "x": -306.391788536534, + "y": 491.0961943850906, + "zoom": 0 + } + }, + { + "width": 320, + "height": 645, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "position": { + "x": -813.7432345424928, + "y": 328.2957752754239, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "BuiltinKnowledgeGraphConfig", + "type_cls": "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig", + "label": "Builtin Graph Config", + "custom_label": null, + "name": "knowledge_graph_config", + "description": "knowledge graph config.", + "category": "knowledge_graph", + "category_label": "Knowledge Graph", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Graph Name", + "name": "name", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "dbgpt_collection", + "placeholder": null, + "description": "The name of Graph, if not set, will use the default name.", + "options": null, + "value": "dbgpt_collection_V1", + "alias": null, + "ui": null + }, + { + "type_name": "Embeddings", + "type_cls": "dbgpt.core.interface.embeddings.Embeddings", + "dynamic": false, + "dynamic_minimum": 0, + "label": "嵌入函数", + "name": "embedding_fn", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "向量存储的嵌入函数，如果未设置，将使用默认的嵌入函数。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "一次加载的最大块数", + "name": "max_chunks_once_load", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 10, + "placeholder": null, + "description": "一次加载的最大块数。如果您的文档很大，可以将此值设置为较大的数字，以加快加载过程。默认值为 10。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "最大线程数", + "name": "max_threads", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 1, + "placeholder": null, + "description": "要使用的最大线程数。默认值为 1。如果您将此值设置为大于 1，请确保您的向量存储是线程安全的。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Knowledge Graph Type", + "name": "graph_store_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "graph store type.", + "options": null, + "value": "TuGraph", + "alias": null, + "ui": null + }, + { + "type_name": "LLMClient", + "type_cls": "dbgpt.core.interface.llm.LLMClient", + "dynamic": false, + "dynamic_minimum": 0, + "label": "LLM Client", + "name": "llm_client", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "llm client for extract graph triplets.", + "options": null, + "value": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "LLM Model Name", + "name": "model_name", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "llm model name.", + "options": null, + "value": "zhipu_proxyllm", + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig", + "dbgpt.storage.knowledge_graph.base.KnowledgeGraphConfig", + "dbgpt.rag.index.base.IndexStoreConfig", + "pydantic.main.BaseModel" + ] + }, + "position_absolute": { + "x": -813.7432345424928, + "y": 328.2957752754239, + "zoom": 0 + } + }, + { + "width": 320, + "height": 234, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "position": { + "x": -251.7635173402225, + "y": -367.01602690631285, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "ChromaStore", + "type_cls": "dbgpt.storage.vector_store.chroma_store.ChromaStore", + "label": "Chroma Vector Store", + "custom_label": null, + "name": "chroma_vector_store", + "description": "Chroma 向量存储。", + "category": "vector_store", + "category_label": "Vector Store", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "ChromaVectorConfig", + "type_cls": "dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Chroma Config", + "name": "vector_store_config", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "the chroma config of vector store.", + "options": null, + "value": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.vector_store.chroma_store.ChromaStore", + "dbgpt.storage.vector_store.base.VectorStoreBase", + "dbgpt.rag.index.base.IndexStoreBase" + ] + }, + "position_absolute": { + "x": -251.7635173402225, + "y": -367.01602690631285, + "zoom": 0 + } + }, + { + "width": 320, + "height": 674, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "position": { + "x": -684.4180723107158, + "y": -934.1745886033843, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "ChromaVectorConfig", + "type_cls": "dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig", + "label": "Chroma Config", + "custom_label": null, + "name": "chroma_vector_config", + "description": "Chroma vector store config.", + "category": "vector_store", + "category_label": "Vector Store", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "集合名称", + "name": "name", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "dbgpt_collection", + "placeholder": null, + "description": "向量存储的名称，如果未设置，将使用默认名称。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "用户", + "name": "user", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "向量存储的用户，如果未设置，将使用默认用户。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "密码", + "name": "password", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "向量存储的密码，如果未设置，将使用默认密码。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "Embeddings", + "type_cls": "dbgpt.core.interface.embeddings.Embeddings", + "dynamic": false, + "dynamic_minimum": 0, + "label": "嵌入函数", + "name": "embedding_fn", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "向量存储的嵌入函数，如果未设置，将使用默认的嵌入函数。", + "options": null, + "value": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "一次加载的最大块数", + "name": "max_chunks_once_load", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 10, + "placeholder": null, + "description": "一次加载的最大块数。如果您的文档很大，可以将此值设置为较大的数字，以加快加载过程。默认值为 10。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "最大线程数", + "name": "max_threads", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 1, + "placeholder": null, + "description": "要使用的最大线程数。默认值为 1。如果您将此值设置为大于 1，请确保您的向量存储是线程安全的。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Persist Path", + "name": "persist_path", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "向量存储的持久化路径。", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig", + "dbgpt.storage.vector_store.base.VectorStoreConfig", + "dbgpt.rag.index.base.IndexStoreConfig", + "pydantic.main.BaseModel" + ] + }, + "position_absolute": { + "x": -684.4180723107158, + "y": -934.1745886033843, + "zoom": 0 + } + }, + { + "width": 320, + "height": 431, + "id": "operator_knowledge_operator___$$___rag___$$___v1_0", + "position": { + "x": -1558.679801266548, + "y": -149.61064934406164, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "文档知识加载算子", + "custom_label": null, + "name": "knowledge_operator", + "description": "知识算子，可以从数据源创建知识。", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": "https://github.com/openai/openai-python", + "id": "operator_knowledge_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "默认数据源", + "name": "datasource", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "默认数据源。", + "options": null, + "value": "../../docs/docs/awel/awel.md", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "知识类型", + "name": "knowledge_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "DOCUMENT", + "placeholder": null, + "description": "知识类型。", + "options": [ + { + "label": "DOCUMENT", + "name": "DOCUMENT", + "value": "DOCUMENT", + "children": null + }, + { + "label": "URL", + "name": "URL", + "value": "URL", + "children": null + }, + { + "label": "TEXT", + "name": "TEXT", + "value": "TEXT", + "children": null + } + ], + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "dict", + "type_cls": "builtins.dict", + "label": "知识数据源", + "custom_label": null, + "name": "knowledge datasource", + "description": "知识数据源，可以是文档、网址或文本。", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "Knowledge", + "type_cls": "dbgpt.rag.knowledge.base.Knowledge", + "label": "知识", + "custom_label": null, + "name": "Knowledge", + "description": "知识对象。", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeOperator", + "type_cls": "dbgpt.rag.operators.knowledge.KnowledgeOperator" + }, + "position_absolute": { + "x": -1558.679801266548, + "y": -149.61064934406164, + "zoom": 0 + } + }, + { + "width": 320, + "height": 602, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "position": { + "x": -2015.3280350941911, + "y": -603.9181210010445, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "字典 HTTP 触发器", + "custom_label": null, + "name": "dict_http_trigger", + "description": "通过 HTTP 请求触发您的工作流，并将请求主体解析为字典", + "category": "trigger", + "category_label": "Trigger", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "API 端点", + "name": "endpoint", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "API 端点", + "options": null, + "value": "/rag/knowledge/hybrid/process", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP 方法", + "name": "methods", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "POST", + "placeholder": null, + "description": "API 端点的方法", + "options": [ + { + "label": "HTTP PUT 方法", + "name": "http_put", + "value": "PUT", + "children": null + }, + { + "label": "HTTP POST 方法", + "name": "http_post", + "value": "POST", + "children": null + } + ], + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "bool", + "type_cls": "builtins.bool", + "dynamic": false, + "dynamic_minimum": 0, + "label": "流式响应", + "name": "streaming_response", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": false, + "placeholder": null, + "description": "响应是否为流式传输", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "BaseHttpBody", + "type_cls": "dbgpt.core.awel.trigger.http_trigger.BaseHttpBody", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP 响应主体", + "name": "http_response_body", + "is_list": false, + "category": "resource", + "resource_type": "class", + "optional": true, + "default": null, + "placeholder": null, + "description": "API 端点的响应主体", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "响应媒体类型", + "name": "response_media_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "响应的媒体类型", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP 状态码", + "name": "status_code", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 200, + "placeholder": null, + "description": "HTTP 状态码", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "input", + "inputs": [], + "outputs": [ + { + "type_name": "dict", + "type_cls": "builtins.dict", + "label": "请求体", + "custom_label": null, + "name": "request_body", + "description": "API 端点的请求主体", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "version": "v1", + "type_name": "DictHttpTrigger", + "type_cls": "dbgpt.core.awel.trigger.http_trigger.DictHttpTrigger" + }, + "position_absolute": { + "x": -2015.3280350941911, + "y": -603.9181210010445, + "zoom": 0 + } + }, + { + "width": 320, + "height": 148, + "id": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "position": { + "x": -1112.4932879687394, + "y": -753.8648984316667, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "DefaultEmbeddings", + "type_cls": "dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings", + "label": "默认Embedding转换", + "custom_label": null, + "name": "default_embeddings", + "description": "默认嵌入式（使用当前系统的默认嵌入式模型）", + "category": "embeddings", + "category_label": "Embeddings", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings", + "dbgpt.core.interface.embeddings.Embeddings" + ] + }, + "position_absolute": { + "x": -1112.4932879687394, + "y": -753.8648984316667, + "zoom": 0 + } + }, + { + "width": 320, + "height": 272, + "id": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "position": { + "x": -1350.535131696419, + "y": 435.2340305150391, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "DefaultLLMClient", + "type_cls": "dbgpt.model.cluster.client.DefaultLLMClient", + "label": "默认 LLM 客户端", + "custom_label": null, + "name": "default_llm_client", + "description": "默认 LLM 客户端（连接到您的 DB-GPT 模型服务）", + "category": "llm_client", + "category_label": "LLM Client", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "bool", + "type_cls": "builtins.bool", + "dynamic": false, + "dynamic_minimum": 0, + "label": "自动转换消息", + "name": "auto_convert_message", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": true, + "placeholder": null, + "description": "是否将 LLM 不支持的消息自动转换为兼容格式", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.model.cluster.client.DefaultLLMClient", + "dbgpt.core.interface.llm.LLMClient" + ] + }, + "position_absolute": { + "x": -1350.535131696419, + "y": 435.2340305150391, + "zoom": 0 + } + }, + { + "width": 320, + "height": 295, + "id": "operator_knowledge_process_operator___$$___rag___$$___v1_0", + "position": { + "x": -614.4846931519926, + "y": -92.5163519851338, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "知识加工处理分支算子", + "custom_label": null, + "name": "knowledge_process_operator", + "description": "Branch the workflow based on the stream flag of the request.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_knowledge_process_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [], + "operator_type": "branch", + "inputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "Document Chunks", + "custom_label": null, + "name": "input_value", + "description": "The input value of the operator.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "Chunks for Vector Storage Connector.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + }, + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "Chunks for Knowledge Graph Connector.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + }, + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "Chunks for Full Text Connector.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeProcessBranchOperator", + "type_cls": "dbgpt.rag.operators.process_branch.KnowledgeProcessBranchOperator" + }, + "position_absolute": { + "x": -614.4846931519926, + "y": -92.5163519851338, + "zoom": 0 + } + } + ], + "edges": [ + { + "source": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_process_join_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_vector_storage_operator___$$___rag___$$___v1_0|operator_knowledge_process_join_operator___$$___rag___$$___v1_0", + "source_handle": "operator_vector_storage_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_process_join_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_process_join_operator___$$___rag___$$___v1_0", + "target_order": 1, + "id": "operator_knowledge_graph_operator___$$___rag___$$___v1_0|operator_knowledge_process_join_operator___$$___rag___$$___v1_0", + "source_handle": "operator_knowledge_graph_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_process_join_operator___$$___rag___$$___v1_0|inputs|1", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "source_order": 0, + "target": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0|operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "source_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0|outputs|0", + "target_handle": "operator_knowledge_graph_operator___$$___rag___$$___v1_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "source_order": 0, + "target": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "target_order": 0, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0|resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "source_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0|outputs|0", + "target_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "source_order": 0, + "target": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0|operator_vector_storage_operator___$$___rag___$$___v1_0", + "source_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0|outputs|0", + "target_handle": "operator_vector_storage_operator___$$___rag___$$___v1_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "source_order": 0, + "target": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "target_order": 0, + "id": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0|resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0", + "source_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0|outputs|0", + "target_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaStore_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "operator_knowledge_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_knowledge_operator___$$___rag___$$___v1_0|operator_chunk_manager_operator___$$___rag___$$___v1_0", + "source_handle": "operator_knowledge_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_chunk_manager_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0|operator_knowledge_operator___$$___rag___$$___v1_0", + "source_handle": "operator_dict_http_trigger___$$___trigger___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0", + "source_order": 0, + "target": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "target_order": 3, + "id": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0|resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0", + "source_handle": "resource_dbgpt.rag.embedding.embedding_factory.DefaultEmbeddings_0|outputs|0", + "target_handle": "resource_dbgpt.storage.vector_store.chroma_store.ChromaVectorConfig_0|parameters|3", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "source_order": 0, + "target": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "target_order": 5, + "id": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0|resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "source_handle": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0|outputs|0", + "target_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0|parameters|5", + "type": "buttonedge" + }, + { + "source": "operator_knowledge_process_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_vector_storage_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_knowledge_process_operator___$$___rag___$$___v1_0|operator_vector_storage_operator___$$___rag___$$___v1_0", + "source_handle": "operator_knowledge_process_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_vector_storage_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "operator_knowledge_process_operator___$$___rag___$$___v1_0", + "source_order": 1, + "target": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_knowledge_process_operator___$$___rag___$$___v1_0|operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "source_handle": "operator_knowledge_process_operator___$$___rag___$$___v1_0|outputs|1", + "target_handle": "operator_knowledge_graph_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_process_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0|operator_knowledge_process_operator___$$___rag___$$___v1_0", + "source_handle": "operator_chunk_manager_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_process_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + } + ], + "viewport": { + "x": 862.7562630728534, + "y": 416.4952851718194, + "zoom": 0.3836649890553727 + } + }, + "description": "结合Embedding和知识图谱抽取的混合知识加工工作流", + "state": "running", + "error_message": "", + "source": "DBGPT-WEB", + "source_url": null, + "version": "0.1.1", + "define_type": "json", + "editable": true, + "user_name": null, + "sys_code": null, + "dag_id": "flow_dag_knowledge_factory_workflow_fa8f01ab-fc7b-4e35-9533-f8e4cf045cce", + "gmt_created": "2024-12-14 15:57:44", + "gmt_modified": "2024-12-14 15:57:44", + "metadata": { + "sse_output": false, + "streaming_output": false, + "tags": {}, + "triggers": [ + { + "trigger_type": "http", + "path": "/api/v1/awel/trigger/rag/knowledge/process", + "methods": [ + "POST" + ], + "trigger_mode": "command" + } + ] + }, + "variables": null, + "authors": null +} +} \ No newline at end of file diff --git a/dbgpt/serve/flow/templates/zh/kg-knowledge-process-flow-template.json b/dbgpt/serve/flow/templates/zh/kg-knowledge-process-flow-template.json new file mode 100644 index 000000000..8f8f4a67d --- /dev/null +++ b/dbgpt/serve/flow/templates/zh/kg-knowledge-process-flow-template.json @@ -0,0 +1,872 @@ +{ + "flow": { + "uid": "feb180ad-0f02-42c9-983d-70eb5ae9b346", + "label": "知识图谱加工工作流", + "name": "knowledge_graph_process_workflow", + "flow_category": null, + "flow_data": { + "nodes": [ + { + "width": 320, + "height": 323, + "id": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "position": { + "x": 6.722768991652174, + "y": -225.32501282124363, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "知识图谱抽取算子", + "custom_label": null, + "name": "knowledge_graph_operator", + "description": "Extract Documents and persist into graph database.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "KnowledgeGraphBase", + "type_cls": "dbgpt.storage.knowledge_graph.base.KnowledgeGraphBase", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Knowledge Graph Connector", + "name": "graph_store", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "The knowledge graph.", + "options": null, + "value": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "The text split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "已组装的块，已持久化到向量存储中。", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeGraphOperator", + "type_cls": "dbgpt.rag.operators.knowledge_graph.KnowledgeGraphOperator" + }, + "position_absolute": { + "x": 6.722768991652174, + "y": -225.32501282124363, + "zoom": 0 + } + }, + { + "width": 320, + "height": 321, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "position": { + "x": -812.1903428806644, + "y": -415.17234393736123, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "文档Chunk切片算子", + "custom_label": null, + "name": "chunk_manager_operator", + "description": " Split Knowledge Documents into chunks.", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "ChunkParameters", + "type_cls": "dbgpt.rag.chunk_manager.ChunkParameters", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Chunk Split Parameters", + "name": "chunk_parameters", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Chunk Split Parameters.", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "Knowledge", + "type_cls": "dbgpt.rag.knowledge.base.Knowledge", + "label": "知识", + "custom_label": null, + "name": "knowledge", + "description": "The knowledge to be loaded.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "List", + "type_cls": "typing.List", + "label": "块", + "custom_label": null, + "name": "chunks", + "description": "The split chunks by chunk manager.", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": true, + "mappers": null + } + ], + "version": "v1", + "type_name": "ChunkManagerOperator", + "type_cls": "dbgpt.rag.operators.chunk_manager.ChunkManagerOperator" + }, + "position_absolute": { + "x": -812.1903428806644, + "y": -415.17234393736123, + "zoom": 0 + } + }, + { + "width": 320, + "height": 234, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "position": { + "x": -446.7662140064656, + "y": 116.76439313193941, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "BuiltinKnowledgeGraph", + "type_cls": "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph", + "label": "Builtin Knowledge Graph", + "custom_label": null, + "name": "builtin_knowledge_graph", + "description": "Builtin Knowledge Graph.", + "category": "knowledge_graph", + "category_label": "Knowledge Graph", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "BuiltinKnowledgeGraphConfig", + "type_cls": "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Builtin Knowledge Graph Config.", + "name": "config", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "Builtin Knowledge Graph Config.", + "options": null, + "value": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph", + "dbgpt.storage.knowledge_graph.base.KnowledgeGraphBase", + "dbgpt.rag.index.base.IndexStoreBase" + ] + }, + "position_absolute": { + "x": -446.7662140064656, + "y": 116.76439313193941, + "zoom": 0 + } + }, + { + "width": 320, + "height": 645, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "position": { + "x": -915.1247640485547, + "y": 148.92845384162234, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "BuiltinKnowledgeGraphConfig", + "type_cls": "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig", + "label": "Builtin Graph Config", + "custom_label": null, + "name": "knowledge_graph_config", + "description": "knowledge graph config.", + "category": "knowledge_graph", + "category_label": "Knowledge Graph", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Graph Name", + "name": "name", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "dbgpt_collection", + "placeholder": null, + "description": "The name of Graph, if not set, will use the default name.", + "options": null, + "value": "dbgpt_collection_V1", + "alias": null, + "ui": null + }, + { + "type_name": "Embeddings", + "type_cls": "dbgpt.core.interface.embeddings.Embeddings", + "dynamic": false, + "dynamic_minimum": 0, + "label": "嵌入函数", + "name": "embedding_fn", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "向量存储的嵌入函数，如果未设置，将使用默认的嵌入函数。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "一次加载的最大块数", + "name": "max_chunks_once_load", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 10, + "placeholder": null, + "description": "一次加载的最大块数。如果您的文档很大，可以将此值设置为较大的数字，以加快加载过程。默认值为 10。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "最大线程数", + "name": "max_threads", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 1, + "placeholder": null, + "description": "要使用的最大线程数。默认值为 1。如果您将此值设置为大于 1，请确保您的向量存储是线程安全的。", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "Knowledge Graph Type", + "name": "graph_store_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "graph store type.", + "options": null, + "value": "TuGraph", + "alias": null, + "ui": null + }, + { + "type_name": "LLMClient", + "type_cls": "dbgpt.core.interface.llm.LLMClient", + "dynamic": false, + "dynamic_minimum": 0, + "label": "LLM Client", + "name": "llm_client", + "is_list": false, + "category": "resource", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "llm client for extract graph triplets.", + "options": null, + "value": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "LLM Model Name", + "name": "model_name", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "llm model name.", + "options": null, + "value": "zhipu_proxyllm", + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig", + "dbgpt.storage.knowledge_graph.base.KnowledgeGraphConfig", + "dbgpt.rag.index.base.IndexStoreConfig", + "pydantic.main.BaseModel" + ] + }, + "position_absolute": { + "x": -915.1247640485547, + "y": 148.92845384162234, + "zoom": 0 + } + }, + { + "width": 320, + "height": 431, + "id": "operator_knowledge_operator___$$___rag___$$___v1_0", + "position": { + "x": -1381.9120062303377, + "y": -370.57039313932444, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "文档知识加载算子", + "custom_label": null, + "name": "knowledge_operator", + "description": "知识算子，可以从数据源创建知识。", + "category": "rag", + "category_label": "RAG", + "flow_type": "operator", + "icon": null, + "documentation_url": "https://github.com/openai/openai-python", + "id": "operator_knowledge_operator___$$___rag___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "默认数据源", + "name": "datasource", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "默认数据源。", + "options": null, + "value": "../../docs/docs/awel/awel.md", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "知识类型", + "name": "knowledge_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "DOCUMENT", + "placeholder": null, + "description": "知识类型。", + "options": [ + { + "label": "DOCUMENT", + "name": "DOCUMENT", + "value": "DOCUMENT", + "children": null + }, + { + "label": "URL", + "name": "URL", + "value": "URL", + "children": null + }, + { + "label": "TEXT", + "name": "TEXT", + "value": "TEXT", + "children": null + } + ], + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "map", + "inputs": [ + { + "type_name": "dict", + "type_cls": "builtins.dict", + "label": "知识数据源", + "custom_label": null, + "name": "knowledge datasource", + "description": "知识数据源，可以是文档、网址或文本。", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "outputs": [ + { + "type_name": "Knowledge", + "type_cls": "dbgpt.rag.knowledge.base.Knowledge", + "label": "知识", + "custom_label": null, + "name": "Knowledge", + "description": "知识对象。", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "version": "v1", + "type_name": "KnowledgeOperator", + "type_cls": "dbgpt.rag.operators.knowledge.KnowledgeOperator" + }, + "position_absolute": { + "x": -1381.9120062303377, + "y": -370.57039313932444, + "zoom": 0 + } + }, + { + "width": 320, + "height": 602, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "position": { + "x": -2020.527087889374, + "y": -445.3470107479735, + "zoom": 0 + }, + "type": "customNode", + "data": { + "label": "字典 HTTP 触发器", + "custom_label": null, + "name": "dict_http_trigger", + "description": "通过 HTTP 请求触发您的工作流，并将请求主体解析为字典", + "category": "trigger", + "category_label": "Trigger", + "flow_type": "operator", + "icon": null, + "documentation_url": null, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "API 端点", + "name": "endpoint", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": false, + "default": null, + "placeholder": null, + "description": "API 端点", + "options": null, + "value": "/rag/knowledge/kg/process", + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP 方法", + "name": "methods", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": "POST", + "placeholder": null, + "description": "API 端点的方法", + "options": [ + { + "label": "HTTP PUT 方法", + "name": "http_put", + "value": "PUT", + "children": null + }, + { + "label": "HTTP POST 方法", + "name": "http_post", + "value": "POST", + "children": null + } + ], + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "bool", + "type_cls": "builtins.bool", + "dynamic": false, + "dynamic_minimum": 0, + "label": "流式响应", + "name": "streaming_response", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": false, + "placeholder": null, + "description": "响应是否为流式传输", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "BaseHttpBody", + "type_cls": "dbgpt.core.awel.trigger.http_trigger.BaseHttpBody", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP 响应主体", + "name": "http_response_body", + "is_list": false, + "category": "resource", + "resource_type": "class", + "optional": true, + "default": null, + "placeholder": null, + "description": "API 端点的响应主体", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "str", + "type_cls": "builtins.str", + "dynamic": false, + "dynamic_minimum": 0, + "label": "响应媒体类型", + "name": "response_media_type", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": null, + "placeholder": null, + "description": "响应的媒体类型", + "options": null, + "value": null, + "alias": null, + "ui": null + }, + { + "type_name": "int", + "type_cls": "builtins.int", + "dynamic": false, + "dynamic_minimum": 0, + "label": "HTTP 状态码", + "name": "status_code", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": 200, + "placeholder": null, + "description": "HTTP 状态码", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "operator_type": "input", + "inputs": [], + "outputs": [ + { + "type_name": "dict", + "type_cls": "builtins.dict", + "label": "请求体", + "custom_label": null, + "name": "request_body", + "description": "API 端点的请求主体", + "dynamic": false, + "dynamic_minimum": 0, + "is_list": false, + "mappers": null + } + ], + "version": "v1", + "type_name": "DictHttpTrigger", + "type_cls": "dbgpt.core.awel.trigger.http_trigger.DictHttpTrigger" + }, + "position_absolute": { + "x": -2020.527087889374, + "y": -445.3470107479735, + "zoom": 0 + } + }, + { + "width": 320, + "height": 272, + "id": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "position": { + "x": -1506.5067155518987, + "y": 313.0562898282468, + "zoom": 0 + }, + "type": "customNode", + "data": { + "type_name": "DefaultLLMClient", + "type_cls": "dbgpt.model.cluster.client.DefaultLLMClient", + "label": "默认 LLM 客户端", + "custom_label": null, + "name": "default_llm_client", + "description": "默认 LLM 客户端（连接到您的 DB-GPT 模型服务）", + "category": "llm_client", + "category_label": "LLM Client", + "flow_type": "resource", + "icon": null, + "documentation_url": null, + "id": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "tags": { + "ui_version": "flow2.0" + }, + "parameters": [ + { + "type_name": "bool", + "type_cls": "builtins.bool", + "dynamic": false, + "dynamic_minimum": 0, + "label": "自动转换消息", + "name": "auto_convert_message", + "is_list": false, + "category": "common", + "resource_type": "instance", + "optional": true, + "default": true, + "placeholder": null, + "description": "是否将 LLM 不支持的消息自动转换为兼容格式", + "options": null, + "value": null, + "alias": null, + "ui": null + } + ], + "resource_type": "instance", + "parent_cls": [ + "dbgpt.model.cluster.client.DefaultLLMClient", + "dbgpt.core.interface.llm.LLMClient" + ] + }, + "position_absolute": { + "x": -1506.5067155518987, + "y": 313.0562898282468, + "zoom": 0 + } + } + ], + "edges": [ + { + "source": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "source_order": 0, + "target": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0|operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "source_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0|outputs|0", + "target_handle": "operator_knowledge_graph_operator___$$___rag___$$___v1_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "source_order": 0, + "target": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "target_order": 0, + "id": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0|resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0", + "source_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0|outputs|0", + "target_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraph_0|parameters|0", + "type": "buttonedge" + }, + { + "source": "operator_knowledge_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_knowledge_operator___$$___rag___$$___v1_0|operator_chunk_manager_operator___$$___rag___$$___v1_0", + "source_handle": "operator_knowledge_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_chunk_manager_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "operator_dict_http_trigger___$$___trigger___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_dict_http_trigger___$$___trigger___$$___v1_0|operator_knowledge_operator___$$___rag___$$___v1_0", + "source_handle": "operator_dict_http_trigger___$$___trigger___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + }, + { + "source": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0", + "source_order": 0, + "target": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "target_order": 5, + "id": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0|resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0", + "source_handle": "resource_dbgpt.model.cluster.client.DefaultLLMClient_0|outputs|0", + "target_handle": "resource_dbgpt.storage.knowledge_graph.knowledge_graph.BuiltinKnowledgeGraphConfig_0|parameters|5", + "type": "buttonedge" + }, + { + "source": "operator_chunk_manager_operator___$$___rag___$$___v1_0", + "source_order": 0, + "target": "operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "target_order": 0, + "id": "operator_chunk_manager_operator___$$___rag___$$___v1_0|operator_knowledge_graph_operator___$$___rag___$$___v1_0", + "source_handle": "operator_chunk_manager_operator___$$___rag___$$___v1_0|outputs|0", + "target_handle": "operator_knowledge_graph_operator___$$___rag___$$___v1_0|inputs|0", + "type": "buttonedge" + } + ], + "viewport": { + "x": 831.8128405437491, + "y": 421.4753242151554, + "zoom": 0.3846854569072972 + } + }, + "description": "知识图谱知识加工工作流", + "state": "running", + "error_message": "", + "source": "DBGPT-WEB", + "source_url": null, + "version": "0.1.1", + "define_type": "json", + "editable": true, + "user_name": null, + "sys_code": null, + "dag_id": "flow_dag_knowledge_graph_process_workflow_feb180ad-0f02-42c9-983d-70eb5ae9b346", + "gmt_created": "2024-12-16 17:51:31", + "gmt_modified": "2024-12-16 17:51:31", + "metadata": { + "sse_output": false, + "streaming_output": false, + "tags": {}, + "triggers": [ + { + "trigger_type": "http", + "path": "/api/v1/awel/trigger/rag/graph/process", + "methods": [ + "POST" + ], + "trigger_mode": "command" + } + ] + }, + "variables": null, + "authors": null +} +} \ No newline at end of file diff --git a/dbgpt/serve/rag/api/schemas.py b/dbgpt/serve/rag/api/schemas.py index 6ae7b994b..5d410fddb 100644 --- a/dbgpt/serve/rag/api/schemas.py +++ b/dbgpt/serve/rag/api/schemas.py @@ -96,6 +96,8 @@ class ChunkServeResponse(BaseModel): content: Optional[str] = Field(None, description="chunk content") meta_info: Optional[str] = Field(None, description="chunk meta info") questions: Optional[str] = Field(None, description="chunk questions") + gmt_created: Optional[str] = Field(None, description="chunk create time") + gmt_modified: Optional[str] = Field(None, description="chunk modify time") class KnowledgeSyncRequest(BaseModel): diff --git a/dbgpt/serve/rag/service/service.py b/dbgpt/serve/rag/service/service.py index 2bb8e337d..1f3b3d7a6 100644 --- a/dbgpt/serve/rag/service/service.py +++ b/dbgpt/serve/rag/service/service.py @@ -323,7 +323,7 @@ def update_document(self, request: DocumentServeRequest): update_chunk = self._chunk_dao.get_one({"document_id": entity.id}) if update_chunk: update_chunk.doc_name = request.doc_name - self._chunk_dao.update_chunk(update_chunk) + self._chunk_dao.update({"id": update_chunk.id}, update_chunk) if len(request.questions) == 0: request.questions = "" questions = [ diff --git a/dbgpt/storage/knowledge_graph/community_summary.py b/dbgpt/storage/knowledge_graph/community_summary.py index 806f9df54..a818f16e5 100644 --- a/dbgpt/storage/knowledge_graph/community_summary.py +++ b/dbgpt/storage/knowledge_graph/community_summary.py @@ -6,22 +6,136 @@ from typing import List, Optional, Tuple from dbgpt._private.pydantic import ConfigDict, Field -from dbgpt.core import Chunk +from dbgpt.core import Chunk, LLMClient +from dbgpt.core.awel.flow import Parameter, ResourceCategory, register_resource from dbgpt.rag.transformer.community_summarizer import CommunitySummarizer from dbgpt.rag.transformer.graph_extractor import GraphExtractor from dbgpt.storage.knowledge_graph.base import ParagraphChunk from dbgpt.storage.knowledge_graph.community.community_store import CommunityStore from dbgpt.storage.knowledge_graph.knowledge_graph import ( + GRAPH_PARAMETERS, BuiltinKnowledgeGraph, BuiltinKnowledgeGraphConfig, ) from dbgpt.storage.vector_store.base import VectorStoreConfig from dbgpt.storage.vector_store.factory import VectorStoreFactory from dbgpt.storage.vector_store.filters import MetadataFilters +from dbgpt.util.i18n_utils import _ logger = logging.getLogger(__name__) +@register_resource( + _("Community Summary KG Config"), + "community_summary_kg_config", + category=ResourceCategory.KNOWLEDGE_GRAPH, + description=_("community Summary kg Config."), + parameters=[ + *GRAPH_PARAMETERS, + Parameter.build_from( + _("Knowledge Graph Type"), + "graph_store_type", + str, + description=_("graph store type."), + optional=True, + default="TuGraph", + ), + Parameter.build_from( + _("LLM Client"), + "llm_client", + LLMClient, + description=_("llm client for extract graph triplets."), + ), + Parameter.build_from( + _("LLM Model Name"), + "model_name", + str, + description=_("llm model name."), + optional=True, + default=None, + ), + Parameter.build_from( + _("Vector Store Type"), + "vector_store_type", + str, + description=_("vector store type."), + optional=True, + default="Chroma", + ), + Parameter.build_from( + _("Topk of Knowledge Graph Extract"), + "extract_topk", + int, + description=_("Topk of knowledge graph extract"), + optional=True, + default=5, + ), + Parameter.build_from( + _("Recall Score of Knowledge Graph Extract"), + "extract_score_threshold", + float, + description=_("Recall score of knowledge graph extract"), + optional=True, + default=0.3, + ), + Parameter.build_from( + _("Recall Score of Community Search in Knowledge Graph"), + "community_topk", + int, + description=_("Recall score of community search in knowledge graph"), + optional=True, + default=50, + ), + Parameter.build_from( + _("Recall Score of Community Search in Knowledge Graph"), + "community_score_threshold", + float, + description=_("Recall score of community search in knowledge graph"), + optional=True, + default=0.0, + ), + Parameter.build_from( + _("Enable the graph search for documents and chunks"), + "triplet_graph_enabled", + bool, + description=_("Enable the graph search for triplets"), + optional=True, + default=True, + ), + Parameter.build_from( + _("Enable the graph search for documents and chunks"), + "document_graph_enabled", + bool, + description=_("Enable the graph search for documents and chunks"), + optional=True, + default=True, + ), + Parameter.build_from( + _("Top size of knowledge graph chunk search"), + "knowledge_graph_chunk_search_top_size", + int, + description=_("Top size of knowledge graph chunk search"), + optional=True, + default=5, + ), + Parameter.build_from( + _("Batch size of triplets extraction from the text"), + "knowledge_graph_extraction_batch_size", + int, + description=_("Batch size of triplets extraction from the text"), + optional=True, + default=20, + ), + Parameter.build_from( + _("Batch size of parallel community building process"), + "community_summary_batch_size", + int, + description=_("TBatch size of parallel community building process"), + optional=True, + default=20, + ), + ], +) class CommunitySummaryKnowledgeGraphConfig(BuiltinKnowledgeGraphConfig): """Community summary knowledge graph config.""" @@ -80,6 +194,22 @@ class CommunitySummaryKnowledgeGraphConfig(BuiltinKnowledgeGraphConfig): ) +@register_resource( + _("Community Summary Knowledge Graph"), + "community_summary_knowledge_graph", + category=ResourceCategory.KNOWLEDGE_GRAPH, + description=_("Community Summary Knowledge Graph."), + parameters=[ + Parameter.build_from( + _("Community Summary Knowledge Graph Config."), + "config", + BuiltinKnowledgeGraphConfig, + description=_("Community Summary Knowledge Graph Config."), + optional=True, + default=None, + ), + ], +) class CommunitySummaryKnowledgeGraph(BuiltinKnowledgeGraph): """Community summary knowledge graph class.""" diff --git a/dbgpt/storage/knowledge_graph/knowledge_graph.py b/dbgpt/storage/knowledge_graph/knowledge_graph.py index ef2d15039..29d73c86d 100644 --- a/dbgpt/storage/knowledge_graph/knowledge_graph.py +++ b/dbgpt/storage/knowledge_graph/knowledge_graph.py @@ -6,7 +6,8 @@ from typing import List, Optional from dbgpt._private.pydantic import ConfigDict, Field -from dbgpt.core import Chunk, LLMClient +from dbgpt.core import Chunk, Embeddings, LLMClient +from dbgpt.core.awel.flow import Parameter, ResourceCategory, register_resource from dbgpt.rag.transformer.keyword_extractor import KeywordExtractor from dbgpt.rag.transformer.triplet_extractor import TripletExtractor from dbgpt.storage.graph_store.base import GraphStoreBase, GraphStoreConfig @@ -16,10 +17,87 @@ from dbgpt.storage.knowledge_graph.community.base import GraphStoreAdapter from dbgpt.storage.knowledge_graph.community.factory import GraphStoreAdapterFactory from dbgpt.storage.vector_store.filters import MetadataFilters +from dbgpt.util.i18n_utils import _ logger = logging.getLogger(__name__) - +GRAPH_PARAMETERS = [ + Parameter.build_from( + _("Graph Name"), + "name", + str, + description=_("The name of Graph, if not set, will use the default name."), + optional=True, + default="dbgpt_collection", + ), + Parameter.build_from( + _("Embedding Function"), + "embedding_fn", + Embeddings, + description=_( + "The embedding function of vector store, if not set, will use " + "the default embedding function." + ), + optional=True, + default=None, + ), + Parameter.build_from( + _("Max Chunks Once Load"), + "max_chunks_once_load", + int, + description=_( + "The max number of chunks to load at once. If your document is " + "large, you can set this value to a larger number to speed up the loading " + "process. Default is 10." + ), + optional=True, + default=10, + ), + Parameter.build_from( + _("Max Threads"), + "max_threads", + int, + description=_( + "The max number of threads to use. Default is 1. If you set " + "this bigger than 1, please make sure your vector store is thread-safe." + ), + optional=True, + default=1, + ), +] + + +@register_resource( + _("Builtin Graph Config"), + "knowledge_graph_config", + category=ResourceCategory.KNOWLEDGE_GRAPH, + description=_("knowledge graph config."), + parameters=[ + *GRAPH_PARAMETERS, + Parameter.build_from( + _("Knowledge Graph Type"), + "graph_store_type", + str, + description=_("graph store type."), + optional=True, + default="TuGraph", + ), + Parameter.build_from( + _("LLM Client"), + "llm_client", + LLMClient, + description=_("llm client for extract graph triplets."), + ), + Parameter.build_from( + _("LLM Model Name"), + "model_name", + str, + description=_("llm model name."), + optional=True, + default=None, + ), + ], +) class BuiltinKnowledgeGraphConfig(KnowledgeGraphConfig): """Builtin knowledge graph config.""" @@ -34,6 +112,22 @@ class BuiltinKnowledgeGraphConfig(KnowledgeGraphConfig): ) +@register_resource( + _("Builtin Knowledge Graph"), + "builtin_knowledge_graph", + category=ResourceCategory.KNOWLEDGE_GRAPH, + description=_("Builtin Knowledge Graph."), + parameters=[ + Parameter.build_from( + _("Builtin Knowledge Graph Config."), + "config", + BuiltinKnowledgeGraphConfig, + description=_("Builtin Knowledge Graph Config."), + optional=True, + default=None, + ), + ], +) class BuiltinKnowledgeGraph(KnowledgeGraphBase): """Builtin knowledge graph class.""" diff --git a/docs/docs/awel/awel_tutorial/templates/Embedding.md b/docs/docs/awel/awel_tutorial/templates/Embedding.md new file mode 100644 index 000000000..ee793956b --- /dev/null +++ b/docs/docs/awel/awel_tutorial/templates/Embedding.md @@ -0,0 +1,55 @@ +# Embedding Process Workflow +# Introduction +the traditional knowledge extraction preparation process of Native RAG aims at the process of turning documents into databases, including reading unstructured documents-> knowledge slices-> document slices turning-> import vector databases. + +# Applicable Scenarios ++ supports simple intelligent question and answer scenarios and recalls context information through semantic similarity. ++ Users can cut and add existing embedded processing processes according to their own business scenarios. + +# How to use ++ enter the AWEL interface and add a workflow + +![](https://intranetproxy.alipay.com/skylark/lark/0/2024/png/26456775/1734354927468-feed0ac7-e0fe-45e8-b85c-aba170084f82.png) + ++ import Knowledge Processing Template + +![](https://intranetproxy.alipay.com/skylark/lark/0/2024/png/26456775/1734358060884-672d3157-a2ee-498b-887e-ea51f1caddae.png) + ++ adjust parameters and save + +![](https://intranetproxy.alipay.com/skylark/lark/0/2024/png/26456775/1734358170081-32d38282-7765-4bbf-9bf7-c068550907d1.png) + + - `document knowledge loader operator `: Knowledge loading factory, by loading the specified document type, find the corresponding document processor for document content parsing. + - `Document Chunk Manager operator `: Slice the loaded document content according to the specified slicing parameters. + - `Vector storage machining operator `: You can connect different vector databases for vector storage, and you can also connect different Embedding models and services for vector extraction. + + + ++ Register Post as http request + +```bash +curl --location --request POST 'http://localhost:5670/api/v1/awel/trigger/rag/knowledge/embedding/process' \ +--header 'Content-Type: application/json' \ +--data-raw '{}' +``` + +```bash +[ + { + "content": "\"What is AWEL?\": Agentic Workflow Expression Language(AWEL) is a set of intelligent agent workflow expression language specially designed for large model application\ndevelopment. It provides great functionality and flexibility. Through the AWEL API, you can focus on the development of business logic for LLMs applications\nwithout paying attention to cumbersome model and environment details. \nAWEL adopts a layered API design. AWEL's layered API design architecture is shown in the figure below. \n

\n\n

", + "metadata": { + "Header1": "What is AWEL?", + "source": "../../docs/docs/awel/awel.md" + }, + "chunk_id": "c1ffa671-76d0-4c7a-b2dd-0b08dfd37712", + "chunk_name": "", + "score": 0.0, + "summary": "", + "separator": "\n", + "retriever": null + },... + ] +``` + + + diff --git a/docs/docs/awel/awel_tutorial/templates/Hybrid_Workflow.md b/docs/docs/awel/awel_tutorial/templates/Hybrid_Workflow.md new file mode 100644 index 000000000..c8a523a22 --- /dev/null +++ b/docs/docs/awel/awel_tutorial/templates/Hybrid_Workflow.md @@ -0,0 +1,46 @@ +# Hybrid Knowledge Process Workflow +# Introduction +At present, the DB-GPT knowledge base provides knowledge processing capabilities such as `document uploading` ->` parsing` ->` chunking` ->` Embedding` -> `Knowledge Graph triple extraction `-> `vector database storage` -> `graph database storage`, but it does not have the ability to extract complex information from documents, including vector extraction and Knowledge Graph extraction from document blocks at the same time. The hybrid knowledge processing template defines complex knowledge processing workflow, it also supports document vector extraction, Keyword extraction and Knowledge Graph extraction. + +# Applicable Scenarios ++ It is not limited to the traditional, single knowledge processing process (only Embedding processing or knowledge graph extraction processing), knowledge processing workflow implements Embedding and Knowledge Graph extraction at the same time, as a mixed knowledge recall retrieval data storage. ++ Users can tailor and add existing knowledge processing processes based on their own business scenarios. + +# How to use ++ Enter the AWEL interface and add a workflow + +![](https://intranetproxy.alipay.com/skylark/lark/0/2024/png/26456775/1734354927468-feed0ac7-e0fe-45e8-b85c-aba170084f82.png) + ++ Import Knowledge Processing Template + +![](https://intranetproxy.alipay.com/skylark/lark/0/2024/png/26456775/1734357236704-5a15be65-3d11-4406-98d7-efb82e5142dc.png) + ++ Adjust parameters and save + +![](https://intranetproxy.alipay.com/skylark/lark/0/2024/png/26456775/1734355123947-3e252e59-2b2a-4bca-adef-13a93ee6cdf3.png) + + - `Document knowledge loading operator `: Knowledge loading factory, by loading the specified document type, find the corresponding document processor for document content parsing. + - `Document Chunk slicing operator `: Slice the loaded document content according to the specified slicing parameters. + - `Knowledge Processing branch operator `: You can connect different knowledge processing processes, including knowledge map processing processes, vector processing processes, and keyword processing processes. + - `Vector storage machining operator `: You can connect different vector databases for vector storage, and you can also connect different Embedding models and services for vector extraction. + - `Knowledge Graph processing operator `: You can connect different knowledge graph processing operators, including native knowledge graph processing operators and community summary Knowledge Graph processing operators. You can also specify different graph databases for storage. Currently, TuGraph databases are supported. + - `Result aggregation operator `: Summarize the results of vector extraction and Knowledge Graph extraction. ++ Register Post as http request + +```bash +curl --location --request POST 'http://localhost:5670/api/v1/awel/trigger/rag/knowledge/hybrid/process' \ +--header 'Content-Type: application/json' \ +--data-raw '{}' +``` + +```bash +[ + "async persist vector store success 1 chunks.", + "async persist graph store success 1 chunks." +] +``` + + + + + diff --git a/docs/docs/awel/awel_tutorial/templates/Knowledge_Graph.md b/docs/docs/awel/awel_tutorial/templates/Knowledge_Graph.md new file mode 100644 index 000000000..1ee780c26 --- /dev/null +++ b/docs/docs/awel/awel_tutorial/templates/Knowledge_Graph.md @@ -0,0 +1,58 @@ +# Knowledge Graph Process Workflow + +# Introduction +Unlike traditional Native RAG, which requires vectors as data carriers, GraphRAG requires triple extraction (entity -> relationship -> entity) to build a knowledge graph, so the entire knowledge processing can also be regarded as the process of building a knowledge graph. + +![](https://intranetproxy.alipay.com/skylark/lark/0/2024/png/26456775/1734357331126-a3a96fd7-c8fb-4208-8e3b-be798d1b73b4.png) + +# Applicable Scenarios ++ It is necessary to use GraphRAG ability to mine the relationship between knowledge for multi-step reasoning. ++ Make up for the lack of integrity of Naive RAG in the recall context. + +# How to use ++ Enter the AWEL interface and add a workflow + +![](https://intranetproxy.alipay.com/skylark/lark/0/2024/png/26456775/1734354927468-feed0ac7-e0fe-45e8-b85c-aba170084f82.png) + ++ Import Knowledge Processing Template + +![](https://intranetproxy.alipay.com/skylark/lark/0/2024/png/26456775/1734356276305-a6e03aff-ba89-40c4-be2d-f88dff29d0f5.png) + ++ Adjust parameters and save + +![](https://intranetproxy.alipay.com/skylark/lark/0/2024/png/26456775/1734356745373-4e449611-d0bc-4735-b142-0aebafaa34d6.png) + + - `document knowledge loading operator `: Knowledge loading factory, by loading the specified document type, find the corresponding document processor for document content parsing. + - `Document Chunk slicing operator `: Slice the loaded document content according to the specified slicing parameters. + - `Knowledge Graph processing operator `: You can connect different knowledge graph processing operators, including native knowledge graph processing operators and community summary Knowledge Graph processing operators. You can also specify different graph databases for storage. Currently, TuGraph databases are supported. + + + ++ Register Post as http request + +```bash +curl --location --request POST 'http://localhost:5670/api/v1/awel/trigger/rag/knowledge/kg/process' \ +--header 'Content-Type: application/json' \ +--data-raw '{}' +``` + +```bash +[ + { + "content": "\"What is AWEL?\": Agentic Workflow Expression Language(AWEL) is a set of intelligent agent workflow expression language specially designed for large model application\ndevelopment. It provides great functionality and flexibility. Through the AWEL API, you can focus on the development of business logic for LLMs applications\nwithout paying attention to cumbersome model and environment details. \nAWEL adopts a layered API design. AWEL's layered API design architecture is shown in the figure below. \n

\n\n

", + "metadata": { + "Header1": "What is AWEL?", + "source": "../../docs/docs/awel/awel.md" + }, + "chunk_id": "c1ffa671-76d0-4c7a-b2dd-0b08dfd37712", + "chunk_name": "", + "score": 0.0, + "summary": "", + "separator": "\n", + "retriever": null + },... + ] +``` + + + diff --git a/docs/sidebars.js b/docs/sidebars.js index b2cbe71d7..808f740f2 100755 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -186,6 +186,24 @@ const sidebars = { id: "awel/awel_tutorial/advanced_guide/4.1_lifecycle" } ] + },, + { + type: "category", + label: "5. AWEL Template", + collapsed: false, + collapsible: false, + items: [ + { + type: "doc", + id: "awel/awel_tutorial/templates/Embedding" + },{ + type: "doc", + id: "awel/awel_tutorial/templates/Knowledge_Graph" + },{ + type: "doc", + id: "awel/awel_tutorial/templates/Hybrid_Workflow" + } + ] }, ], link: {