Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ChatKnowledge):Add knowledge Reference && Deploy Document Refactor #773

Merged
merged 38 commits into from
Nov 4, 2023
Merged
Changes from 1 commit
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
fc656e1
feat:knowledge rag graph
Aries-ckt Oct 10, 2023
e2a1990
Merge remote-tracking branch 'origin/main' into feat_rag_graph
Aries-ckt Oct 10, 2023
eb2c220
feat:rag graph conponent
Aries-ckt Oct 12, 2023
8a5e35c
feat:knowledge rag graph
Aries-ckt Oct 12, 2023
fa6a904
feat:knowledge rag graph
Aries-ckt Oct 13, 2023
2f82f98
feat:knowledge rag graph
Aries-ckt Oct 13, 2023
37bcbff
Merge remote-tracking branch 'origin/main' into feat_rag_graph
Aries-ckt Oct 15, 2023
71c31c3
Merge remote-tracking branch 'origin/main' into feat_rag_graph
Aries-ckt Oct 16, 2023
b63fa2d
feat:rag graph
Aries-ckt Oct 16, 2023
68c9010
feat:rag graph
Aries-ckt Oct 16, 2023
f93af98
feat:rag graph
Aries-ckt Oct 17, 2023
f6694d9
Merge remote-tracking branch 'origin/main' into feat_rag_graph
Aries-ckt Oct 18, 2023
aff0553
style:fmt
Aries-ckt Oct 19, 2023
39219a4
feat:rag_graph
Aries-ckt Oct 19, 2023
93de9c7
Merge remote-tracking branch 'origin/main' into feat_rag_graph
Aries-ckt Oct 19, 2023
318979a
Merge remote-tracking branch 'origin/main' into feat_rag_graph
Aries-ckt Oct 22, 2023
724456d
feat:extract summary
Aries-ckt Oct 25, 2023
e4c96e3
Merge remote-tracking branch 'origin/main' into feat_rag_graph
Aries-ckt Oct 28, 2023
53b1fc4
feat:document summary
Aries-ckt Oct 30, 2023
07ad8fa
Merge remote-tracking branch 'origin/main' into feat_rag_graph
Aries-ckt Oct 31, 2023
dca3ddb
feat:add summary
Aries-ckt Oct 31, 2023
16dd8e3
feat:document summary
Aries-ckt Oct 31, 2023
523838f
feat:document summary
Aries-ckt Oct 31, 2023
7dcfa19
feat:document summary
Aries-ckt Oct 31, 2023
b3dbf31
feat:document summary
Aries-ckt Oct 31, 2023
40eed54
chore:discord expire
Aries-ckt Oct 31, 2023
04dcd90
feat:document summary
Aries-ckt Oct 31, 2023
de90244
feat:document summary
Aries-ckt Oct 31, 2023
67f4155
feat:mapreduce summary
Aries-ckt Oct 31, 2023
be1e1cb
feat:document summary set max iteration
Aries-ckt Oct 31, 2023
606d384
feat:add knowledge reference
Aries-ckt Nov 1, 2023
9cc6386
doc:update deploy doc
Aries-ckt Nov 3, 2023
6fe7bfd
feat:merge main branch
Aries-ckt Nov 3, 2023
0874fa2
doc:update deploy doc
Aries-ckt Nov 3, 2023
3111760
doc:update document
Aries-ckt Nov 3, 2023
3ea85a6
feat:web add reference
Aries-ckt Nov 3, 2023
463b97f
doc:css
Aries-ckt Nov 3, 2023
76975d9
feat:knowledge refernce
Aries-ckt Nov 3, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat:knowledge rag graph
Aries-ckt committed Oct 13, 2023
commit fa6a9040d5e0670ba491b5574101e5de3e8f046a
25 changes: 25 additions & 0 deletions pilot/scene/chat_knowledge/v1/chat.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import os
from typing import Dict

from pilot.component import ComponentType
from pilot.scene.base_chat import BaseChat
from pilot.scene.base import ChatScene
from pilot.configs.config import Config
@@ -47,6 +48,11 @@ def __init__(self, chat_param: Dict):
"vector_store_name": self.knowledge_space,
"vector_store_type": CFG.VECTOR_STORE_TYPE,
}
from pilot.graph_engine.graph_factory import RAGGraphFactory

self.rag_engine = CFG.SYSTEM_APP.get_component(
ComponentType.RAG_GRAPH_DEFAULT.value, RAGGraphFactory
).create()
embedding_factory = CFG.SYSTEM_APP.get_component(
"embedding_factory", EmbeddingFactory
)
@@ -82,6 +88,25 @@ def generate_input_values(self):
if self.space_context:
self.prompt_template.template_define = self.space_context["prompt"]["scene"]
self.prompt_template.template = self.space_context["prompt"]["template"]
# docs = self.rag_engine.search(query=self.current_user_input)
# import httpx
# with httpx.Client() as client:
# request = client.build_request(
# "post",
# "http://127.0.0.1/api/knowledge/entities/extract",
# json="application/json", # using json for data to ensure it sends as application/json
# params={"text": self.current_user_input},
# headers={},
# )
#
# response = client.send(request)
# if response.status_code != 200:
# error_msg = f"request /api/knowledge/entities/extract failed, error: {response.text}"
# raise Exception(error_msg)
# docs = response.json()
# import requests
# docs = requests.post("http://127.0.0.1:5000/api/knowledge/entities/extract", headers={}, json={"text": self.current_user_input})

docs = self.knowledge_embedding_client.similar_search(
self.current_user_input, self.top_k
)
1 change: 1 addition & 0 deletions pilot/server/component_configs.py
Original file line number Diff line number Diff line change
@@ -31,6 +31,7 @@ def initialize_components(

# Register global default RAGGraphFactory
from pilot.graph_engine.graph_factory import DefaultRAGGraphFactory

system_app.register(DefaultRAGGraphFactory)

_initialize_embedding_model(
35 changes: 35 additions & 0 deletions pilot/server/knowledge/api.py
Original file line number Diff line number Diff line change
@@ -24,6 +24,7 @@
ChunkQueryRequest,
DocumentQueryRequest,
SpaceArgumentRequest,
EntityExtractRequest,
)

from pilot.server.knowledge.request.request import KnowledgeSpaceRequest
@@ -198,3 +199,37 @@ def similar_query(space_name: str, query_request: KnowledgeQueryRequest):
for d in docs
]
return {"response": res}


@router.post("/knowledge/entity/extract")
async def entity_extract(request: EntityExtractRequest):
logger.info(f"Received params: {request}")
try:
# from pilot.graph_engine.graph_factory import RAGGraphFactory
# from pilot.component import ComponentType
# rag_engine = CFG.SYSTEM_APP.get_component(
# ComponentType.RAG_GRAPH_DEFAULT.value, RAGGraphFactory
# ).create()
# return Result.succ(await rag_engine.search(request.text))
from pilot.scene.base import ChatScene
from pilot.common.chat_util import llm_chat_response_nostream
import uuid

chat_param = {
"chat_session_id": uuid.uuid1(),
"current_user_input": request.text,
"select_param": "entity",
"model_name": request.model_name,
}

# import nest_asyncio
# nest_asyncio.apply()
# loop = asyncio.get_event_loop()
# loop.stop()
# loop = utils.get_or_create_event_loop()
res = await llm_chat_response_nostream(
ChatScene.ExtractEntity.value(), **{"chat_param": chat_param}
)
return Result.succ(res)
except Exception as e:
return Result.faild(code="E000X", msg=f"entity extract error {e}")
7 changes: 7 additions & 0 deletions pilot/server/knowledge/request/request.py
Original file line number Diff line number Diff line change
@@ -104,3 +104,10 @@ class SpaceArgumentRequest(BaseModel):
"""argument: argument"""

argument: str


class EntityExtractRequest(BaseModel):
"""argument: argument"""

text: str
model_name: str
22 changes: 21 additions & 1 deletion pilot/server/knowledge/service.py
Original file line number Diff line number Diff line change
@@ -58,7 +58,11 @@ class SyncStatus(Enum):
# @singleton
class KnowledgeService:
def __init__(self):
pass
from pilot.graph_engine.graph_engine import RAGGraphEngine

# source = "/Users/chenketing/Desktop/project/llama_index/examples/paul_graham_essay/data/test/test_kg_text.txt"

# pass

"""create knowledge space"""

@@ -229,6 +233,10 @@ def sync_knowledge_document(self, space_name, sync_request: DocumentSyncRequest)
pre_separator=sync_request.pre_separator,
text_splitter_impl=text_splitter,
)
from pilot.graph_engine.graph_engine import RAGGraphEngine

# source = "/Users/chenketing/Desktop/project/llama_index/examples/paul_graham_essay/data/test/test_kg_text.txt"
# engine = RAGGraphEngine(knowledge_source=source, model_name="proxyllm", text_splitter=text_splitter)
embedding_factory = CFG.SYSTEM_APP.get_component(
"embedding_factory", EmbeddingFactory
)
@@ -244,6 +252,18 @@ def sync_knowledge_document(self, space_name, sync_request: DocumentSyncRequest)
embedding_factory=embedding_factory,
)
chunk_docs = client.read()
from pilot.graph_engine.graph_factory import RAGGraphFactory

rag_engine = CFG.SYSTEM_APP.get_component(
ComponentType.RAG_GRAPH_DEFAULT.value, RAGGraphFactory
).create()
rag_engine.knowledge_graph(docs=chunk_docs)
# docs = engine.search(
# "Comparing Curry and James in terms of their positions, playing styles, and achievements in the NBA"
# )
embedding_factory = CFG.SYSTEM_APP.get_component(
"embedding_factory", EmbeddingFactory
)
# update document status
doc.status = SyncStatus.RUNNING.name
doc.chunk_size = len(chunk_docs)