[Bug]: Error using pydantic response with chat engine #15694
Comments
Royisaboy
added
bug
|
@Royisaboy Hey there! I'm here to help you with any bugs, questions, or contributions you might have. Let's get to the bottom of this together. To resolve the You should use the Here is the relevant part of the code that shows how def _escape_braces(text: str) -> str:
"""
Escape braces in text.
Only captures template variables, skips already escaped braces.
"""
def replace(match):
if match.group(0).startswith("{{") and match.group(0).endswith("}}"):
return match.group(0) # Already escaped, return as is
return "{{" + match.group(1) + "}}"
pattern = r"(?<!\{)\{([^{}]+?)\}(?!\})"
return re.sub(pattern, replace, text)
def _escape_json(messages: Sequence[ChatMessage]) -> Sequence[ChatMessage]:
"""Escape JSON in messages."""
new_messages = []
for message in messages:
if isinstance(message.content, str):
escaped_msg = _escape_braces(message.content)
new_messages.append(
ChatMessage(
role=message.role,
content=escaped_msg,
additional_kwargs=message.additional_kwargs,
)
)
else:
new_messages.append(message)
return new_messagesIn the @llm_chat_callback()
def chat(self, messages: Sequence[ChatMessage], **kwargs: Any) -> ChatResponse:
"""Chat endpoint for LLM."""
chat_prompt = ChatPromptTemplate(message_templates=_escape_json(messages))
output = self.llm.structured_predict(
output_cls=self.output_cls, prompt=chat_prompt
)
return ChatResponse(
message=ChatMessage(
role=MessageRole.ASSISTANT, content=output.model_dump_json()
),
raw=output,
)Ensure that your messages are passed through |
|
I suspect you are using llama-index-core v0.11.x with lower versions of llama-index-embeddings-openai We migrated to pydantic v2 in v0.11.x and you'll need to use the latest versions of integration packages as well for compatibility UPDATE: My bad, it looks like I have it the other way around. You're core is |
|
If you install these versions of the packages, then everything should run as expected. |
|
KeyError Traceback (most recent call last) 10 frames /usr/local/lib/python3.10/dist-packages/llama_index/core/callbacks/utils.py in wrapper(self, *args, **kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/chat_engine/condense_plus_context.py in chat(self, message, chat_history) /usr/local/lib/python3.10/dist-packages/llama_index/core/instrumentation/dispatcher.py in wrapper(func, instance, args, kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/llms/callbacks.py in wrapped_llm_chat(_self, messages, **kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/llms/structured_llm.py in chat(self, messages, **kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/instrumentation/dispatcher.py in wrapper(func, instance, args, kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/llms/llm.py in structured_predict(self, output_cls, prompt, **prompt_args) /usr/local/lib/python3.10/dist-packages/llama_index/core/instrumentation/dispatcher.py in wrapper(func, instance, args, kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/program/function_program.py in call(self, llm_kwargs, *args, **kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/prompts/base.py in format_messages(failed resolving arguments) KeyError: '\n "project_name"' project_name is a field in our data model |
|
Sorry, could we also import both from llama_index.core.bridge.pydantic import BaseModel, Field |
|
still same error |
|
Could you try this and see if you get the same errors? I was able to run through this code with no problem. It looks like there's an issue with the output from the llm from your traceback, where there is an extra "\n " in the from llama_index.core.bridge.pydantic import BaseModel, Field
from llama_index.core.llms import ChatMessage
from typing import List
class ExecSummary(BaseModel):
"""Data model for a executive summary for a project."""
project_name: str = Field(..., description="The project name.")
summary: str = Field(..., description="A high level summary of the project progress. Please limit it to max 100 characters.")
risk_description: str = Field(..., description="A risk description that highlights the most outstanding risk. Please limit it to max 15 characters")
risk_level: str = Field(..., description="A risk level that is choosen from 'high', 'medium' and 'low'.")
class AllExecSummary(BaseModel):
"""Data model for a list of executive summaries for all projects."""
summaries: List[ExecSummary] = Field(..., description="A list of executive summaries.")
sllm = llm.as_structured_llm(output_cls=AllExecSummary)
input_msg = ChatMessage.from_str("Generate an all exec summary.")
output = sllm.chat([input_msg])
output_obj = output.raw
print(output_obj) |
|
this one works for me (it also worked before) but why it doesn't work for chat engine? |
|
I suspect there's something not right with the LLM output of the chat engine which breaks things when trying to construct the Pydantic BaseModel. Doing some more digging. |
nerdai
added
P0
|
Would you be able to share a bit more code for me to replicate the error? |
|
openai.api_key = "XXX" llm = llama_index_openai(temperature=0.1, model="gpt-4o") |
|
hmmm this seems to work for me rather than producing the error from before (shared again below) The other thing of note tho is that my |
|
can you share your full working codes? |
%pip install llama-index-vector-stores-pinecone==0.2.1 llama-index-embeddings-openai==0.2.3 -q
import json
from llama_index.core import VectorStoreIndex
from llama_index.llms.openai import OpenAI
from llama_index.embeddings.openai import OpenAIEmbedding
from llama_index.vector_stores.pinecone import PineconeVectorStore
from pinecone import Pinecone
from llama_index.core.bridge.pydantic import BaseModel, Field
from llama_index.core.llms import ChatMessage
from typing import List
class ExecSummary(BaseModel):
"""Data model for a executive summary for a project."""
project_name: str = Field(..., description="The project name.")
summary: str = Field(..., description="A high level summary of the project progress. Please limit it to max 100 characters.")
risk_description: str = Field(..., description="A risk description that highlights the most outstanding risk. Please limit it to max 15 characters")
risk_level: str = Field(..., description="A risk level that is choosen from 'high', 'medium' and 'low'.")
class AllExecSummary(BaseModel):
"""Data model for a list of executive summaries for all projects."""
summaries: List[ExecSummary] = Field(..., description="A list of executive summaries.")
pinecone_api_key = "..."
embed_model = OpenAIEmbedding(model="text-embedding-3-small")
llm = OpenAI(temperature=0.1, model="gpt-4o")
pc = Pinecone(api_key=pinecone_api_key)
pinecone_index = pc.Index("quickstart")
vector_store = PineconeVectorStore(pinecone_index=pinecone_index)
loaded_index = VectorStoreIndex.from_vector_store(vector_store=vector_store, embed_model=embed_model)
sllm = llm.as_structured_llm(output_cls=AllExecSummary)
chat_engine = loaded_index.as_chat_engine(
chat_mode="context",
similarity_top_k=5,
llm=sllm
)
prompt = '''
Provide executive summaries for project amber and project lina from last two weeks.
You must call the tool to generate the formatted output.
The response of this instruction must be in JSON format.
'''
response = chat_engine.chat(prompt)
obj = AllExecSummary(**json.loads(response.response)) |
|
it cannot be reproduced. for example, openai is not being imported. but after i get all libraries ready i saw same issue. can you show a full google colab script? |
|
Hmmm. I don't really need to import openai directly here. The script above should work completely, though it does require two api keys:
I can share a Google collab version of this shortly. |
|
@Royisaboy: here is a google colab version of the code snippet I shared earlier. |
|
same code producing error when i am trying to run it on my end: KeyError Traceback (most recent call last) 10 frames /usr/local/lib/python3.10/dist-packages/llama_index/core/callbacks/utils.py in wrapper(self, *args, **kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/chat_engine/context.py in chat(self, message, chat_history, prev_chunks) /usr/local/lib/python3.10/dist-packages/llama_index/core/instrumentation/dispatcher.py in wrapper(func, instance, args, kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/llms/callbacks.py in wrapped_llm_chat(_self, messages, **kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/llms/structured_llm.py in chat(self, messages, **kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/instrumentation/dispatcher.py in wrapper(func, instance, args, kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/llms/llm.py in structured_predict(self, output_cls, prompt, **prompt_args) /usr/local/lib/python3.10/dist-packages/llama_index/core/instrumentation/dispatcher.py in wrapper(func, instance, args, kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/program/function_program.py in call(self, llm_kwargs, *args, **kwargs) /usr/local/lib/python3.10/dist-packages/llama_index/core/prompts/base.py in format_messages(failed resolving arguments) KeyError: '\n "project_name"' |
|
are you using python3.10? |
|
Hi ! I encountered the same issue. I tried using llama-index-agent-openai==0.3.1 and 0.3.2 (latest version). import json
import os
from typing import List
from llama_index.core import Settings, SimpleDirectoryReader, VectorStoreIndex
from llama_index.core.bridge.pydantic import BaseModel, Field
from llama_index.embeddings.openai import OpenAIEmbedding
from llama_index.llms.openai import OpenAI
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
class Movie(BaseModel):
name: str = Field(..., alias="name")
length: int = Field(..., alias="length")
class MovieList(BaseModel):
name: str = Field(..., alias="name")
movies: List[Movie] = Field(..., alias="movies")
Settings.embed_model = OpenAIEmbedding()
Settings.llm = OpenAI("gpt-4", temperature=0).as_structured_llm(
output_cls=MovieList
)
movies_lists = [
{
"name": "Horror movies list",
"movies": [
{"name": "The Conjuring", "length": 112},
{"name": "The Exorcist", "length": 122},
{"name": "The Shining", "length": 146},
],
},
{
"name": "Sci-fi movies list",
"movies": [
{"name": "Star Wars", "length": 121},
{"name": "Interstellar", "length": 169},
{"name": "The Matrix", "length": 136},
],
},
]
# Save as text file
with open("movie_list.txt", "w") as f:
f.write(json.dumps(movies_lists))
# load only the file
documents = SimpleDirectoryReader(input_files=["movie_list.txt"]).load_data()
index = VectorStoreIndex.from_documents(
documents,
)
query_engine = index.as_query_engine(response_mode="refine", verbose=True)
response = query_engine.query(
"Give me the horror movies lists and their associated movies"
)
# Print the response to debug
print(f"Response: {response}")Output |
Bug Description
I'm trying to get pydantic response with chat engine but it threw errors. i tried different versions of llamaindex but none of them worked.
Version
0.10.36
Steps to Reproduce
!pip install llama-index-vector-stores-pinecone==0.1.7 pinecone-client==3.2.2 llama-index-embeddings-openai
!pip install llama-index==0.10.36
from llama_index.llms.openai import OpenAI as llama_index_openai
import openai
from llama_index.embeddings.openai import OpenAIEmbedding
from pinecone import Pinecone
from llama_index.vector_stores.pinecone import PineconeVectorStore
from llama_index.core import (
VectorStoreIndex
)
from typing import List
from pydantic.v1 import BaseModel, Field
class Movie(BaseModel):
"""Object representing a single movie."""
name: str = Field(..., description="Name of the movie.")
year: int = Field(..., description="Year of the movie.")
class Movies(BaseModel):
"""Object representing a list of movies."""
movies: List[Movie] = Field(..., description="List of movies.")
openai.api_key = "XXX"
embed_model = OpenAIEmbedding(model="text-embedding-3-small")
llm = llama_index_openai(temperature=0.1, model="gpt-4o")
pc = Pinecone(api_key="XXX")
pinecone_index = pc.Index("quick_start")
vector_store = PineconeVectorStore(pinecone_index=pinecone_index)
loaded_index = VectorStoreIndex.from_vector_store(vector_store=vector_store, embed_model=embed_model)
sllm = llm.as_structured_llm(Movies)
query_engine = loaded_index.as_chat_engine(
chat_mode="context",
similarity_top_k=5,
llm=sllm
)
prompt = '''
Please generate related movies to Titanic
'''
response = query_engine.chat(prompt)
Relevant Logs/Tracbacks
The text was updated successfully, but these errors were encountered: