From 32e4a0357804417195fffe01b5c6b41fab10a6a8 Mon Sep 17 00:00:00 2001 From: Josh Reini Date: Thu, 4 Jan 2024 19:02:09 -0500 Subject: [PATCH] bump py scripts --- .../py_script_quickstarts/all_tools.py | 705 +++++++++++------- .../langchain_quickstart.py | 110 ++- .../llama_index_quickstart.py | 94 ++- .../py_script_quickstarts/quickstart.py | 142 ++-- .../text2text_quickstart.py | 68 +- 5 files changed, 712 insertions(+), 407 deletions(-) diff --git a/trulens_eval/examples/quickstart/py_script_quickstarts/all_tools.py b/trulens_eval/examples/quickstart/py_script_quickstarts/all_tools.py index b2545aa06..7bb48dd20 100644 --- a/trulens_eval/examples/quickstart/py_script_quickstarts/all_tools.py +++ b/trulens_eval/examples/quickstart/py_script_quickstarts/all_tools.py @@ -2,30 +2,33 @@ # coding: utf-8 # # Langchain Quickstart -# +# # In this quickstart you will create a simple LLM Chain and learn how to log it and get feedback on an LLM response. -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/langchain_quickstart.ipynb) # ## Setup # ### Add API keys # For this quickstart you will need Open AI and Huggingface keys + + # ! pip install trulens_eval==0.20.0 openai==1.3.7 langchain chromadb langchainhub bs4 -import os + + +import os os.environ["OPENAI_API_KEY"] = "sk-..." + # ### Import from LangChain and TruLens + + # Imports main tools: -from trulens_eval import Feedback -from trulens_eval import Huggingface -from trulens_eval import Tru -from trulens_eval import TruChain +from trulens_eval import TruChain, Feedback, Huggingface, Tru from trulens_eval.schema import FeedbackResult - tru = Tru() tru.reset_database() @@ -40,8 +43,11 @@ from langchain.vectorstores import Chroma from langchain_core.runnables import RunnablePassthrough + # ### Load documents + + loader = WebBaseLoader( web_paths=("https://lilianweng.github.io/posts/2023-06-23-agent/",), bs_kwargs=dict( @@ -52,196 +58,236 @@ ) docs = loader.load() + # ### Create Vector Store -text_splitter = RecursiveCharacterTextSplitter( - chunk_size=1000, chunk_overlap=200 -) + + +text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) splits = text_splitter.split_documents(docs) -vectorstore = Chroma.from_documents( - documents=splits, embedding=OpenAIEmbeddings() -) +vectorstore = Chroma.from_documents(documents=splits, embedding=OpenAIEmbeddings( +)) + # ### Create RAG + + retriever = vectorstore.as_retriever() prompt = hub.pull("rlm/rag-prompt") llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0) - def format_docs(docs): return "\n\n".join(doc.page_content for doc in docs) - rag_chain = ( - { - "context": retriever | format_docs, - "question": RunnablePassthrough() - } | prompt | llm | StrOutputParser() + {"context": retriever | format_docs, "question": RunnablePassthrough()} + | prompt + | llm + | StrOutputParser() ) + # ### Send your first request + + rag_chain.invoke("What is Task Decomposition?") + # ## Initialize Feedback Function(s) -import numpy as np + from trulens_eval.feedback.provider import OpenAI +import numpy as np # Initialize provider class openai = OpenAI() # select context to be used in feedback. the location of context is app specific. from trulens_eval.app import App - context = App.select_context(rag_chain) from trulens_eval.feedback import Groundedness - grounded = Groundedness(groundedness_provider=OpenAI()) # Define a groundedness feedback function f_groundedness = ( - Feedback(grounded.groundedness_measure_with_cot_reasons - ).on(context.collect()) # collect context chunks into a list - .on_output().aggregate(grounded.grounded_statements_aggregator) + Feedback(grounded.groundedness_measure_with_cot_reasons) + .on(context.collect()) # collect context chunks into a list + .on_output() + .aggregate(grounded.grounded_statements_aggregator) ) # Question/answer relevance between overall question and answer. f_qa_relevance = Feedback(openai.relevance).on_input_output() # Question/statement relevance between question and each context chunk. f_context_relevance = ( - Feedback(openai.qs_relevance).on_input().on(context).aggregate(np.mean) -) + Feedback(openai.qs_relevance) + .on_input() + .on(context) + .aggregate(np.mean) + ) + # ## Instrument chain for logging with TruLens -tru_recorder = TruChain( - rag_chain, + + +tru_recorder = TruChain(rag_chain, app_id='Chain1_ChatApplication', - feedbacks=[f_qa_relevance, f_context_relevance, f_groundedness] -) + feedbacks=[f_qa_relevance, f_context_relevance, f_groundedness]) + + + with tru_recorder as recording: llm_response = rag_chain.invoke("What is Task Decomposition?") print(llm_response) + # ## Retrieve records and feedback + + # The record of the app invocation can be retrieved from the `recording`: -rec = recording.get() # use .get if only one record +rec = recording.get() # use .get if only one record # recs = recording.records # use .records if multiple print(rec) + + + # The results of the feedback functions can be rertireved from the record. These # are `Future` instances (see `concurrent.futures`). You can use `as_completed` # to wait until they have finished evaluating. from concurrent.futures import as_completed -for feedback_future in as_completed(rec.feedback_results): +for feedback_future in as_completed(rec.feedback_results): feedback, feedback_result = feedback_future.result() - + feedback: Feedback feedbac_result: FeedbackResult print(feedback.name, feedback_result.result) -records, feedback = tru.get_records_and_feedback( - app_ids=["Chain1_ChatApplication"] -) + + + +records, feedback = tru.get_records_and_feedback(app_ids=["Chain1_ChatApplication"]) records.head() + + + tru.get_leaderboard(app_ids=["Chain1_ChatApplication"]) + # ## Explore in a Dashboard -tru.run_dashboard() # open a local streamlit app to explore + + +tru.run_dashboard() # open a local streamlit app to explore # tru.stop_dashboard() # stop if needed + # Alternatively, you can run `trulens-eval` from a command line in the same folder to start the dashboard. # Note: Feedback functions evaluated in the deferred manner can be seen in the "Progress" page of the TruLens dashboard. # # Llama-Index Quickstart -# +# # In this quickstart you will create a simple Llama Index App and learn how to log it and get feedback on an LLM response. -# +# # For evaluation, we will leverage the "hallucination triad" of groundedness, context relevance and answer relevance. -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/llama_index_quickstart.ipynb) # ## Setup -# +# # ### Install dependencies # Let's install some of the dependencies for this notebook if we don't have them already -# pip install trulens_eval==0.20.0 llama_index>=0.9.15post2 html2text>=2020.1.16 + + +# pip install trulens_eval==0.20.0 llama_index>=0.9.15post2 html2text>=2020.1.16 + # ### Add API keys # For this quickstart, you will need Open AI and Huggingface keys. The OpenAI key is used for embeddings and GPT, and the Huggingface key is used for evaluation. -import os + +import os os.environ["OPENAI_API_KEY"] = "sk-..." + # ### Import from TruLens -from trulens_eval import Tru + +from trulens_eval import Tru tru = Tru() + # ### Create Simple LLM Application -# +# # This example uses LlamaIndex which internally uses an OpenAI LLM. + + from llama_index import VectorStoreIndex from llama_index.readers.web import SimpleWebPageReader -documents = SimpleWebPageReader(html_to_text=True).load_data( - ["http://paulgraham.com/worked.html"] -) +documents = SimpleWebPageReader( + html_to_text=True +).load_data(["http://paulgraham.com/worked.html"]) index = VectorStoreIndex.from_documents(documents) query_engine = index.as_query_engine() + # ### Send your first request + + response = query_engine.query("What did the author do growing up?") print(response) + # ## Initialize Feedback Function(s) + + import numpy as np # Initialize provider class from trulens_eval.feedback.provider.openai import OpenAI - openai = OpenAI() # select context to be used in feedback. the location of context is app specific. from trulens_eval.app import App - context = App.select_context(query_engine) # imports for feedback from trulens_eval import Feedback + # Define a groundedness feedback function from trulens_eval.feedback import Groundedness - grounded = Groundedness(groundedness_provider=OpenAI()) f_groundedness = ( - Feedback(grounded.groundedness_measure_with_cot_reasons - ).on(context.collect()) # collect context chunks into a list - .on_output().aggregate(grounded.grounded_statements_aggregator) + Feedback(grounded.groundedness_measure_with_cot_reasons) + .on(context.collect()) # collect context chunks into a list + .on_output() + .aggregate(grounded.grounded_statements_aggregator) ) # Question/answer relevance between overall question and answer. @@ -249,82 +295,111 @@ def format_docs(docs): # Question/statement relevance between question and each context chunk. f_qs_relevance = ( - Feedback(openai.qs_relevance).on_input().on(context).aggregate(np.mean) + Feedback(openai.qs_relevance) + .on_input() + .on(context) + .aggregate(np.mean) ) + # ## Instrument app for logging with TruLens -from trulens_eval import TruLlama -tru_query_engine_recorder = TruLlama( - query_engine, + +from trulens_eval import TruLlama +tru_query_engine_recorder = TruLlama(query_engine, app_id='LlamaIndex_App1', - feedbacks=[f_groundedness, f_qa_relevance, f_qs_relevance] -) + feedbacks=[f_groundedness, f_qa_relevance, f_qs_relevance]) + + + # or as context manager with tru_query_engine_recorder as recording: query_engine.query("What did the author do growing up?") + # ## Retrieve records and feedback + + # The record of the app invocation can be retrieved from the `recording`: -rec = recording.get() # use .get if only one record +rec = recording.get() # use .get if only one record # recs = recording.records # use .records if multiple print(rec) + + + # The results of the feedback functions can be rertireved from the record. These # are `Future` instances (see `concurrent.futures`). You can use `as_completed` # to wait until they have finished evaluating. -from concurrent.futures import as_completed - from trulens_eval.schema import FeedbackResult -for feedback_future in as_completed(rec.feedback_results): - feedback, feedback_result = feedback_future.result() +from concurrent.futures import as_completed +for feedback_future in as_completed(rec.feedback_results): + feedback, feedback_result = feedback_future.result() + feedback: Feedback feedbac_result: FeedbackResult print(feedback.name, feedback_result.result) + + + records, feedback = tru.get_records_and_feedback(app_ids=["LlamaIndex_App1"]) records.head() + + + tru.get_leaderboard(app_ids=["LlamaIndex_App1"]) + # ## Explore in a Dashboard -tru.run_dashboard() # open a local streamlit app to explore + + +tru.run_dashboard() # open a local streamlit app to explore # tru.stop_dashboard() # stop if needed + # Alternatively, you can run `trulens-eval` from a command line in the same folder to start the dashboard. # Note: Feedback functions evaluated in the deferred manner can be seen in the "Progress" page of the TruLens dashboard. # # TruLens Quickstart -# +# # In this quickstart you will create a RAG from scratch and learn how to log it and get feedback on an LLM response. -# +# # For evaluation, we will leverage the "hallucination triad" of groundedness, context relevance and answer relevance. -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/quickstart.ipynb) -# ! pip install trulens_eval==0.20.0 chromadb==0.4.18 openai==1.3.7 -import os +# ! pip install trulens_eval==0.20.1 chromadb==0.4.18 openai==1.3.7 + + + + +import os os.environ["OPENAI_API_KEY"] = "..." + # ## Get Data -# +# # In this case, we'll just initialize some simple text in the notebook. + + university_info = """ The University of Washington, founded in 1861 in Seattle, is a public research university with over 45,000 students across three campuses in Seattle, Tacoma, and Bothell. @@ -333,53 +408,66 @@ def format_docs(docs): including one of the largest library systems in the world. """ + # ## Create Vector Store -# +# # Create a chromadb vector store in memory. -from openai import OpenAI + +from openai import OpenAI oai_client = OpenAI() oai_client.embeddings.create( - model="text-embedding-ada-002", input=university_info -) + model="text-embedding-ada-002", + input=university_info + ) + + + import chromadb from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction -embedding_function = OpenAIEmbeddingFunction( - api_key=os.environ.get('OPENAI_API_KEY'), - model_name="text-embedding-ada-002" -) +embedding_function = OpenAIEmbeddingFunction(api_key=os.environ.get('OPENAI_API_KEY'), + model_name="text-embedding-ada-002") + chroma_client = chromadb.Client() -vector_store = chroma_client.get_or_create_collection( - name="Universities", embedding_function=embedding_function -) +vector_store = chroma_client.get_or_create_collection(name="Universities", + embedding_function=embedding_function) + # Add the university_info to the embedding database. + + vector_store.add("uni_info", documents=university_info) + # ## Build RAG from scratch -# +# # Build a custom RAG from scratch, and add TruLens custom instrumentation. + + from trulens_eval import Tru from trulens_eval.tru_custom_app import instrument - tru = Tru() -class RAG_from_scratch: + +class RAG_from_scratch: @instrument def retrieve(self, query: str) -> list: """ Retrieve relevant text from vector store. """ - results = vector_store.query(query_texts=query, n_results=2) + results = vector_store.query( + query_texts=query, + n_results=2 + ) return results['documents'][0] @instrument @@ -388,19 +476,19 @@ def generate_completion(self, query: str, context_str: list) -> str: Generate answer from context. """ completion = oai_client.chat.completions.create( - model="gpt-3.5-turbo", - temperature=0, - messages=[ - { - "role": "user", - "content": - f"We have provided context information below. \n" - f"---------------------\n" - f"{context_str}" - f"\n---------------------\n" - f"Given this information, please answer the question: {query}" - } - ] + model="gpt-3.5-turbo", + temperature=0, + messages= + [ + {"role": "user", + "content": + f"We have provided context information below. \n" + f"---------------------\n" + f"{context_str}" + f"\n---------------------\n" + f"Given this information, please answer the question: {query}" + } + ] ).choices[0].message.content return completion @@ -410,20 +498,21 @@ def query(self, query: str) -> str: completion = self.generate_completion(query, context_str) return completion - rag = RAG_from_scratch() + # ## Set up feedback functions. -# +# # Here we'll use groundedness, answer relevance and context relevance to detect hallucination. -import numpy as np -from trulens_eval import Feedback -from trulens_eval import Select + +from trulens_eval import Feedback, Select from trulens_eval.feedback import Groundedness from trulens_eval.feedback.provider.openai import OpenAI as fOpenAI +import numpy as np + # Initialize provider class fopenai = fOpenAI() @@ -431,61 +520,75 @@ def query(self, query: str) -> str: # Define a groundedness feedback function f_groundedness = ( - Feedback( - grounded.groundedness_measure_with_cot_reasons, name="Groundedness" - ).on(Select.RecordCalls.retrieve.rets.collect() - ).on_output().aggregate(grounded.grounded_statements_aggregator) + Feedback(grounded.groundedness_measure_with_cot_reasons, name = "Groundedness") + .on(Select.RecordCalls.retrieve.rets.collect()) + .on_output() + .aggregate(grounded.grounded_statements_aggregator) ) # Question/answer relevance between overall question and answer. f_qa_relevance = ( - Feedback(fopenai.relevance_with_cot_reasons, name="Answer Relevance").on( - Select.RecordCalls.retrieve.args.query - ).on_output() + Feedback(fopenai.relevance_with_cot_reasons, name = "Answer Relevance") + .on(Select.RecordCalls.retrieve.args.query) + .on_output() ) # Question/statement relevance between question and each context chunk. f_context_relevance = ( - Feedback(fopenai.qs_relevance_with_cot_reasons, - name="Context Relevance").on( - Select.RecordCalls.retrieve.args.query - ).on(Select.RecordCalls.retrieve.rets.collect() - ).aggregate(np.mean) + Feedback(fopenai.qs_relevance_with_cot_reasons, name = "Context Relevance") + .on(Select.RecordCalls.retrieve.args.query) + .on(Select.RecordCalls.retrieve.rets.collect()) + .aggregate(np.mean) ) + # ## Construct the app # Wrap the custom RAG with TruCustomApp, add list of feedbacks for eval + + from trulens_eval import TruCustomApp +tru_rag = TruCustomApp(rag, + app_id = 'RAG v1', + feedbacks = [f_groundedness, f_qa_relevance, f_context_relevance]) -tru_rag = TruCustomApp( - rag, - app_id='RAG v1', - feedbacks=[f_groundedness, f_qa_relevance, f_context_relevance] -) # ## Run the app # Use `tru_rag` as a context manager for the custom RAG-from-scratch app. + + with tru_rag as recording: rag.query("When was the University of Washington founded?") + + + tru.get_leaderboard(app_ids=["RAG v1"]) + + + tru.run_dashboard() + # # Prototype Evals # This notebook shows the use of the dummy feedback function provider which # behaves like the huggingface provider except it does not actually perform any # network calls and just produces constant results. It can be used to prototype # feedback function wiring for your apps before invoking potentially slow (to # run/to load) feedback functions. -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/prototype_evals.ipynb) # ## Import libraries -# ! pip install trulens_eval==0.20.0 + + +# ! pip install trulens_eval==0.20.1 + + + from trulens_eval import Feedback from trulens_eval import Tru @@ -494,44 +597,49 @@ def query(self, query: str) -> str: tru.run_dashboard() + # ## Set keys -import os + +import os os.environ["OPENAI_API_KEY"] = "..." + # ## Build the app -from openai import OpenAI + +from openai import OpenAI oai_client = OpenAI() from trulens_eval.tru_custom_app import instrument - class APP: - @instrument def completion(self, prompt): completion = oai_client.chat.completions.create( - model="gpt-3.5-turbo", - temperature=0, - messages=[ - { - "role": "user", - "content": f"Please answer the question: {prompt}" - } - ] - ).choices[0].message.content + model="gpt-3.5-turbo", + temperature=0, + messages= + [ + {"role": "user", + "content": + f"Please answer the question: {prompt}" + } + ] + ).choices[0].message.content return completion - - + llm_app = APP() + # ## Create dummy feedback -# +# # By setting the provider as `Dummy()`, you can erect your evaluation suite and then easily substitute in a real model provider (e.g. OpenAI) later. + + from trulens_eval.feedback.provider.hugs import Dummy # hugs = Huggingface() @@ -539,29 +647,43 @@ def completion(self, prompt): f_positive_sentiment = Feedback(hugs.positive_sentiment).on_output() + # ## Create the app + + # add trulens as a context manager for llm_app with dummy feedback from trulens_eval import TruCustomApp +tru_app = TruCustomApp(llm_app, + app_id = 'LLM App v1', + feedbacks = [f_positive_sentiment]) -tru_app = TruCustomApp( - llm_app, app_id='LLM App v1', feedbacks=[f_positive_sentiment] -) # ## Run the app + + with tru_app as recording: llm_app.completion('give me a good name for a colorful sock company') + + + tru.get_leaderboard(app_ids=[tru_app.app_id]) + # ## Logging Human Feedback -# +# # In many situations, it can be useful to log human feedback from your users about your LLM app's performance. Combining human feedback along with automated feedback can help you drill down on subsets of your app that underperform, and uncover new failure modes. This example will walk you through a simple example of recording human feedback with TruLens. -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/human_feedback.ipynb) -# ! pip install trulens_eval==0.20.0 openai==1.3.7 + + +# ! pip install trulens_eval==0.20.1 openai==1.3.7 + + + import os @@ -570,82 +692,92 @@ def completion(self, prompt): tru = Tru() + # ## Set Keys -# +# # For this example, you need an OpenAI key. + + os.environ["OPENAI_API_KEY"] = "..." + # ## Set up your app -# +# # Here we set up a custom application using just an OpenAI chat completion. The process for logging human feedback is the same however you choose to set up your app. -from openai import OpenAI + +from openai import OpenAI oai_client = OpenAI() from trulens_eval.tru_custom_app import instrument - class APP: - @instrument def completion(self, prompt): completion = oai_client.chat.completions.create( - model="gpt-3.5-turbo", - temperature=0, - messages=[ - { - "role": "user", - "content": f"Please answer the question: {prompt}" - } - ] - ).choices[0].message.content + model="gpt-3.5-turbo", + temperature=0, + messages= + [ + {"role": "user", + "content": + f"Please answer the question: {prompt}" + } + ] + ).choices[0].message.content return completion - - + llm_app = APP() # add trulens as a context manager for llm_app -tru_app = TruCustomApp(llm_app, app_id='LLM App v1') +tru_app = TruCustomApp(llm_app, app_id = 'LLM App v1') + # ## Run the app + + with tru_app as recording: llm_app.completion("Give me 10 names for a colorful sock company") + + + # Get the record to add the feedback to. record = recording.get() + # ## Create a mechamism for recording human feedback. -# +# # Be sure to click an emoji in the record to record `human_feedback` to log. -from ipywidgets import Button -from ipywidgets import HBox -from ipywidgets import VBox + + +from ipywidgets import Button, HBox, VBox thumbs_up_button = Button(description='👍') thumbs_down_button = Button(description='👎') human_feedback = None - def on_thumbs_up_button_clicked(b): global human_feedback human_feedback = 1 - def on_thumbs_down_button_clicked(b): global human_feedback human_feedback = 0 - thumbs_up_button.on_click(on_thumbs_up_button_clicked) thumbs_down_button.on_click(on_thumbs_down_button_clicked) HBox([thumbs_up_button, thumbs_down_button]) + + + # add the human feedback to a particular app and record tru.add_feedback( name="Human Feedack", @@ -654,103 +786,126 @@ def on_thumbs_down_button_clicked(b): result=human_feedback ) + # ## See the result logged with your app. + + tru.get_leaderboard(app_ids=[tru_app.app_id]) + + + + + + # # Ground Truth Evaluations -# +# # In this quickstart you will create a evaluate a LangChain app using ground truth. Ground truth evaluation can be especially useful during early LLM experiments when you have a small set of example queries that are critical to get right. -# +# # Ground truth evaluation works by comparing the similarity of an LLM response compared to its matching verified response. -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/groundtruth_evals.ipynb) # ### Add API keys # For this quickstart, you will need Open AI keys. -# ! pip install trulens_eval==0.20.0 openai==1.3.7 + + +# ! pip install trulens_eval==0.20.1 openai==1.3.7 + + + import os +os.environ["OPENAI_API_KEY"] = "sk-..." + + -os.environ["OPENAI_API_KEY"] = "..." from trulens_eval import Tru tru = Tru() + # ### Create Simple LLM Application -from openai import OpenAI + +from openai import OpenAI oai_client = OpenAI() from trulens_eval.tru_custom_app import instrument - class APP: - @instrument def completion(self, prompt): completion = oai_client.chat.completions.create( - model="gpt-3.5-turbo", - temperature=0, - messages=[ - { - "role": "user", - "content": f"Please answer the question: {prompt}" - } - ] - ).choices[0].message.content + model="gpt-3.5-turbo", + temperature=0, + messages= + [ + {"role": "user", + "content": + f"Please answer the question: {prompt}" + } + ] + ).choices[0].message.content return completion - - + llm_app = APP() + # ## Initialize Feedback Function(s) + + from trulens_eval import Feedback from trulens_eval.feedback import GroundTruthAgreement golden_set = [ - { - "query": "who invented the lightbulb?", - "response": "Thomas Edison" - }, { - "query": "¿quien invento la bombilla?", - "response": "Thomas Edison" - } + {"query": "who invented the lightbulb?", "response": "Thomas Edison"}, + {"query": "¿quien invento la bombilla?", "response": "Thomas Edison"} ] -f_groundtruth = Feedback( - GroundTruthAgreement(golden_set).agreement_measure, name="Ground Truth" -).on_input_output() +f_groundtruth = Feedback(GroundTruthAgreement(golden_set).agreement_measure, name = "Ground Truth").on_input_output() + # ## Instrument chain for logging with TruLens + + # add trulens as a context manager for llm_app from trulens_eval import TruCustomApp +tru_app = TruCustomApp(llm_app, app_id = 'LLM App v1', feedbacks = [f_groundtruth]) + + -tru_app = TruCustomApp(llm_app, app_id='LLM App v1', feedbacks=[f_groundtruth]) # Instrumented query engine can operate as a context manager: with tru_app as recording: llm_app.completion("¿quien invento la bombilla?") llm_app.completion("who invented the lightbulb?") + # ## See results + + tru.get_leaderboard(app_ids=[tru_app.app_id]) + # # Logging Methods -# +# # ## Automatic Logging -# +# # The simplest method for logging with TruLens is by wrapping with TruChain and # including the tru argument, as shown in the quickstart. -# +# # This is done like so: + + # Imports main tools: from trulens_eval import Feedback from trulens_eval import Huggingface @@ -781,12 +936,19 @@ def completion(self, prompt): chain = LLMChain(llm=llm, prompt=chat_prompt_template, verbose=True) -truchain = TruChain(chain, app_id='Chain1_ChatApplication', tru=tru) +truchain = TruChain( + chain, + app_id='Chain1_ChatApplication', + tru=tru +) truchain("This will be automatically logged.") + # Feedback functions can also be logged automatically by providing them in a list # to the feedbacks arg. + + # Initialize Huggingface-based feedback function collection class: hugs = Huggingface() @@ -795,79 +957,109 @@ def completion(self, prompt): # By default this will check language match on the main app input and main app # output. + + + truchain = TruChain( chain, app_id='Chain1_ChatApplication', - feedbacks=[f_lang_match], # feedback functions + feedbacks=[f_lang_match], # feedback functions tru=tru ) truchain("This will be automatically logged.") + # ## Manual Logging -# +# # ### Wrap with TruChain to instrument your chain + + tc = TruChain(chain, app_id='Chain1_ChatApplication') + # ### Set up logging and instrumentation -# +# # Making the first call to your wrapped LLM Application will now also produce a log or "record" of the chain execution. -# +# + + prompt_input = 'que hora es?' gpt3_response, record = tc.call_with_record(prompt_input) + # We can log the records but first we need to log the chain itself. + + tru.add_app(app=truchain) + # Then we can log the record: + + tru.add_record(record) + # ### Log App Feedback # Capturing app feedback such as user feedback of the responses can be added with # one call. + + thumb_result = True tru.add_feedback( - name="👍 (1) or 👎 (0)", record_id=record.record_id, result=thumb_result + name="👍 (1) or 👎 (0)", + record_id=record.record_id, + result=thumb_result ) + # ### Evaluate Quality -# +# # Following the request to your app, you can then evaluate LLM quality using # feedback functions. This is completed in a sequential call to minimize latency # for your application, and evaluations will also be logged to your local machine. -# +# # To get feedback on the quality of your LLM, you can use any of the provided # feedback functions or add your own. -# +# # To assess your LLM quality, you can provide the feedback functions to # `tru.run_feedback()` in a list provided to `feedback_functions`. -# +# + + feedback_results = tru.run_feedback_functions( - record=record, feedback_functions=[f_lang_match] + record=record, + feedback_functions=[f_lang_match] ) for result in feedback_results: print(result) + # After capturing feedback, you can then log it to your local database. + + tru.add_feedbacks(feedback_results) + # ### Out-of-band Feedback evaluation -# +# # In the above example, the feedback function evaluation is done in the same # process as the chain evaluation. The alternative approach is the use the # provided persistent evaluator started via # `tru.start_deferred_feedback_evaluator`. Then specify the `feedback_mode` for # `TruChain` as `deferred` to let the evaluator handle the feedback functions. -# +# # For demonstration purposes, we start the evaluator here but it can be started in # another process. + + truchain: TruChain = TruChain( chain, app_id='Chain1_ChatApplication', @@ -882,23 +1074,21 @@ def completion(self, prompt): tru.start_evaluator() # tru.stop_evaluator() + # # Custom Functions -# +# # Feedback functions are an extensible framework for evaluating LLMs. You can add your own feedback functions to evaluate the qualities required by your application by updating `trulens_eval/feedback.py`, or simply creating a new provider class and feedback function in youre notebook. If your contributions would be useful for others, we encourage you to contribute to TruLens! -# +# # Feedback functions are organized by model provider into Provider classes. -# +# # The process for adding new feedback functions is: # 1. Create a new Provider class or locate an existing one that applies to your feedback function. If your feedback function does not rely on a model provider, you can create a standalone class. Add the new feedback function method to your selected class. Your new method can either take a single text (str) as a parameter or both prompt (str) and response (str). It should return a float between 0 (worst) and 1 (best). -from trulens_eval import Feedback -from trulens_eval import Provider -from trulens_eval import Select -from trulens_eval import Tru -class StandAlone(Provider): +from trulens_eval import Provider, Feedback, Select, Tru +class StandAlone(Provider): def custom_feedback(self, my_text_field: str) -> float: """ A dummy function of text inputs to float outputs. @@ -914,64 +1104,69 @@ def custom_feedback(self, my_text_field: str) -> float: # 2. Instantiate your provider and feedback functions. The feedback function is wrapped by the trulens-eval Feedback class which helps specify what will get sent to your function parameters (For example: Select.RecordInput or Select.RecordOutput) + + standalone = StandAlone() -f_custom_function = Feedback(standalone.custom_feedback - ).on(my_text_field=Select.RecordOutput) +f_custom_function = Feedback(standalone.custom_feedback).on( + my_text_field=Select.RecordOutput +) + # 3. Your feedback function is now ready to use just like the out of the box feedback functions. Below is an example of it being used. + + tru = Tru() feedback_results = tru.run_feedback_functions( - record=record, feedback_functions=[f_custom_function] + record=record, + feedback_functions=[f_custom_function] ) tru.add_feedbacks(feedback_results) + # ## Multi-Output Feedback functions # Trulens also supports multi-output feedback functions. As a typical feedback function will output a float between 0 and 1, multi-output should output a dictionary of `output_key` to a float between 0 and 1. The feedbacks table will print the feedback with column `feedback_name:::outputkey` -multi_output_feedback = Feedback( - lambda input_param: { - 'output_key1': 0.1, - 'output_key2': 0.9 - }, name="multi" -).on(input_param=Select.RecordOutput) + + +multi_output_feedback = Feedback(lambda input_param: {'output_key1': 0.1, 'output_key2': 0.9}, name="multi").on( + input_param=Select.RecordOutput +) feedback_results = tru.run_feedback_functions( - record=record, feedback_functions=[multi_output_feedback] + record=record, + feedback_functions=[multi_output_feedback] ) tru.add_feedbacks(feedback_results) + + + # Aggregators will run on the same dict keys. import numpy as np - -multi_output_feedback = Feedback( - lambda input_param: { - 'output_key1': 0.1, - 'output_key2': 0.9 - }, - name="multi-agg" -).on(input_param=Select.RecordOutput).aggregate(np.mean) +multi_output_feedback = Feedback(lambda input_param: {'output_key1': 0.1, 'output_key2': 0.9}, name="multi-agg").on( + input_param=Select.RecordOutput +).aggregate(np.mean) feedback_results = tru.run_feedback_functions( - record=record, feedback_functions=[multi_output_feedback] + record=record, + feedback_functions=[multi_output_feedback] ) tru.add_feedbacks(feedback_results) + + # For multi-context chunking, an aggregator can operate on a list of multi output dictionaries. def dict_aggregator(list_dict_input): agg = 0 for dict_input in list_dict_input: agg += dict_input['output_key1'] return agg - - -multi_output_feedback = Feedback( - lambda input_param: { - 'output_key1': 0.1, - 'output_key2': 0.9 - }, - name="multi-agg-dict" -).on(input_param=Select.RecordOutput).aggregate(dict_aggregator) +multi_output_feedback = Feedback(lambda input_param: {'output_key1': 0.1, 'output_key2': 0.9}, name="multi-agg-dict").on( + input_param=Select.RecordOutput +).aggregate(dict_aggregator) feedback_results = tru.run_feedback_functions( - record=record, feedback_functions=[multi_output_feedback] + record=record, + feedback_functions=[multi_output_feedback] ) tru.add_feedbacks(feedback_results) + diff --git a/trulens_eval/examples/quickstart/py_script_quickstarts/langchain_quickstart.py b/trulens_eval/examples/quickstart/py_script_quickstarts/langchain_quickstart.py index 4e25a4460..5c1a2ebcb 100644 --- a/trulens_eval/examples/quickstart/py_script_quickstarts/langchain_quickstart.py +++ b/trulens_eval/examples/quickstart/py_script_quickstarts/langchain_quickstart.py @@ -2,30 +2,33 @@ # coding: utf-8 # # Langchain Quickstart -# +# # In this quickstart you will create a simple LLM Chain and learn how to log it and get feedback on an LLM response. -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/langchain_quickstart.ipynb) # ## Setup # ### Add API keys # For this quickstart you will need Open AI and Huggingface keys + + # ! pip install trulens_eval==0.20.0 openai==1.3.7 langchain chromadb langchainhub bs4 -import os + + +import os os.environ["OPENAI_API_KEY"] = "sk-..." + # ### Import from LangChain and TruLens + + # Imports main tools: -from trulens_eval import Feedback -from trulens_eval import Huggingface -from trulens_eval import Tru -from trulens_eval import TruChain +from trulens_eval import TruChain, Feedback, Huggingface, Tru from trulens_eval.schema import FeedbackResult - tru = Tru() tru.reset_database() @@ -40,8 +43,11 @@ from langchain.vectorstores import Chroma from langchain_core.runnables import RunnablePassthrough + # ### Load documents + + loader = WebBaseLoader( web_paths=("https://lilianweng.github.io/posts/2023-06-23-agent/",), bs_kwargs=dict( @@ -52,121 +58,147 @@ ) docs = loader.load() + # ### Create Vector Store -text_splitter = RecursiveCharacterTextSplitter( - chunk_size=1000, chunk_overlap=200 -) + + +text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200) splits = text_splitter.split_documents(docs) -vectorstore = Chroma.from_documents( - documents=splits, embedding=OpenAIEmbeddings() -) +vectorstore = Chroma.from_documents(documents=splits, embedding=OpenAIEmbeddings( +)) + # ### Create RAG + + retriever = vectorstore.as_retriever() prompt = hub.pull("rlm/rag-prompt") llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0) - def format_docs(docs): return "\n\n".join(doc.page_content for doc in docs) - rag_chain = ( - { - "context": retriever | format_docs, - "question": RunnablePassthrough() - } | prompt | llm | StrOutputParser() + {"context": retriever | format_docs, "question": RunnablePassthrough()} + | prompt + | llm + | StrOutputParser() ) + # ### Send your first request + + rag_chain.invoke("What is Task Decomposition?") + # ## Initialize Feedback Function(s) -import numpy as np + from trulens_eval.feedback.provider import OpenAI +import numpy as np # Initialize provider class openai = OpenAI() # select context to be used in feedback. the location of context is app specific. from trulens_eval.app import App - context = App.select_context(rag_chain) from trulens_eval.feedback import Groundedness - grounded = Groundedness(groundedness_provider=OpenAI()) # Define a groundedness feedback function f_groundedness = ( - Feedback(grounded.groundedness_measure_with_cot_reasons - ).on(context.collect()) # collect context chunks into a list - .on_output().aggregate(grounded.grounded_statements_aggregator) + Feedback(grounded.groundedness_measure_with_cot_reasons) + .on(context.collect()) # collect context chunks into a list + .on_output() + .aggregate(grounded.grounded_statements_aggregator) ) # Question/answer relevance between overall question and answer. f_qa_relevance = Feedback(openai.relevance).on_input_output() # Question/statement relevance between question and each context chunk. f_context_relevance = ( - Feedback(openai.qs_relevance).on_input().on(context).aggregate(np.mean) -) + Feedback(openai.qs_relevance) + .on_input() + .on(context) + .aggregate(np.mean) + ) + # ## Instrument chain for logging with TruLens -tru_recorder = TruChain( - rag_chain, + + +tru_recorder = TruChain(rag_chain, app_id='Chain1_ChatApplication', - feedbacks=[f_qa_relevance, f_context_relevance, f_groundedness] -) + feedbacks=[f_qa_relevance, f_context_relevance, f_groundedness]) + + + with tru_recorder as recording: llm_response = rag_chain.invoke("What is Task Decomposition?") print(llm_response) + # ## Retrieve records and feedback + + # The record of the app invocation can be retrieved from the `recording`: -rec = recording.get() # use .get if only one record +rec = recording.get() # use .get if only one record # recs = recording.records # use .records if multiple print(rec) + + + # The results of the feedback functions can be rertireved from the record. These # are `Future` instances (see `concurrent.futures`). You can use `as_completed` # to wait until they have finished evaluating. from concurrent.futures import as_completed -for feedback_future in as_completed(rec.feedback_results): +for feedback_future in as_completed(rec.feedback_results): feedback, feedback_result = feedback_future.result() - + feedback: Feedback feedbac_result: FeedbackResult print(feedback.name, feedback_result.result) -records, feedback = tru.get_records_and_feedback( - app_ids=["Chain1_ChatApplication"] -) + + + +records, feedback = tru.get_records_and_feedback(app_ids=["Chain1_ChatApplication"]) records.head() + + + tru.get_leaderboard(app_ids=["Chain1_ChatApplication"]) + # ## Explore in a Dashboard -tru.run_dashboard() # open a local streamlit app to explore + + +tru.run_dashboard() # open a local streamlit app to explore # tru.stop_dashboard() # stop if needed + # Alternatively, you can run `trulens-eval` from a command line in the same folder to start the dashboard. # Note: Feedback functions evaluated in the deferred manner can be seen in the "Progress" page of the TruLens dashboard. diff --git a/trulens_eval/examples/quickstart/py_script_quickstarts/llama_index_quickstart.py b/trulens_eval/examples/quickstart/py_script_quickstarts/llama_index_quickstart.py index 32e895587..36c3e2174 100644 --- a/trulens_eval/examples/quickstart/py_script_quickstarts/llama_index_quickstart.py +++ b/trulens_eval/examples/quickstart/py_script_quickstarts/llama_index_quickstart.py @@ -2,76 +2,90 @@ # coding: utf-8 # # Llama-Index Quickstart -# +# # In this quickstart you will create a simple Llama Index App and learn how to log it and get feedback on an LLM response. -# +# # For evaluation, we will leverage the "hallucination triad" of groundedness, context relevance and answer relevance. -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/llama_index_quickstart.ipynb) # ## Setup -# +# # ### Install dependencies # Let's install some of the dependencies for this notebook if we don't have them already -# pip install trulens_eval==0.20.0 llama_index>=0.9.15post2 html2text>=2020.1.16 + + +# pip install trulens_eval==0.20.0 llama_index>=0.9.15post2 html2text>=2020.1.16 + # ### Add API keys # For this quickstart, you will need Open AI and Huggingface keys. The OpenAI key is used for embeddings and GPT, and the Huggingface key is used for evaluation. -import os + +import os os.environ["OPENAI_API_KEY"] = "sk-..." + # ### Import from TruLens -from trulens_eval import Tru + +from trulens_eval import Tru tru = Tru() + # ### Create Simple LLM Application -# +# # This example uses LlamaIndex which internally uses an OpenAI LLM. + + from llama_index import VectorStoreIndex from llama_index.readers.web import SimpleWebPageReader -documents = SimpleWebPageReader(html_to_text=True).load_data( - ["http://paulgraham.com/worked.html"] -) +documents = SimpleWebPageReader( + html_to_text=True +).load_data(["http://paulgraham.com/worked.html"]) index = VectorStoreIndex.from_documents(documents) query_engine = index.as_query_engine() + # ### Send your first request + + response = query_engine.query("What did the author do growing up?") print(response) + # ## Initialize Feedback Function(s) + + import numpy as np # Initialize provider class from trulens_eval.feedback.provider.openai import OpenAI - openai = OpenAI() # select context to be used in feedback. the location of context is app specific. from trulens_eval.app import App - context = App.select_context(query_engine) # imports for feedback from trulens_eval import Feedback + # Define a groundedness feedback function from trulens_eval.feedback import Groundedness - grounded = Groundedness(groundedness_provider=OpenAI()) f_groundedness = ( - Feedback(grounded.groundedness_measure_with_cot_reasons - ).on(context.collect()) # collect context chunks into a list - .on_output().aggregate(grounded.grounded_statements_aggregator) + Feedback(grounded.groundedness_measure_with_cot_reasons) + .on(context.collect()) # collect context chunks into a list + .on_output() + .aggregate(grounded.grounded_statements_aggregator) ) # Question/answer relevance between overall question and answer. @@ -79,60 +93,82 @@ # Question/statement relevance between question and each context chunk. f_qs_relevance = ( - Feedback(openai.qs_relevance).on_input().on(context).aggregate(np.mean) + Feedback(openai.qs_relevance) + .on_input() + .on(context) + .aggregate(np.mean) ) + # ## Instrument app for logging with TruLens -from trulens_eval import TruLlama -tru_query_engine_recorder = TruLlama( - query_engine, + +from trulens_eval import TruLlama +tru_query_engine_recorder = TruLlama(query_engine, app_id='LlamaIndex_App1', - feedbacks=[f_groundedness, f_qa_relevance, f_qs_relevance] -) + feedbacks=[f_groundedness, f_qa_relevance, f_qs_relevance]) + + + # or as context manager with tru_query_engine_recorder as recording: query_engine.query("What did the author do growing up?") + # ## Retrieve records and feedback + + # The record of the app invocation can be retrieved from the `recording`: -rec = recording.get() # use .get if only one record +rec = recording.get() # use .get if only one record # recs = recording.records # use .records if multiple print(rec) + + + # The results of the feedback functions can be rertireved from the record. These # are `Future` instances (see `concurrent.futures`). You can use `as_completed` # to wait until they have finished evaluating. -from concurrent.futures import as_completed - from trulens_eval.schema import FeedbackResult -for feedback_future in as_completed(rec.feedback_results): - feedback, feedback_result = feedback_future.result() +from concurrent.futures import as_completed +for feedback_future in as_completed(rec.feedback_results): + feedback, feedback_result = feedback_future.result() + feedback: Feedback feedbac_result: FeedbackResult print(feedback.name, feedback_result.result) + + + records, feedback = tru.get_records_and_feedback(app_ids=["LlamaIndex_App1"]) records.head() + + + tru.get_leaderboard(app_ids=["LlamaIndex_App1"]) + # ## Explore in a Dashboard -tru.run_dashboard() # open a local streamlit app to explore + + +tru.run_dashboard() # open a local streamlit app to explore # tru.stop_dashboard() # stop if needed + # Alternatively, you can run `trulens-eval` from a command line in the same folder to start the dashboard. # Note: Feedback functions evaluated in the deferred manner can be seen in the "Progress" page of the TruLens dashboard. diff --git a/trulens_eval/examples/quickstart/py_script_quickstarts/quickstart.py b/trulens_eval/examples/quickstart/py_script_quickstarts/quickstart.py index aa64be45c..520d1e04b 100644 --- a/trulens_eval/examples/quickstart/py_script_quickstarts/quickstart.py +++ b/trulens_eval/examples/quickstart/py_script_quickstarts/quickstart.py @@ -2,23 +2,30 @@ # coding: utf-8 # # TruLens Quickstart -# +# # In this quickstart you will create a RAG from scratch and learn how to log it and get feedback on an LLM response. -# +# # For evaluation, we will leverage the "hallucination triad" of groundedness, context relevance and answer relevance. -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/quickstart.ipynb) -# ! pip install trulens_eval==0.20.0 chromadb==0.4.18 openai==1.3.7 -import os +# ! pip install trulens_eval==0.20.1 chromadb==0.4.18 openai==1.3.7 + + + + +import os os.environ["OPENAI_API_KEY"] = "..." + # ## Get Data -# +# # In this case, we'll just initialize some simple text in the notebook. + + university_info = """ The University of Washington, founded in 1861 in Seattle, is a public research university with over 45,000 students across three campuses in Seattle, Tacoma, and Bothell. @@ -27,53 +34,66 @@ including one of the largest library systems in the world. """ + # ## Create Vector Store -# +# # Create a chromadb vector store in memory. -from openai import OpenAI + +from openai import OpenAI oai_client = OpenAI() oai_client.embeddings.create( - model="text-embedding-ada-002", input=university_info -) + model="text-embedding-ada-002", + input=university_info + ) + + + import chromadb from chromadb.utils.embedding_functions import OpenAIEmbeddingFunction -embedding_function = OpenAIEmbeddingFunction( - api_key=os.environ.get('OPENAI_API_KEY'), - model_name="text-embedding-ada-002" -) +embedding_function = OpenAIEmbeddingFunction(api_key=os.environ.get('OPENAI_API_KEY'), + model_name="text-embedding-ada-002") + chroma_client = chromadb.Client() -vector_store = chroma_client.get_or_create_collection( - name="Universities", embedding_function=embedding_function -) +vector_store = chroma_client.get_or_create_collection(name="Universities", + embedding_function=embedding_function) + # Add the university_info to the embedding database. + + vector_store.add("uni_info", documents=university_info) + # ## Build RAG from scratch -# +# # Build a custom RAG from scratch, and add TruLens custom instrumentation. + + from trulens_eval import Tru from trulens_eval.tru_custom_app import instrument - tru = Tru() -class RAG_from_scratch: + +class RAG_from_scratch: @instrument def retrieve(self, query: str) -> list: """ Retrieve relevant text from vector store. """ - results = vector_store.query(query_texts=query, n_results=2) + results = vector_store.query( + query_texts=query, + n_results=2 + ) return results['documents'][0] @instrument @@ -82,19 +102,19 @@ def generate_completion(self, query: str, context_str: list) -> str: Generate answer from context. """ completion = oai_client.chat.completions.create( - model="gpt-3.5-turbo", - temperature=0, - messages=[ - { - "role": "user", - "content": - f"We have provided context information below. \n" - f"---------------------\n" - f"{context_str}" - f"\n---------------------\n" - f"Given this information, please answer the question: {query}" - } - ] + model="gpt-3.5-turbo", + temperature=0, + messages= + [ + {"role": "user", + "content": + f"We have provided context information below. \n" + f"---------------------\n" + f"{context_str}" + f"\n---------------------\n" + f"Given this information, please answer the question: {query}" + } + ] ).choices[0].message.content return completion @@ -104,20 +124,21 @@ def query(self, query: str) -> str: completion = self.generate_completion(query, context_str) return completion - rag = RAG_from_scratch() + # ## Set up feedback functions. -# +# # Here we'll use groundedness, answer relevance and context relevance to detect hallucination. -import numpy as np -from trulens_eval import Feedback -from trulens_eval import Select + +from trulens_eval import Feedback, Select from trulens_eval.feedback import Groundedness from trulens_eval.feedback.provider.openai import OpenAI as fOpenAI +import numpy as np + # Initialize provider class fopenai = fOpenAI() @@ -125,45 +146,54 @@ def query(self, query: str) -> str: # Define a groundedness feedback function f_groundedness = ( - Feedback( - grounded.groundedness_measure_with_cot_reasons, name="Groundedness" - ).on(Select.RecordCalls.retrieve.rets.collect() - ).on_output().aggregate(grounded.grounded_statements_aggregator) + Feedback(grounded.groundedness_measure_with_cot_reasons, name = "Groundedness") + .on(Select.RecordCalls.retrieve.rets.collect()) + .on_output() + .aggregate(grounded.grounded_statements_aggregator) ) # Question/answer relevance between overall question and answer. f_qa_relevance = ( - Feedback(fopenai.relevance_with_cot_reasons, name="Answer Relevance").on( - Select.RecordCalls.retrieve.args.query - ).on_output() + Feedback(fopenai.relevance_with_cot_reasons, name = "Answer Relevance") + .on(Select.RecordCalls.retrieve.args.query) + .on_output() ) # Question/statement relevance between question and each context chunk. f_context_relevance = ( - Feedback(fopenai.qs_relevance_with_cot_reasons, - name="Context Relevance").on( - Select.RecordCalls.retrieve.args.query - ).on(Select.RecordCalls.retrieve.rets.collect() - ).aggregate(np.mean) + Feedback(fopenai.qs_relevance_with_cot_reasons, name = "Context Relevance") + .on(Select.RecordCalls.retrieve.args.query) + .on(Select.RecordCalls.retrieve.rets.collect()) + .aggregate(np.mean) ) + # ## Construct the app # Wrap the custom RAG with TruCustomApp, add list of feedbacks for eval + + from trulens_eval import TruCustomApp +tru_rag = TruCustomApp(rag, + app_id = 'RAG v1', + feedbacks = [f_groundedness, f_qa_relevance, f_context_relevance]) -tru_rag = TruCustomApp( - rag, - app_id='RAG v1', - feedbacks=[f_groundedness, f_qa_relevance, f_context_relevance] -) # ## Run the app # Use `tru_rag` as a context manager for the custom RAG-from-scratch app. + + with tru_rag as recording: rag.query("When was the University of Washington founded?") + + + tru.get_leaderboard(app_ids=["RAG v1"]) + + + tru.run_dashboard() + diff --git a/trulens_eval/examples/quickstart/py_script_quickstarts/text2text_quickstart.py b/trulens_eval/examples/quickstart/py_script_quickstarts/text2text_quickstart.py index e4286339b..dc9a96c25 100644 --- a/trulens_eval/examples/quickstart/py_script_quickstarts/text2text_quickstart.py +++ b/trulens_eval/examples/quickstart/py_script_quickstarts/text2text_quickstart.py @@ -2,92 +2,104 @@ # coding: utf-8 # # Text to Text Quickstart -# +# # In this quickstart you will create a simple text to text application and learn how to log it and get feedback. -# +# # [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/truera/trulens/blob/main/trulens_eval/examples/quickstart/text2text_quickstart.ipynb) # ## Setup # ### Add API keys # For this quickstart you will need an OpenAI Key. -# ! pip install trulens_eval==0.20.0 openai==1.3.1 -import os +# ! pip install trulens_eval==0.20.1 openai==1.3.1 + + + + +import os os.environ["OPENAI_API_KEY"] = "..." + # ### Import from TruLens + + # Create openai client from openai import OpenAI - client = OpenAI() # Imports main tools: -from trulens_eval import Feedback -from trulens_eval import OpenAI as fOpenAI -from trulens_eval import Tru - +from trulens_eval import Feedback, OpenAI as fOpenAI, Tru tru = Tru() tru.reset_database() + # ### Create Simple Text to Text Application -# +# # This example uses a bare bones OpenAI LLM, and a non-LLM just for demonstration purposes. + def llm_standalone(prompt): return client.chat.completions.create( - model="gpt-3.5-turbo", - messages=[ - { - "role": - "system", - "content": - "You are a question and answer bot, and you answer super upbeat." - }, { - "role": "user", - "content": prompt - } + model="gpt-3.5-turbo", + messages=[ + {"role": "system", "content": "You are a question and answer bot, and you answer super upbeat."}, + {"role": "user", "content": prompt} ] ).choices[0].message.content # ### Send your first request -prompt_input = "How good is language AI?" + + +prompt_input="How good is language AI?" prompt_output = llm_standalone(prompt_input) prompt_output + # ## Initialize Feedback Function(s) + + # Initialize OpenAI-based feedback function collection class: fopenai = fOpenAI() # Define a relevance function from openai f_relevance = Feedback(fopenai.relevance).on_input_output() + # ## Instrument the callable for logging with TruLens + + from trulens_eval import TruBasicApp +tru_llm_standalone_recorder = TruBasicApp(llm_standalone, app_id="Happy Bot", feedbacks=[f_relevance]) + + -tru_llm_standalone_recorder = TruBasicApp( - llm_standalone, app_id="Happy Bot", feedbacks=[f_relevance] -) with tru_llm_standalone_recorder as recording: tru_llm_standalone_recorder.app(prompt_input) + # ## Explore in a Dashboard -tru.run_dashboard() # open a local streamlit app to explore + + +tru.run_dashboard() # open a local streamlit app to explore # tru.stop_dashboard() # stop if needed + # Alternatively, you can run `trulens-eval` from a command line in the same folder to start the dashboard. # ## Or view results directly in your notebook -tru.get_records_and_feedback(app_ids=[] - )[0] # pass an empty list of app_ids to get all + + +tru.get_records_and_feedback(app_ids=[])[0] # pass an empty list of app_ids to get all +