04_04_RAG-Evaluation-Challenge.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RAG Evaluation Challenge\n",
    "\n",
    "1. Prepare the evaluation data set using the products pdf files. For this challenge, we can simulate with at least 10 rows.\n",
    "2. Perform evaluations on Groundedness, Relevance, Coherence, Fluency, Similarity as well as the 4 Risk and Safety Metrics.\n",
    "3. Display the results in Azure AI Foundry"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Azure Configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dotenv import load_dotenv\n",
    "import os\n",
    "\n",
    "load_dotenv() # take environment variables from .env.\n",
    "\n",
    "azure_openai_endpoint = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n",
    "azure_openai_key = os.getenv(\"AZURE_OPENAI_KEY\")\n",
    "azure_openai_deployment = os.getenv(\"AZURE_OPENAI_DEPLOYMENT\")\n",
    "azure_openai_embeddings_deployment = os.getenv(\"AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT\")\n",
    "azure_openai_api_version = \"2024-10-01-preview\"\n",
    "azure_search_service_endpoint = os.getenv(\"AZURE_SEARCH_SERVICE_ENDPOINT\")\n",
    "azure_search_service_admin_key = os.getenv(\"AZURE_SEARCH_ADMIN_KEY\")\n",
    "azure_search_service_index_name = \"product-index\"\n",
    "azure_storage_connection_string = os.getenv(\"AZURE_STORAGE_CONNECTION_STRING\")\n",
    "\n",
    "model_config = {\n",
    "    \"azure_endpoint\": azure_openai_endpoint,\n",
    "    \"api_key\": azure_openai_key,\n",
    "    \"azure_deployment\": azure_openai_deployment,\n",
    "}\n",
    "\n",
    "azure_subscription_id = os.getenv(\"AZURE_SUBSCRIPTION_ID\")\n",
    "azure_resource_group_name = os.getenv(\"AZURE_RESOURCE_GROUP_NAME\")\n",
    "azure_project_name = os.getenv(\"AZURE_PROJECT_NAME\")\n",
    "\n",
    "azure_ai_project = {\n",
    "    \"subscription_id\": azure_subscription_id,\n",
    "    \"resource_group_name\": azure_resource_group_name,\n",
    "    \"project_name\": azure_project_name,\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.document_loaders import DirectoryLoader\n",
    "\n",
    "path = \"data/products/\"\n",
    "loader = DirectoryLoader(path, glob=\"**/*.pdf\")\n",
    "docs = loader.load()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare the Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_openai import AzureChatOpenAI\n",
    "from langchain_openai import AzureOpenAIEmbeddings\n",
    "from ragas.llms import LangchainLLMWrapper\n",
    "from ragas.embeddings import LangchainEmbeddingsWrapper\n",
    "\n",
    "generator_llm = LangchainLLMWrapper(AzureChatOpenAI(\n",
    "    openai_api_version=azure_openai_api_version,\n",
    "    azure_endpoint=azure_openai_endpoint,\n",
    "    azure_deployment=azure_openai_deployment,\n",
    "    model=azure_openai_deployment,\n",
    "    validate_base_url=False,\n",
    "    api_key=azure_openai_key\n",
    "))\n",
    "\n",
    "# init the embeddings for answer_relevancy, answer_correctness and answer_similarity\n",
    "generator_embeddings = LangchainEmbeddingsWrapper(AzureOpenAIEmbeddings(\n",
    "    openai_api_version=azure_openai_api_version,\n",
    "    azure_endpoint=azure_openai_endpoint,\n",
    "    azure_deployment=azure_openai_embeddings_deployment,\n",
    "    model=azure_openai_embeddings_deployment,\n",
    "    api_key=azure_openai_key\n",
    "))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate the Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ragas.testset import TestsetGenerator\n",
    "\n",
    "generator = TestsetGenerator(llm=generator_llm, embedding_model=generator_embeddings)\n",
    "dataset = generator.generate_with_langchain_docs(docs, testset_size=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save just the query and ground truth to a JSONL file for evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = dataset.to_pandas()\n",
    "\n",
    "# Create DataFrame\n",
    "df = dataset.to_pandas()\n",
    "\n",
    "# Function to clean text\n",
    "def clean_text(text_list):\n",
    "    cleaned_text_list = []\n",
    "    for text in text_list:\n",
    "        # Remove the UUID (assuming it's always at the start and followed by two newlines)\n",
    "        cleaned_text = text.split('\\n\\n', 1)[-1]\n",
    "        cleaned_text_list.append(cleaned_text)\n",
    "    return cleaned_text_list\n",
    "\n",
    "# Apply the function to the 'reference_contexts' column to remove UUID at the start\n",
    "df['reference_contexts'] = df['reference_contexts'].apply(clean_text)\n",
    "\n",
    "# Save to CSV file\n",
    "df.to_csv('data/output/product-evalset.csv', index=False)\n",
    "\n",
    "# Create a new DataFrame for EvalCollection\n",
    "eval_collection = pd.DataFrame(columns=['query', 'response', 'context', 'ground_truth'])\n",
    "\n",
    "# Populate the new DataFrame\n",
    "eval_collection['query'] = df['user_input']\n",
    "eval_collection['ground_truth'] = df['reference']\n",
    "eval_collection['response'] = ''\n",
    "eval_collection['context'] = ''\n",
    "\n",
    "# Save the DataFrame as a JSONL file\n",
    "eval_collection.to_json('data/output/product-eval.jsonl', orient='records', lines=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate the Response and Context from the Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azure.search.documents import SearchClient\n",
    "from azure.search.documents.models import VectorizableTextQuery\n",
    "from azure.core.credentials import AzureKeyCredential\n",
    "from openai import AzureOpenAI\n",
    "\n",
    "# Get credential from Azure AI Search Admin key\n",
    "credential = AzureKeyCredential(azure_search_service_admin_key)\n",
    "search_client = SearchClient(endpoint=azure_search_service_endpoint, \n",
    "                             credential=credential, \n",
    "                             index_name=azure_search_service_index_name)\n",
    "\n",
    "# Azure OpenAI client\n",
    "openai_client = AzureOpenAI(\n",
    "    # to get version: https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-deprecation\n",
    "    api_version=azure_openai_api_version,\n",
    "    azure_endpoint=azure_openai_endpoint,\n",
    "    api_key=azure_openai_key)\n",
    "\n",
    "# Provide instructions to the model\n",
    "SYSTEM_PROMPT=\"\"\"\n",
    "You are an AI assistant that helps users learn from the information found in the source material.\n",
    "Answer the query using only the sources provided below.\n",
    "Use bullets if the answer has multiple points.\n",
    "If the answer is longer than 3 sentences, provide a summary.\n",
    "Answer ONLY with the facts listed in the list of sources below. Cite your source when you answer the question\n",
    "If there isn't enough information below, say you don't know.\n",
    "Do not generate answers that don't use the sources below.\n",
    "Query: {query}\n",
    "Sources:\\n{sources}\n",
    "\"\"\"\n",
    "# Iterate over each row in eval_collection\n",
    "for index, row in eval_collection.iterrows():\n",
    "    # User Query\n",
    "    query = row['query']  \n",
    "\n",
    "    # Convert query into vector form\n",
    "    vector_query = VectorizableTextQuery(text=query, \n",
    "                                        k_nearest_neighbors=50, \n",
    "                                        fields=\"text_vector\",\n",
    "                                        weight=1)\n",
    "\n",
    "    results = search_client.search(\n",
    "        query_type=\"semantic\", \n",
    "        semantic_configuration_name='my-semantic-config',\n",
    "        search_text=query,\n",
    "        vector_queries= [vector_query],\n",
    "        select=[\"title\",\"chunk\"],\n",
    "        top=3,\n",
    "    )\n",
    "\n",
    "    # Use a unique separator to make the sources distinct. \n",
    "    # We chose repeated equal signs (=) followed by a newline because it's unlikely the source documents contain this sequence.\n",
    "    sources_formatted = \"=================\\n\".join([f'TITLE: {document[\"title\"]}, CONTENT: {document[\"chunk\"]}' for document in results])\n",
    "\n",
    "    # Update the context in the DataFrame\n",
    "    eval_collection.at[index, 'context'] = sources_formatted\n",
    "\n",
    "    response = openai_client.chat.completions.create(\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": SYSTEM_PROMPT.format(query=query, sources=sources_formatted)\n",
    "            }\n",
    "        ],\n",
    "        model=azure_openai_deployment\n",
    "    )\n",
    "\n",
    "    # Update the response in the DataFrame\n",
    "    eval_collection.at[index, 'response'] = response.choices[0].message.content\n",
    "\n",
    "# Save the updated DataFrame as a JSONL file\n",
    "eval_collection.to_json('data/output/product-eval.jsonl', orient='records', lines=True)\n",
    "\n",
    "# Print success message\n",
    "print(\"EvalCollection has been saved to product-eval.jsonl\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run evaluation and view in Azure AI Foundry"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azure.ai.evaluation import evaluate\n",
    "from azure.ai.evaluation import GroundednessEvaluator, RetrievalEvaluator, RelevanceEvaluator, CoherenceEvaluator, FluencyEvaluator, SimilarityEvaluator, F1ScoreEvaluator\n",
    "from azure.ai.evaluation import RougeScoreEvaluator, RougeType\n",
    "from azure.ai.evaluation import BleuScoreEvaluator\n",
    "from azure.ai.evaluation import MeteorScoreEvaluator\n",
    "from azure.ai.evaluation import GleuScoreEvaluator\n",
    "from azure.ai.evaluation import ViolenceEvaluator, HateUnfairnessEvaluator, SelfHarmEvaluator,SexualEvaluator\n",
    "from azure.identity import DefaultAzureCredential\n",
    "import pandas as pd\n",
    "\n",
    "groundedness_eval = GroundednessEvaluator(model_config)\n",
    "retrieval_eval = RetrievalEvaluator(model_config)\n",
    "relevance_eval = RelevanceEvaluator(model_config)\n",
    "coherence_eval = CoherenceEvaluator(model_config)\n",
    "fluency_eval = FluencyEvaluator(model_config)\n",
    "similarity_eval = SimilarityEvaluator(model_config)\n",
    "f1_eval = F1ScoreEvaluator()\n",
    "rouge_eval = RougeScoreEvaluator(rouge_type=RougeType.ROUGE_1)\n",
    "bleu_eval = BleuScoreEvaluator()\n",
    "meteor_eval = MeteorScoreEvaluator(\n",
    "    alpha=0.9,\n",
    "    beta=3.0,\n",
    "    gamma=0.5\n",
    ")\n",
    "gleu_eval = GleuScoreEvaluator()\n",
    "violence_eval = ViolenceEvaluator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())\n",
    "hateunfairness_eval = HateUnfairnessEvaluator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())\n",
    "selfharm_eval = SelfHarmEvaluator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())\n",
    "sexual_eval = SexualEvaluator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())\n",
    "\n",
    "path = \"data/output/product-eval.jsonl\"\n",
    "\n",
    "result = evaluate(\n",
    "    data=path, # provide your data here\n",
    "    evaluators={\n",
    "        \"groundedness\": groundedness_eval,\n",
    "        \"retrieval\": retrieval_eval,\n",
    "        \"relevance\": relevance_eval,\n",
    "        \"coherence\": coherence_eval,\n",
    "        \"fluency\": fluency_eval,\n",
    "        \"similarity\": similarity_eval,\n",
    "        \"f1_score\": f1_eval,\n",
    "        \"rouge_score\": rouge_eval,\n",
    "        \"bleu_score\": bleu_eval,\n",
    "        \"meteor_score\": meteor_eval,\n",
    "        \"gleu_score\": gleu_eval,\n",
    "        \"violence_score\": violence_eval,\n",
    "        \"hateunfairness_score\": hateunfairness_eval,\n",
    "        \"selfharm_score\": selfharm_eval,\n",
    "        \"sexual_score\": sexual_eval \n",
    "    },\n",
    "    # column mapping\n",
    "    evaluator_config={\n",
    "        \"default\": {\n",
    "            \"query\": \"${data.query}\",\n",
    "            \"response\": \"${data.response}\",\n",
    "            \"context\": \"${data.context}\",\n",
    "            \"ground_truth\": \"${data.ground_truth}\"\n",
    "        }\n",
    "    },\n",
    "    azure_ai_project = azure_ai_project\n",
    ")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}