6-LangchainEvaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2f53c32b-ab49-45a6-a323-c48fb7680b47",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"NVIDIA_API_KEY\"] = \"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "c54ba82e-2195-4b12-b3e6-69859eb22c14",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_nvidia_ai_endpoints import ChatNVIDIA\n",
    "llm_nvidia = ChatNVIDIA(model=\"mistralai/mixtral-8x7b-instruct-v0.1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0605a9a6-fb00-4d68-8808-d9ddfa4b6ce9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.chat_models import ChatOllama\n",
    "\n",
    "llm_ollama = ChatOllama(model=\"llama3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "95da3cf0-5166-44e6-9682-0217ce0f45b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "llm = llm_nvidia"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "da2a0037-5974-47f5-afde-ea2046db20f1",
   "metadata": {},
   "source": [
    "# LangChain Evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e46e561-cad6-4bb1-b331-47889f1b8852",
   "metadata": {},
   "source": [
    "## Create our QandA application"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "03f30186-8350-4d9c-a4a4-29ee5ba6447a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.chains import RetrievalQA\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain.document_loaders import CSVLoader\n",
    "from langchain.indexes import VectorstoreIndexCreator\n",
    "from langchain.vectorstores import DocArrayInMemorySearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5598fbe7-6e48-4b5a-82fb-2ffc0f51bebd",
   "metadata": {},
   "outputs": [],
   "source": [
    "file = 'data/files/OutdoorClothingCatalog_1000.csv'\n",
    "loader = CSVLoader(file_path=file)\n",
    "data = loader.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "af67d88f-d9dd-4b0c-9b37-9061bfd48692",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mojo/miniconda3/lib/python3.12/site-packages/sentence_transformers/cross_encoder/CrossEncoder.py:11: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from tqdm.autonotebook import tqdm, trange\n",
      "/home/mojo/miniconda3/lib/python3.12/site-packages/huggingface_hub/file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "/home/mojo/miniconda3/lib/python3.12/site-packages/pydantic/_migration.py:283: UserWarning: `pydantic.error_wrappers:ValidationError` has been moved to `pydantic:ValidationError`.\n",
      "  warnings.warn(f'`{import_path}` has been moved to `{new_location}`.')\n"
     ]
    }
   ],
   "source": [
    "from langchain_huggingface import HuggingFaceEmbeddings\n",
    "\n",
    "embedding = HuggingFaceEmbeddings(model_name=\"BAAI/bge-small-en-v1.5\")\n",
    "\n",
    "index = VectorstoreIndexCreator(\n",
    "    vectorstore_cls=DocArrayInMemorySearch,\n",
    "    embedding = embedding\n",
    ").from_loaders([loader])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6e1f3fac-4a5b-4e85-ba71-ba783394bdf6",
   "metadata": {},
   "outputs": [],
   "source": [
    "qa = RetrievalQA.from_chain_type(\n",
    "    llm=llm, \n",
    "    chain_type=\"stuff\", \n",
    "    retriever=index.vectorstore.as_retriever(), \n",
    "    verbose=True,\n",
    "    chain_type_kwargs = {\n",
    "        \"document_separator\": \"<<<<>>>>>\"\n",
    "    }\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "518be883-ba6b-4c36-ad81-a0b5a31cdce8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content=\": 10\\nname: Cozy Comfort Pullover Set, Stripe\\ndescription: Perfect for lounging, this striped knit set lives up to its name. We used ultrasoft fabric and an easy design that's as comfortable at bedtime as it is when we have to make a quick run out.\\n\\nSize & Fit\\n- Pants are Favorite Fit: Sits lower on the waist.\\n- Relaxed Fit: Our most generous fit sits farthest from the body.\\n\\nFabric & Care\\n- In the softest blend of 63% polyester, 35% rayon and 2% spandex.\\n\\nAdditional Features\\n- Relaxed fit top with raglan sleeves and rounded hem.\\n- Pull-on pants have a wide elastic waistband and drawstring, side pockets and a modern slim leg.\\n\\nImported.\", metadata={'source': 'OutdoorClothingCatalog_1000.csv', 'row': 10})"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2141dc27-0f6e-4aff-96bc-79c4e5479c64",
   "metadata": {},
   "source": [
    "### Hard-coded examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "af7848b8-4cae-4a7e-9f93-8552added1f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "examples = [\n",
    "    {\"query\": \"Do the Cozy Comfort Pullover Set\\\n",
    "    have side pockets?\",\n",
    "    \"answer\": \"Yes\"\n",
    "    },\n",
    "     {\"query\": \"What collection is the Ultra-Lofty \\\n",
    "    850 Stretch Down Hooded Jacket from?\",\n",
    "    \"answer\": \"The DownTek collection\"\n",
    "     }\n",
    "]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df3f3d7d-8b5f-4e9f-9cd9-bd7650cef6cb",
   "metadata": {},
   "source": [
    "### LLM-Generated examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "deb23d33-0e5f-4b9a-8c99-62aaf0e7e4a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.evaluation.qa import QAGenerateChain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "c0f49ca3-265d-47bc-81d1-1fda73c5309d",
   "metadata": {},
   "outputs": [],
   "source": [
    "example_gen_chain = QAGenerateChain.from_llm(llm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "95ca368b-e31c-4a64-a5ce-c044aaece945",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mojo/miniconda3/lib/python3.12/site-packages/langchain/chains/llm.py:367: UserWarning: The apply_and_parse method is deprecated, instead pass an output parser directly to LLMChain.\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "new_examples = example_gen_chain.apply_and_parse(\n",
    "    [{\"doc\": t} for t in data[:5]]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "320cce2f-5afc-4c75-8df1-2df18fcf42d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_examples = [ex.get(\"qa_pairs\") for ex in new_examples]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3c979288-8770-4f3b-a5ef-0201eac71157",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': \"What is the material of the Women's Campside Oxfords and what is its special feature?\",\n",
       " 'answer': \"The Women's Campside Oxfords are made of soft canvas material, which provides a broken-in feel and look. One special feature of these shoes is the Cleansport NXT® antimicrobial odor control in the innersole.\"}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_examples[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8bf1cb93-b8b3-4ac1-8e32-102708317eb1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Document(page_content=\": 0\\nname: Women's Campside Oxfords\\ndescription: This ultracomfortable lace-to-toe Oxford boasts a super-soft canvas, thick cushioning, and quality construction for a broken-in feel from the first time you put them on. \\n\\nSize & Fit: Order regular shoe size. For half sizes not offered, order up to next whole size. \\n\\nSpecs: Approx. weight: 1 lb.1 oz. per pair. \\n\\nConstruction: Soft canvas material for a broken-in feel and look. Comfortable EVA innersole with Cleansport NXT® antimicrobial odor control. Vintage hunt, fish and camping motif on innersole. Moderate arch contour of innersole. EVA foam midsole for cushioning and support. Chain-tread-inspired molded rubber outsole with modified chain-tread pattern. Imported. \\n\\nQuestions? Please contact us for any inquiries.\", metadata={'source': 'OutdoorClothingCatalog_1000.csv', 'row': 0})"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7c4f5ce4-051d-4f2d-b9c1-f6d6bc025c77",
   "metadata": {},
   "outputs": [],
   "source": [
    "examples += new_examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "411892a7-0b21-4456-b4c7-c1e1dd486cde",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': 'Do the Cozy Comfort Pullover Set    have side pockets?',\n",
       " 'answer': 'Yes'}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "examples[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f7076eca-d480-466a-aa4b-215dfc6e1770",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/mojo/miniconda3/lib/python3.12/site-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The method `Chain.run` was deprecated in langchain 0.1.0 and will be removed in 0.3.0. Use invoke instead.\n",
      "  warn_deprecated(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "' Yes, the Cozy Comfort Pullover Set, Stripe (named 10 and 73) has side pockets on the pull-on pants according to the provided descriptions.'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa.run(examples[0][\"query\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6d73372-3d1d-4d1d-9d8c-f92707ac8547",
   "metadata": {},
   "source": [
    "### Manual Evaluation¶"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7cd40a9d-97a2-4c39-8d28-22cad569b3f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import langchain\n",
    "langchain.debug = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "589e2d5b-2aa7-49ba-8f53-29844719b03d",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[chain:RetrievalQA] Entering Chain run with input:\n",
      "\u001b[0m{\n",
      "  \"query\": \"Do the Cozy Comfort Pullover Set    have side pockets?\"\n",
      "}\n",
      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[chain:RetrievalQA > chain:StuffDocumentsChain] Entering Chain run with input:\n",
      "\u001b[0m[inputs]\n",
      "\u001b[32;1m\u001b[1;3m[chain/start]\u001b[0m \u001b[1m[chain:RetrievalQA > chain:StuffDocumentsChain > chain:LLMChain] Entering Chain run with input:\n",
      "\u001b[0m{\n",
      "  \"question\": \"Do the Cozy Comfort Pullover Set    have side pockets?\",\n",
      "  \"context\": \": 10\\nname: Cozy Comfort Pullover Set, Stripe\\ndescription: Perfect for lounging, this striped knit set lives up to its name. We used ultrasoft fabric and an easy design that's as comfortable at bedtime as it is when we have to make a quick run out.\\n\\nSize & Fit\\n- Pants are Favorite Fit: Sits lower on the waist.\\n- Relaxed Fit: Our most generous fit sits farthest from the body.\\n\\nFabric & Care\\n- In the softest blend of 63% polyester, 35% rayon and 2% spandex.\\n\\nAdditional Features\\n- Relaxed fit top with raglan sleeves and rounded hem.\\n- Pull-on pants have a wide elastic waistband and drawstring, side pockets and a modern slim leg.\\n\\nImported.<<<<>>>>>: 73\\nname: Cozy Cuddles Knit Pullover Set\\ndescription: Perfect for lounging, this knit set lives up to its name. We used ultrasoft fabric and an easy design that's as comfortable at bedtime as it is when we have to make a quick run out. \\n\\nSize & Fit \\nPants are Favorite Fit: Sits lower on the waist. \\nRelaxed Fit: Our most generous fit sits farthest from the body. \\n\\nFabric & Care \\nIn the softest blend of 63% polyester, 35% rayon and 2% spandex.\\n\\nAdditional Features \\nRelaxed fit top with raglan sleeves and rounded hem. \\nPull-on pants have a wide elastic waistband and drawstring, side pockets and a modern slim leg. \\nImported.<<<<>>>>>: 854\\nname: Urban Striped Quarter-Zip Terry Pullover\\ndescription: Our all-cotton french terry quarter-zip is designed with appealing texture, a versatile season-spanning weight and nautical-inspired stripes. Size & Fit: Slim Fit - cut slim through the chest, sleeve and waist. Why We Love It: We've turned the typical sweatshirt inside out. Our pullover features the reverse side of our french terry fabric on the outside for outstanding texture, while the inside is nice and smooth. It has the undeniable comfort of cotton, with a neater look than most sweatshirts and a modern, slimmer fit. Our striped pullover is the perfect weekend layer, especially between seasons - and we love the side pockets! Fabric & Care: 100% cotton in a French terry knit. Machine wash and dry. Additional Features: Sturdy metal zipper adds to its rugged appeal. On-seam side pockets. Ribbed cuffs keep their shape. Side vents at hem. Imported.<<<<>>>>>: 830\\nname: Relaxed Fit Pull-On Pants\"\n",
      "}\n",
      "\u001b[32;1m\u001b[1;3m[llm/start]\u001b[0m \u001b[1m[chain:RetrievalQA > chain:StuffDocumentsChain > chain:LLMChain > llm:ChatNVIDIA] Entering LLM run with input:\n",
      "\u001b[0m{\n",
      "  \"prompts\": [\n",
      "    \"System: Use the following pieces of context to answer the user's question. \\nIf you don't know the answer, just say that you don't know, don't try to make up an answer.\\n----------------\\n: 10\\nname: Cozy Comfort Pullover Set, Stripe\\ndescription: Perfect for lounging, this striped knit set lives up to its name. We used ultrasoft fabric and an easy design that's as comfortable at bedtime as it is when we have to make a quick run out.\\n\\nSize & Fit\\n- Pants are Favorite Fit: Sits lower on the waist.\\n- Relaxed Fit: Our most generous fit sits farthest from the body.\\n\\nFabric & Care\\n- In the softest blend of 63% polyester, 35% rayon and 2% spandex.\\n\\nAdditional Features\\n- Relaxed fit top with raglan sleeves and rounded hem.\\n- Pull-on pants have a wide elastic waistband and drawstring, side pockets and a modern slim leg.\\n\\nImported.<<<<>>>>>: 73\\nname: Cozy Cuddles Knit Pullover Set\\ndescription: Perfect for lounging, this knit set lives up to its name. We used ultrasoft fabric and an easy design that's as comfortable at bedtime as it is when we have to make a quick run out. \\n\\nSize & Fit \\nPants are Favorite Fit: Sits lower on the waist. \\nRelaxed Fit: Our most generous fit sits farthest from the body. \\n\\nFabric & Care \\nIn the softest blend of 63% polyester, 35% rayon and 2% spandex.\\n\\nAdditional Features \\nRelaxed fit top with raglan sleeves and rounded hem. \\nPull-on pants have a wide elastic waistband and drawstring, side pockets and a modern slim leg. \\nImported.<<<<>>>>>: 854\\nname: Urban Striped Quarter-Zip Terry Pullover\\ndescription: Our all-cotton french terry quarter-zip is designed with appealing texture, a versatile season-spanning weight and nautical-inspired stripes. Size & Fit: Slim Fit - cut slim through the chest, sleeve and waist. Why We Love It: We've turned the typical sweatshirt inside out. Our pullover features the reverse side of our french terry fabric on the outside for outstanding texture, while the inside is nice and smooth. It has the undeniable comfort of cotton, with a neater look than most sweatshirts and a modern, slimmer fit. Our striped pullover is the perfect weekend layer, especially between seasons - and we love the side pockets! Fabric & Care: 100% cotton in a French terry knit. Machine wash and dry. Additional Features: Sturdy metal zipper adds to its rugged appeal. On-seam side pockets. Ribbed cuffs keep their shape. Side vents at hem. Imported.<<<<>>>>>: 830\\nname: Relaxed Fit Pull-On Pants\\nHuman: Do the Cozy Comfort Pullover Set    have side pockets?\"\n",
      "  ]\n",
      "}\n",
      "\u001b[36;1m\u001b[1;3m[llm/end]\u001b[0m \u001b[1m[chain:RetrievalQA > chain:StuffDocumentsChain > chain:LLMChain > llm:ChatNVIDIA] [681ms] Exiting LLM run with output:\n",
      "\u001b[0m{\n",
      "  \"generations\": [\n",
      "    [\n",
      "      {\n",
      "        \"text\": \" Yes, the Cozy Comfort Pullover Set, Stripe (named 10 and 73) has side pockets on the pull-on pants according to the provided descriptions.\",\n",
      "        \"generation_info\": null,\n",
      "        \"type\": \"ChatGeneration\",\n",
      "        \"message\": {\n",
      "          \"lc\": 1,\n",
      "          \"type\": \"constructor\",\n",
      "          \"id\": [\n",
      "            \"langchain\",\n",
      "            \"schema\",\n",
      "            \"messages\",\n",
      "            \"ChatMessage\"\n",
      "          ],\n",
      "          \"kwargs\": {\n",
      "            \"content\": \" Yes, the Cozy Comfort Pullover Set, Stripe (named 10 and 73) has side pockets on the pull-on pants according to the provided descriptions.\",\n",
      "            \"response_metadata\": {\n",
      "              \"role\": \"assistant\",\n",
      "              \"content\": \" Yes, the Cozy Comfort Pullover Set, Stripe (named 10 and 73) has side pockets on the pull-on pants according to the provided descriptions.\",\n",
      "              \"token_usage\": {\n",
      "                \"prompt_tokens\": 737,\n",
      "                \"total_tokens\": 776,\n",
      "                \"completion_tokens\": 39\n",
      "              },\n",
      "              \"model_name\": \"mistralai/mixtral-8x7b-instruct-v0.1\"\n",
      "            },\n",
      "            \"type\": \"chat\",\n",
      "            \"id\": \"run-6dd1b2e4-8ccc-4e2a-aa04-1cd5cc2afac7-0\",\n",
      "            \"role\": \"assistant\"\n",
      "          }\n",
      "        }\n",
      "      }\n",
      "    ]\n",
      "  ],\n",
      "  \"llm_output\": {\n",
      "    \"role\": \"assistant\",\n",
      "    \"content\": \" Yes, the Cozy Comfort Pullover Set, Stripe (named 10 and 73) has side pockets on the pull-on pants according to the provided descriptions.\",\n",
      "    \"token_usage\": {\n",
      "      \"prompt_tokens\": 737,\n",
      "      \"total_tokens\": 776,\n",
      "      \"completion_tokens\": 39\n",
      "    },\n",
      "    \"model_name\": \"mistralai/mixtral-8x7b-instruct-v0.1\"\n",
      "  },\n",
      "  \"run\": null\n",
      "}\n",
      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[chain:RetrievalQA > chain:StuffDocumentsChain > chain:LLMChain] [681ms] Exiting Chain run with output:\n",
      "\u001b[0m{\n",
      "  \"text\": \" Yes, the Cozy Comfort Pullover Set, Stripe (named 10 and 73) has side pockets on the pull-on pants according to the provided descriptions.\"\n",
      "}\n",
      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[chain:RetrievalQA > chain:StuffDocumentsChain] [683ms] Exiting Chain run with output:\n",
      "\u001b[0m{\n",
      "  \"output_text\": \" Yes, the Cozy Comfort Pullover Set, Stripe (named 10 and 73) has side pockets on the pull-on pants according to the provided descriptions.\"\n",
      "}\n",
      "\u001b[36;1m\u001b[1;3m[chain/end]\u001b[0m \u001b[1m[chain:RetrievalQA] [714ms] Exiting Chain run with output:\n",
      "\u001b[0m{\n",
      "  \"result\": \" Yes, the Cozy Comfort Pullover Set, Stripe (named 10 and 73) has side pockets on the pull-on pants according to the provided descriptions.\"\n",
      "}\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "' Yes, the Cozy Comfort Pullover Set, Stripe (named 10 and 73) has side pockets on the pull-on pants according to the provided descriptions.'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qa.run(examples[0][\"query\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "66b320db-3a59-49f3-a447-a29123777f3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Turn off the debug mode\n",
    "langchain.debug = False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a21aed1-ad9c-4735-a5ed-154a080497b0",
   "metadata": {},
   "source": [
    "### LLM assisted evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "74ef0f7f-fbbd-4c03-9359-93d1877c1c94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'query': \"What is the material of the Women's Campside Oxfords and what is its special feature?\",\n",
       " 'answer': \"The Women's Campside Oxfords are made of soft canvas material, which provides a broken-in feel and look. One special feature of these shoes is the Cleansport NXT® antimicrobial odor control in the innersole.\"}"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "examples[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5bb0baed-934c-44be-8ffc-f18520d4374b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new RetrievalQA chain...\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "predictions = qa.batch(examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "acd64667-db47-461c-9fd8-8ee1df29ac08",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'query': 'Do the Cozy Comfort Pullover Set    have side pockets?',\n",
       "  'answer': 'Yes',\n",
       "  'result': ' Yes, the Cozy Comfort Pullover Set, Stripe (named 10 and 73) has side pockets on the pull-on pants according to the provided descriptions.'},\n",
       " {'query': 'What collection is the Ultra-Lofty     850 Stretch Down Hooded Jacket from?',\n",
       "  'answer': 'The DownTek collection',\n",
       "  'result': ' The Ultra-Lofty 850 Stretch Down Hooded Jacket is from the DownTek collection. I know this because the name \"DownTek\" is mentioned in the description of the jacket, and it is a collection mentioned in the context provided.'},\n",
       " {'query': \"What is the material of the Women's Campside Oxfords and what is its special feature?\",\n",
       "  'answer': \"The Women's Campside Oxfords are made of soft canvas material, which provides a broken-in feel and look. One special feature of these shoes is the Cleansport NXT® antimicrobial odor control in the innersole.\",\n",
       "  'result': \" The Women's Campside Oxfords are made of a soft canvas material. A special feature of these Oxfords is the Cleansport NXT® antimicrobial odor control in the innersole.\"},\n",
       " {'query': 'What is the material and construction of the Recycled Waterhog Dog Mat, Chevron Weave?',\n",
       "  'answer': 'The Recycled Waterhog Dog Mat, Chevron Weave is made of 24 oz. polyester fabric which consists of 94% recycled materials. It has a rubber backing for added durability. The mat features thick and thin fibers that are designed for scraping dirt and absorbing water. It is quick drying, resistant to fading, rotting, mildew, and shedding, and can be used both indoors and outdoors. The chevron weave design is exclusive to this product.',\n",
       "  'result': ' The Recycled Waterhog Dog Mat, Chevron Weave is made of 24 oz. polyester fabric made from 94% recycled materials. It has a rubber backing and features thick and thin fibers for scraping dirt and absorbing water. The mat can be vacuumed or hosed clean and can be used indoors or outdoors. It is constructed in the USA.'},\n",
       " {'query': \"What are the key features of the Infant and Toddler Girls' Coastal Chill Swimsuit?\",\n",
       "  'answer': \"The Infant and Toddler Girls' Coastal Chill Swimsuit is a two-piece swimsuit with bright colors, ruffles, and exclusive whimsical prints. The fabric has four-way-stretch and is chlorine-resistant, which helps it keep its shape and resist snags. The UPF 50+ rated fabric provides the highest rated sun protection possible, blocking 98% of the sun's harmful rays. The crossover no-slip straps and fully lined bottom ensure a secure fit and maximum coverage. Machine washing and line drying are recommended for this imported item.\",\n",
       "  'result': \" The Infant and Toddler Girls' Coastal Chill Swimsuit has the following key features:\\n\\n1. It is a two-piece swimsuit that is designed for toddlers.\\n2. The swimsuit features bright colors, ruffles, and exclusive whimsical prints.\\n3. It is made of four-way-stretch and chlorine-resistant fabric that keeps its shape and resists snags.\\n4. The fabric is UPF 50+ rated, providing the highest rated sun protection possible and blocking 98% of the sun's harmful rays.\\n5. The no-slip straps and fully lined bottom ensure a secure fit and maximum coverage.\\n6. The swimsuit is machine washable and imported.\"},\n",
       " {'query': 'What is the composition of the body and lining of the Refresh Swimwear, V-Neck Tankini Contrasts and what sun protection rating does it have?',\n",
       "  'answer': 'The body of the Refresh Swimwear, V-Neck Tankini Contrasts is made of 82% recycled nylon and 18% Lycra® spandex, while the lining is made of 90% recycled nylon and 10% Lycra® spandex. This swimwear features UPF 50+ rating, the highest rated sun protection possible.',\n",
       "  'result': ' The body of the Refresh Swimwear, V-Neck Tankini Contrasts is made of 82% recycled nylon and 18% Lycra® spandex. The lining is made of 90% recycled nylon and 10% Lycra® spandex. This fabric is UPF 50+ rated, which is the highest rated sun protection possible.'},\n",
       " {'query': 'What technology is used in the EcoFlex 3L Storm Pants to increase breathability while maintaining waterproof protection?',\n",
       "  'answer': \"The EcoFlex 3L Storm Pants use TEK O2 technology to increase breathability while maintaining waterproof protection. This technology has been tested to offer the most breathability in the brand's line of products, making it suitable for various outdoor activities throughout the year.\",\n",
       "  'result': ' The EcoFlex 3L Storm Pants use TEK O2 technology to increase their breathability while maintaining waterproof protection. This exclusive technology is designed to instantly move moisture away, providing maximum protection against the elements during various outdoor activities throughout the year.'}]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c8599e05-d6c9-4bff-a4d8-748cc0f83ae5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain.evaluation.qa import QAEvalChain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c3222c1a-1899-4547-8e76-5ca9cf56b315",
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_chain = QAEvalChain.from_llm(llm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1d4544c3-ba65-4add-ab2d-84a8654bed64",
   "metadata": {},
   "outputs": [],
   "source": [
    "graded_outputs = eval_chain.evaluate(examples, predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7a426d0f-6ad2-4a84-98ea-db5c8943243a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'results': ' CORRECT\\n\\nThe student answer correctly states that the Cozy Comfort Pullover Set has side pockets, based on the provided descriptions. This matches the true answer.'},\n",
       " {'results': ' CORRECT\\n\\nThe student answer correctly identifies the Ultra-Lofty 850 Stretch Down Hooded Jacket as being from the DownTek collection. The fact that the student included additional information about how they determined this does not affect the accuracy of the answer.'},\n",
       " {'results': ' CORRECT'},\n",
       " {'results': ' GRADE: CORRECT\\n\\nThe student answer includes all the necessary and accurate information as stated in the true answer. The student answer expands on the true answer by mentioning that the mat can be vacuumed or hosed clean, and that it is constructed in the USA – both of which are correct and do not conflict with the true answer. Therefore, the student answer is factually accurate and can be considered correct.'},\n",
       " {'results': \" CORRECT\\n\\nThe student answer correctly lists all the key features of the Infant and Toddler Girls' Coastal Chill Swimsuit. While the true answer phrases some features differently, both answers contain the same information, so the student answer is correct.\"},\n",
       " {'results': ' CORRECT\\n\\nThe student answer correctly identifies the composition of the body and lining of the Refresh Swimwear, V-Neck Tankini Contrasts and the sun protection rating it has, which is UPF 50+. Although the student answer presents the information in a slightly different way than the true answer, the information provided is factually accurate and matches the true answer.'},\n",
       " {'results': \" CORRECT\\n\\nThe student answer correctly identifies the EcoFlex 3L Storm Pants' use of TEK O2 technology for increased breathability while maintaining waterproof protection. While the student answer adds some additional details, it does not contain any conflicting statements. The slight differences in phrasing do not affect the factual accuracy of the student answer.\"}]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "graded_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "1ddc0ca2-07a2-4f71-b080-044d3640c990",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________\n",
      "Example 0:\n",
      "Question: Do the Cozy Comfort Pullover Set    have side pockets?\n",
      "Real Answer: Yes\n",
      "Predicted Answer:  Yes, the Cozy Comfort Pullover Set, Stripe (named 10 and 73) has side pockets on the pull-on pants according to the provided descriptions.\n",
      "Predicted Grade:  CORRECT\n",
      "\n",
      "The student answer correctly states that the Cozy Comfort Pullover Set has side pockets, based on the provided descriptions. This matches the true answer.\n",
      "__________________________________________________\n",
      "__________________________________________________\n",
      "Example 1:\n",
      "Question: What collection is the Ultra-Lofty     850 Stretch Down Hooded Jacket from?\n",
      "Real Answer: The DownTek collection\n",
      "Predicted Answer:  The Ultra-Lofty 850 Stretch Down Hooded Jacket is from the DownTek collection. I know this because the name \"DownTek\" is mentioned in the description of the jacket, and it is a collection mentioned in the context provided.\n",
      "Predicted Grade:  CORRECT\n",
      "\n",
      "The student answer correctly identifies the Ultra-Lofty 850 Stretch Down Hooded Jacket as being from the DownTek collection. The fact that the student included additional information about how they determined this does not affect the accuracy of the answer.\n",
      "__________________________________________________\n",
      "__________________________________________________\n",
      "Example 2:\n",
      "Question: What is the material of the Women's Campside Oxfords and what is its special feature?\n",
      "Real Answer: The Women's Campside Oxfords are made of soft canvas material, which provides a broken-in feel and look. One special feature of these shoes is the Cleansport NXT® antimicrobial odor control in the innersole.\n",
      "Predicted Answer:  The Women's Campside Oxfords are made of a soft canvas material. A special feature of these Oxfords is the Cleansport NXT® antimicrobial odor control in the innersole.\n",
      "Predicted Grade:  CORRECT\n",
      "__________________________________________________\n",
      "__________________________________________________\n",
      "Example 3:\n",
      "Question: What is the material and construction of the Recycled Waterhog Dog Mat, Chevron Weave?\n",
      "Real Answer: The Recycled Waterhog Dog Mat, Chevron Weave is made of 24 oz. polyester fabric which consists of 94% recycled materials. It has a rubber backing for added durability. The mat features thick and thin fibers that are designed for scraping dirt and absorbing water. It is quick drying, resistant to fading, rotting, mildew, and shedding, and can be used both indoors and outdoors. The chevron weave design is exclusive to this product.\n",
      "Predicted Answer:  The Recycled Waterhog Dog Mat, Chevron Weave is made of 24 oz. polyester fabric made from 94% recycled materials. It has a rubber backing and features thick and thin fibers for scraping dirt and absorbing water. The mat can be vacuumed or hosed clean and can be used indoors or outdoors. It is constructed in the USA.\n",
      "Predicted Grade:  GRADE: CORRECT\n",
      "\n",
      "The student answer includes all the necessary and accurate information as stated in the true answer. The student answer expands on the true answer by mentioning that the mat can be vacuumed or hosed clean, and that it is constructed in the USA – both of which are correct and do not conflict with the true answer. Therefore, the student answer is factually accurate and can be considered correct.\n",
      "__________________________________________________\n",
      "__________________________________________________\n",
      "Example 4:\n",
      "Question: What are the key features of the Infant and Toddler Girls' Coastal Chill Swimsuit?\n",
      "Real Answer: The Infant and Toddler Girls' Coastal Chill Swimsuit is a two-piece swimsuit with bright colors, ruffles, and exclusive whimsical prints. The fabric has four-way-stretch and is chlorine-resistant, which helps it keep its shape and resist snags. The UPF 50+ rated fabric provides the highest rated sun protection possible, blocking 98% of the sun's harmful rays. The crossover no-slip straps and fully lined bottom ensure a secure fit and maximum coverage. Machine washing and line drying are recommended for this imported item.\n",
      "Predicted Answer:  The Infant and Toddler Girls' Coastal Chill Swimsuit has the following key features:\n",
      "\n",
      "1. It is a two-piece swimsuit that is designed for toddlers.\n",
      "2. The swimsuit features bright colors, ruffles, and exclusive whimsical prints.\n",
      "3. It is made of four-way-stretch and chlorine-resistant fabric that keeps its shape and resists snags.\n",
      "4. The fabric is UPF 50+ rated, providing the highest rated sun protection possible and blocking 98% of the sun's harmful rays.\n",
      "5. The no-slip straps and fully lined bottom ensure a secure fit and maximum coverage.\n",
      "6. The swimsuit is machine washable and imported.\n",
      "Predicted Grade:  CORRECT\n",
      "\n",
      "The student answer correctly lists all the key features of the Infant and Toddler Girls' Coastal Chill Swimsuit. While the true answer phrases some features differently, both answers contain the same information, so the student answer is correct.\n",
      "__________________________________________________\n",
      "__________________________________________________\n",
      "Example 5:\n",
      "Question: What is the composition of the body and lining of the Refresh Swimwear, V-Neck Tankini Contrasts and what sun protection rating does it have?\n",
      "Real Answer: The body of the Refresh Swimwear, V-Neck Tankini Contrasts is made of 82% recycled nylon and 18% Lycra® spandex, while the lining is made of 90% recycled nylon and 10% Lycra® spandex. This swimwear features UPF 50+ rating, the highest rated sun protection possible.\n",
      "Predicted Answer:  The body of the Refresh Swimwear, V-Neck Tankini Contrasts is made of 82% recycled nylon and 18% Lycra® spandex. The lining is made of 90% recycled nylon and 10% Lycra® spandex. This fabric is UPF 50+ rated, which is the highest rated sun protection possible.\n",
      "Predicted Grade:  CORRECT\n",
      "\n",
      "The student answer correctly identifies the composition of the body and lining of the Refresh Swimwear, V-Neck Tankini Contrasts and the sun protection rating it has, which is UPF 50+. Although the student answer presents the information in a slightly different way than the true answer, the information provided is factually accurate and matches the true answer.\n",
      "__________________________________________________\n",
      "__________________________________________________\n",
      "Example 6:\n",
      "Question: What technology is used in the EcoFlex 3L Storm Pants to increase breathability while maintaining waterproof protection?\n",
      "Real Answer: The EcoFlex 3L Storm Pants use TEK O2 technology to increase breathability while maintaining waterproof protection. This technology has been tested to offer the most breathability in the brand's line of products, making it suitable for various outdoor activities throughout the year.\n",
      "Predicted Answer:  The EcoFlex 3L Storm Pants use TEK O2 technology to increase their breathability while maintaining waterproof protection. This exclusive technology is designed to instantly move moisture away, providing maximum protection against the elements during various outdoor activities throughout the year.\n",
      "Predicted Grade:  CORRECT\n",
      "\n",
      "The student answer correctly identifies the EcoFlex 3L Storm Pants' use of TEK O2 technology for increased breathability while maintaining waterproof protection. While the student answer adds some additional details, it does not contain any conflicting statements. The slight differences in phrasing do not affect the factual accuracy of the student answer.\n",
      "__________________________________________________\n"
     ]
    }
   ],
   "source": [
    "for i, eg in enumerate(examples):\n",
    "    print(\"_\"*50)\n",
    "    print(f\"Example {i}:\")\n",
    "    print(\"Question: \" + predictions[i]['query'])\n",
    "    print(\"Real Answer: \" + predictions[i]['answer'])\n",
    "    print(\"Predicted Answer: \" + predictions[i]['result'])\n",
    "    print(\"Predicted Grade: \" + graded_outputs[i]['results'])\n",
    "    print(\"_\"*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ef579e0-ef48-42a9-a6a6-f4a19b10db0d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}