eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from core.agent import agent_executor, agent_llm\n",
    "from core.CustomTools import fetch_and_rerank\n",
    "\n",
    "from langchain.agents.openai_functions_agent.agent_token_buffer_memory import (\n",
    "    AgentTokenBufferMemory,\n",
    ")\n",
    "\n",
    "\n",
    "from datasets import Dataset\n",
    "\n",
    "from ragas import evaluate\n",
    "from ragas.metrics import (\n",
    "    faithfulness,\n",
    "    answer_relevancy,\n",
    "    context_recall,\n",
    "    context_precision,\n",
    ")\n",
    "\n",
    "import regex as re\n",
    "import glob\n",
    "\n",
    "from core.ragas import RagasCallbackHandler\n",
    "\n",
    "# ---\n",
    "\n",
    "#agent_memory = AgentTokenBufferMemory(llm=agent_llm)\n",
    "callback_handler = RagasCallbackHandler()\n",
    "def query_agent(prompt):\n",
    "   response = agent_executor(\n",
    "            {\"input\": prompt, \"history\": []},                        \n",
    "            callbacks=[callback_handler],\n",
    "            include_run_info=True,\n",
    "        )\n",
    "   return response\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Questions: 6\n",
      "Grounded Truths: 6\n"
     ]
    }
   ],
   "source": [
    "# source data\n",
    "\n",
    "filenames = []\n",
    "for _file in glob.glob(\"ragas_data/*.txt\"):\n",
    "    filenames.append(_file)\n",
    "\n",
    "questions = []\n",
    "ground_truths = []\n",
    "\n",
    "DELIM = \"__###__\"\n",
    "for name in filenames:\n",
    "    file_content = None\n",
    "    with open(name) as f:                \n",
    "        file_content = f.read()\n",
    "\n",
    "    content = file_content.split(DELIM)\n",
    "    questions.append(content[0])\n",
    "    truths = []\n",
    "    truths.append(content[1])\n",
    "    ground_truths.append(truths)\n",
    "\n",
    "print(\"Questions:\", len(questions))\n",
    "print(\"Grounded Truths:\", len(ground_truths))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "starting tool ...  global error handler RouteBuilder\n",
      "Embedding ms:  0.34453701972961426\n",
      "0.802 :  _manual_error-handler.html.txt_1\n",
      "0.786 :  _next_lpr-component.html.txt_3\n",
      "0.799 :  _eips_message-history.html.txt_2\n",
      "0.8 :  _manual_exception-clause.html.txt_12\n",
      "0.801 :  _manual_error-handler.html.txt_2\n",
      "ending tool ... \n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "Input:  1\n",
      "Outputs:  1\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "starting tool ...  jBang\n",
      "ending tool ... \n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "Input:  1\n",
      "Outputs:  1\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "starting tool ...  Rest DSL\n",
      "Embedding ms:  0.3432190418243408\n",
      "0.825 :  _manual_rest-dsl.html.txt_2\n",
      "0.819 :  _manual_rest-dsl.html.txt_1\n",
      "0.811 :  _manual_rest-dsl.html.txt_0\n",
      "0.81 :  _manual_rest-dsl.html.txt_7\n",
      "0.805 :  _next_rest-component.html.txt_0\n",
      "ending tool ... \n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "Input:  1\n",
      "Outputs:  1\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "starting tool ...  file idempotent\n",
      "Embedding ms:  0.6420547962188721\n",
      "0.812 :  _next_file-component.html.txt_26\n",
      "0.805 :  _next_file-component.html.txt_25\n",
      "0.794 :  _next_azure-files-component.html.txt_11\n",
      "0.788 :  _next_file-component.html.txt_14\n",
      "0.781 :  _eips_competing-consumers.html.txt_2\n",
      "ending tool ... \n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "Input:  1\n",
      "Outputs:  1\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "starting tool ...  event aggregation in Apache Camel\n",
      "Embedding ms:  0.3488612174987793\n",
      "0.831 :  _faq_how-do-the-direct-event-seda-and-vm-endpoints-compare.html.txt\n",
      "0.829 :  _others_cloudevents.html.txt\n",
      "0.82 :  _eips_aggregate-eip.html.txt_10\n",
      "0.829 :  _next_timer-component.html.txt_3\n",
      "0.83 :  _eips_split-eip.html.txt_10\n",
      "ending tool ... \n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "Input:  1\n",
      "Outputs:  1\n",
      "\n",
      "\n",
      "\u001b[1m> Entering new AgentExecutor chain...\u001b[0m\n",
      "starting tool ...  file reading\n",
      "Embedding ms:  0.35344600677490234\n",
      "0.777 :  _next_file-component.html.txt_22\n",
      "0.769 :  _dataformats_univocityFixed-dataformat.html.txt_0\n",
      "0.739 :  _support_.txt\n",
      "0.773 :  _next_sftp-component.html.txt_13\n",
      "0.771 :  _next_ftps-component.html.txt_2\n",
      "ending tool ... \n",
      "starting tool ...  AWS S3\n",
      "Embedding ms:  0.3646571636199951\n",
      "0.86 :  _next_aws2-s3-component.html.txt_11\n",
      "0.849 :  _next_aws2-s3-component.html.txt_1\n",
      "0.842 :  _next_aws2-s3-component.html.txt_14\n",
      "0.84 :  _next_aws2-s3-component.html.txt_3\n",
      "0.831 :  _next_aws-summary.html.txt\n",
      "ending tool ... \n",
      "\n",
      "\u001b[1m> Finished chain.\u001b[0m\n",
      "Input:  2\n",
      "Outputs:  2\n",
      "Answer len: 6\n",
      "Context len: 6\n"
     ]
    }
   ],
   "source": [
    "answers = []\n",
    "contexts = []\n",
    "\n",
    "# Inference\n",
    "for query in questions:\n",
    "  \n",
    "  # retrieve an answer\n",
    "  response = query_agent(query)\n",
    "  answers.append(response[\"output\"])\n",
    "\n",
    "  inputs, outputs = callback_handler.clear_collected_outputs()\n",
    "  print(\"Input: \", len(inputs))\n",
    "  print(\"Outputs: \", len(outputs))\n",
    "\n",
    "  results = []  \n",
    "  [results.append(doc) for doc in outputs]          \n",
    "  contexts.append(results)  \n",
    "\n",
    "print(\"Answer len:\", len(contexts))\n",
    "print(\"Context len:\", len(contexts))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluating with [context_precision]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:06<00:00,  6.74s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluating with [context_recall]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [03:43<00:00, 223.19s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluating with [faithfulness]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:58<00:00, 58.98s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "evaluating with [answer_relevancy]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:07<00:00,  7.72s/it]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>question</th>\n",
       "      <th>answer</th>\n",
       "      <th>contexts</th>\n",
       "      <th>ground_truths</th>\n",
       "      <th>context_precision</th>\n",
       "      <th>context_recall</th>\n",
       "      <th>faithfulness</th>\n",
       "      <th>answer_relevancy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>how you can I register a global error handler ...</td>\n",
       "      <td>To register a global error handler for the Rou...</td>\n",
       "      <td>[page_content='\"errorHandler(noErrorHandler())...</td>\n",
       "      <td>[\\nTo register a global error handler for the ...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.250</td>\n",
       "      <td>0.940842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>How can I test a Camel route using jBang?\\n</td>\n",
       "      <td>To test a Camel route using jBang, you can use...</td>\n",
       "      <td>[[Document(page_content='\"CHAPTER 4. USING CAM...</td>\n",
       "      <td>[\\nTo test a Camel route using jBang, you can ...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.989555</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Can you show an example how to use the Rest DS...</td>\n",
       "      <td>To use the Rest DSL in Java to build an endpoi...</td>\n",
       "      <td>[page_content='\"Inline Rest DSL as a single ro...</td>\n",
       "      <td>[\\nTo create a REST endpoint that responds to ...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.700</td>\n",
       "      <td>0.813892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>This is my route\\n\\n    from(\"sftp://userName:...</td>\n",
       "      <td>To avoid processing the same files in case of ...</td>\n",
       "      <td>[page_content='\"Using a file based idempotent ...</td>\n",
       "      <td>[\\nTo avoid processing the same files in case ...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.875</td>\n",
       "      <td>0.763041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I am having an issue while working on the requ...</td>\n",
       "      <td>To listen to events generated by one route and...</td>\n",
       "      <td>[page_content='\"Article source: https://rhaeto...</td>\n",
       "      <td>[\\nTo listen to the events of one route by ano...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.791901</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            question  \\\n",
       "0  how you can I register a global error handler ...   \n",
       "1        How can I test a Camel route using jBang?\\n   \n",
       "2  Can you show an example how to use the Rest DS...   \n",
       "3  This is my route\\n\\n    from(\"sftp://userName:...   \n",
       "4  I am having an issue while working on the requ...   \n",
       "\n",
       "                                              answer  \\\n",
       "0  To register a global error handler for the Rou...   \n",
       "1  To test a Camel route using jBang, you can use...   \n",
       "2  To use the Rest DSL in Java to build an endpoi...   \n",
       "3  To avoid processing the same files in case of ...   \n",
       "4  To listen to events generated by one route and...   \n",
       "\n",
       "                                            contexts  \\\n",
       "0  [page_content='\"errorHandler(noErrorHandler())...   \n",
       "1  [[Document(page_content='\"CHAPTER 4. USING CAM...   \n",
       "2  [page_content='\"Inline Rest DSL as a single ro...   \n",
       "3  [page_content='\"Using a file based idempotent ...   \n",
       "4  [page_content='\"Article source: https://rhaeto...   \n",
       "\n",
       "                                       ground_truths  context_precision  \\\n",
       "0  [\\nTo register a global error handler for the ...                1.0   \n",
       "1  [\\nTo test a Camel route using jBang, you can ...                0.0   \n",
       "2  [\\nTo create a REST endpoint that responds to ...                0.0   \n",
       "3  [\\nTo avoid processing the same files in case ...                0.0   \n",
       "4  [\\nTo listen to the events of one route by ano...                1.0   \n",
       "\n",
       "   context_recall  faithfulness  answer_relevancy  \n",
       "0             1.0         0.250          0.940842  \n",
       "1             1.0         1.000          0.989555  \n",
       "2             1.0         0.700          0.813892  \n",
       "3             1.0         0.875          0.763041  \n",
       "4             1.0         1.000          0.791901  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# To dict\n",
    "data = {\n",
    "    \"question\": questions,\n",
    "    \"answer\": answers,\n",
    "    \"contexts\": contexts,\n",
    "    \"ground_truths\": ground_truths\n",
    "}\n",
    "\n",
    "# Convert dict to dataset\n",
    "dataset = Dataset.from_dict(data)\n",
    "\n",
    "result = evaluate(\n",
    "    dataset = dataset, \n",
    "    metrics=[\n",
    "        context_precision,\n",
    "        context_recall,\n",
    "        faithfulness,\n",
    "        answer_relevancy,\n",
    "    ],\n",
    ")\n",
    "\n",
    "df = result.to_pandas()\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}