llm-evaluator-monitoring.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import openai\n",
    "import os\n",
    "import IPython\n",
    "import json\n",
    "import pandas as pd\n",
    "import comet_llm\n",
    "from dotenv import load_dotenv\n",
    "from langchain.llms import OpenAI\n",
    "from langchain.chat_models import ChatOpenAI\n",
    "from langchain import PromptTemplate, LLMChain\n",
    "from langchain.prompts.chat import (\n",
    "    ChatPromptTemplate,\n",
    "    SystemMessagePromptTemplate,\n",
    "    AIMessagePromptTemplate,\n",
    "    HumanMessagePromptTemplate,\n",
    ")\n",
    "from langchain.schema import (\n",
    "    AIMessage,\n",
    "    HumanMessage,\n",
    "    SystemMessage\n",
    ")\n",
    "\n",
    "from langchain.evaluation.qa import QAEvalChain\n",
    "\n",
    "from langchain.prompts import (\n",
    "    FewShotChatMessagePromptTemplate,\n",
    "    ChatPromptTemplate,\n",
    ")\n",
    "\n",
    "from langchain.prompts.few_shot import FewShotPromptTemplate\n",
    "from langchain.prompts.prompt import PromptTemplate\n",
    "\n",
    "load_dotenv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Comet keys\n",
    "COMET_API_KEY = os.getenv(\"COMET_API_KEY\")\n",
    "COMET_WORKSPACE = os.getenv(\"COMET_WORKSPACE\")\n",
    "COMET_PROJECT_NAME = os.getenv(\"COMET_PROJECT_NAME\")\n",
    "\n",
    "# set the API key\n",
    "os.environ[\"OPENAI_API_KEY\"] = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "# create LLM instance\n",
    "chat = ChatOpenAI(temperature=0, model_name=\"gpt-3.5-turbo\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load json data at path: data/article-tags.json into a dataframe\n",
    "with open('data/article-tags.json') as f:\n",
    "    val_data = json.load(f)\n",
    "\n",
    "with open('data/few_shot.json') as f:\n",
    "    few_shot_data = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a custom selector for the few shot data\n",
    "from langchain.prompts.example_selector.base import BaseExampleSelector\n",
    "from typing import Dict, List\n",
    "import numpy as np\n",
    "\n",
    "class CustomExampleSelector(BaseExampleSelector):\n",
    "    \n",
    "    def __init__(self, examples: List[Dict[str, str]]):\n",
    "        self.examples = examples\n",
    "    \n",
    "    def add_example(self, example: Dict[str, str]) -> None:\n",
    "        \"\"\"Add new example to store for a key.\"\"\"\n",
    "        self.examples.append(example)\n",
    "\n",
    "    def select_examples(self, size) -> List[dict]:\n",
    "        \"\"\"Select which examples to use based on the inputs.\"\"\"\n",
    "        return np.random.choice(self.examples, size=size, replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# few-shot selector\n",
    "example_selector = CustomExampleSelector(few_shot_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# few-shot template\n",
    "template = \"\"\"\n",
    "Abstract: {abstract}\n",
    "Tags: {tags}\n",
    "\"\"\"\n",
    "\n",
    "human_template = ChatPromptTemplate.from_messages(\n",
    "    [\n",
    "        (\"human\", \"{abstract}\"),\n",
    "        (\"ai\", \"{tags}\")\n",
    "    ]\n",
    ")\n",
    "\n",
    "few_shot_prompt = FewShotChatMessagePromptTemplate(\n",
    "    examples = list(example_selector.select_examples(3)), # rerun code to generate different examples\n",
    "    example_prompt=human_template,\n",
    ")\n",
    "\n",
    "final_few_shot_prompt = ChatPromptTemplate.from_messages(\n",
    "    [\n",
    "        (\"system\", \"Your task is to extract model names from machine learning paper abstracts. Your response is an an array of the model names in the format [\\\"model_name\\\"]. If you don't find model names in the abstract or you are not sure, return [\\\"NA\\\"]\"),\n",
    "        few_shot_prompt,\n",
    "        (\"human\", \"{abstract}\"),\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# zero-shot prompt\n",
    "zero_shot_template = \"\"\"\n",
    "Your task is extract model names from machine learning paper abstracts. Your response is an an array of the model names in the format [\\\"model_name\\\"]. If you don't find model names in the abstract or you are not sure, return [\\\"NA\\\"]\n",
    "\n",
    "Abstract: {abstract}\n",
    "Tags:\n",
    "\"\"\"\n",
    "\n",
    "message_prompt = HumanMessagePromptTemplate.from_template(zero_shot_template)\n",
    "\n",
    "final_zero_shot_prompt = ChatPromptTemplate.from_messages([message_prompt])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prompt Evaluation with QAEval Chain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#zero_chain = LLMChain(llm=llm, prompt=zero_shot_prompt)\n",
    "few_shot_chain = LLMChain(llm=chat, prompt=final_few_shot_prompt)\n",
    "zero_shot_chain = LLMChain(llm=chat, prompt=final_zero_shot_prompt)\n",
    "\n",
    "# run few-shot predictions\n",
    "fwpredictions = few_shot_chain.apply(val_data)\n",
    "\n",
    "# run zeroshot predictions\n",
    "zeroshot_predictions = zero_shot_chain.apply(val_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"['LLM', 'Evol-Instruct', 'LLaMA', 'WizardLM', 'OpenAI ChatGPT']\""
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fwpredictions[0]['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# QA Evaluation Chain\n",
    "llm_choice = \"text-davinci-003\"\n",
    "\n",
    "if llm_choice == \"text-davinci-003\":\n",
    "    ev_llm  = OpenAI(model_name=llm_choice)\n",
    "    eval_chain = QAEvalChain.from_llm(ev_llm)\n",
    "else:\n",
    "    eval_chain = QAEvalChain.from_llm(chat)\n",
    "\n",
    "# zero-shot\n",
    "zero_graded_outputs = eval_chain.evaluate(val_data, zeroshot_predictions, question_key=\"abstract\", prediction_key=\"text\", answer_key=\"tags\")\n",
    "\n",
    "# few-shot\n",
    "fw_graded_outputs = eval_chain.evaluate(val_data, fwpredictions, question_key=\"abstract\", prediction_key=\"text\", answer_key=\"tags\")\n",
    "\n",
    "# model name\n",
    "model_name = few_shot_chain.to_json()['kwargs']['llm'].model_name\n",
    "\n",
    "# few-shot prompts\n",
    "for i, eg in enumerate(val_data):\n",
    "    comet_llm.log_prompt(\n",
    "        prompt = final_few_shot_prompt.format(abstract=val_data[i]['abstract']) + \"\\nAI: \",\n",
    "        tags = [llm_choice, str(fw_graded_outputs[i]['results']), \"few-shot\" ],\n",
    "        metadata = {\n",
    "            \"expected_response\": str(val_data[i]['tags']),\n",
    "            \"llm_evaluator_result\": str(fw_graded_outputs[i]['results']),\n",
    "            \"llm_evaluator\": llm_choice,\n",
    "            \"response_llm\": model_name,\n",
    "        },\n",
    "        output = fwpredictions[i]['text'],\n",
    "        api_key = COMET_API_KEY,\n",
    "        workspace = COMET_WORKSPACE,\n",
    "        project = COMET_PROJECT_NAME,\n",
    "    )\n",
    "\n",
    "# do the same for zero-shot predictions\n",
    "for i, eg in enumerate(val_data):\n",
    "    comet_llm.log_prompt(\n",
    "        prompt = final_zero_shot_prompt.format(abstract=val_data[i]['abstract']) + \"\\nAI: \",\n",
    "        tags = [llm_choice, str(zero_graded_outputs[i]['results']), \"zero-shot\" ],\n",
    "        metadata = {\n",
    "            \"expected_response\": str(val_data[i]['tags']),\n",
    "            \"llm_evaluator_result\": str(zero_graded_outputs[i]['results']),\n",
    "            \"llm_evaluator\": llm_choice,\n",
    "            \"response_llm\": model_name,\n",
    "        },\n",
    "        output = zeroshot_predictions[i]['text'],\n",
    "        api_key = COMET_API_KEY,\n",
    "        workspace = COMET_WORKSPACE,\n",
    "        project = COMET_PROJECT_NAME,\n",
    "    )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "comet",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.17"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}