metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import json\n",
    "import random \n",
    "import time\n",
    "\n",
    "from datetime import datetime\n",
    "from typing import Literal\n",
    "\n",
    "from bs4 import BeautifulSoup\n",
    "import unicodedata\n",
    "import numpy as np\n",
    "import pandas as pd \n",
    "\n",
    "from langchain_community.vectorstores import FAISS\n",
    "from langchain_huggingface import HuggingFaceEmbeddings\n",
    "from langchain_community.document_loaders import RecursiveUrlLoader\n",
    "from langchain_core.tools import tool\n",
    "from langchain_core.messages import HumanMessage, SystemMessage\n",
    "from langchain_mistralai import ChatMistralAI\n",
    "from langgraph.graph import END, StateGraph, MessagesState, StateGraph\n",
    "from langgraph.prebuilt import ToolNode\n",
    "from langgraph.checkpoint.memory import MemorySaver\n",
    "\n",
    "from mistralai import Mistral\n",
    "\n",
    "from apps.rag_system.rag_pipeline import FinanceMultiAgentRAG\n",
    "\n",
    "from dotenv import load_dotenv\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "\n",
    "load_dotenv('./.env')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Проверяем RAG пайплайн по методу `Response vs input` \n",
    "\n",
    "### P.S. В качестве судьи будет `ministral-8b-latest`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "api_key = os.environ.get(\"MISTRAL_KEY\")\n",
    "\n",
    "FAISS_DB = \"D:\\programming\\ITMO\\IntroLLM\\FinanceInsight\\dataset\\index_finance_insight\"\n",
    "\n",
    "EMBEDDING_MODEL_NAME = 'intfloat/multilingual-e5-small'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_model = HuggingFaceEmbeddings(\n",
    "    model_name = EMBEDDING_MODEL_NAME,\n",
    "    multi_process = True,\n",
    "    encode_kwargs = {\"normalize_embeddings\": True}\n",
    ")\n",
    "\n",
    "vector_store = FAISS.load_local(\n",
    "    FAISS_DB, \n",
    "    embedding_model, \n",
    "    allow_dangerous_deserialization = True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = FinanceMultiAgentRAG(\n",
    "    api_key = api_key\n",
    ")\n",
    "\n",
    "judge = Mistral(\n",
    "    api_key = api_key\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'0': \"Какие моральные дилеммы сталкиваются герои в фильме 'Сделка'?\",\n",
       " '1': 'Какие операции по карте считаются бесплатными, если оплачиваете покупку кредиткой?',\n",
       " '2': 'What strategies could a monopolist employ to maintain its market dominance in the face of a new competitor charging 10% less?',\n",
       " '3': 'Какое наказание предусмотрено за искусственное создание доказательств обвинения?',\n",
       " '4': 'What are some examples of how market forces have been insufficient to eliminate discrimination against specific groups, despite economic incentives?',\n",
       " '5': 'Какое наказание предусмотрено за распространение ложной информации о угрозах безопасности граждан?',\n",
       " '6': 'Какие документы необходимо предоставить для постановки на учет по месту пребывания лиц без гражданства, не осуществляющих трудовую или иную не запрещенную законодательством Российской Федерации деятельность?',\n",
       " '7': \"What is the formula for estimating the value of a firm's stock using the price/earnings ratio?\",\n",
       " '8': 'What specific disclosures should the advertisement include to comply with the regulations?',\n",
       " '9': 'What are the conditions under which derivative contracts are considered legal and valid according to the given context?',\n",
       " '10': 'What is the required rate of return (discount rate) for the stock in Figure 11.6?',\n",
       " '11': 'Какой тип опциона (колл или пут) будет в деньгах, если текущая цена актива выше страйка?',\n",
       " '12': 'What specific conditions might SEBI impose when recognizing certain entities as distinct and separate under the validity of agreements or treaties?',\n",
       " '13': 'What are the conditions under which a promoter or promoter group can make an institutional placement programme?',\n",
       " '14': 'Какие действия могут быть квалифицированы как угроза или насильственные действия в связи с осуществлением правосудия или производством предварительного расследования?',\n",
       " '15': \"What is the impact of a 1% increase in interest rates on the value of a bond with a medium maturity, according to Vanguard's data?\",\n",
       " '16': 'Какие активы из предложенного списка наиболее подходят для пассивной части портфеля, учитывая текущие риски и ограничения?',\n",
       " '17': 'What are the most common benchmark indices used in benchmarked instruments?',\n",
       " '18': 'Какие ситуации могут быть достаточными основаниями для продажи акций из вашего портфеля, несмотря на долгосрочную стратегию?',\n",
       " '19': 'Какое наказание предусмотрено за повторное совершение данного деяния?',\n",
       " '20': 'What are the conditions specified by SEBI for the reporting of trades on recognized stock exchanges or other platforms?',\n",
       " '21': 'What is the marginal utility of the fifth movie José attends?',\n",
       " '22': 'What are the two basic types of nontransaction deposits?',\n",
       " '23': 'Какие виды государственных облигаций позволяют защититься от инфляции?',\n",
       " '24': 'What is the difference between compound growth rates and compound interest rates?',\n",
       " '25': 'What is the relationship between the points on the indifference curves Uh and Um, and how does it relate to the concept of utility?',\n",
       " '26': \"What is the price elasticity of demand for the band's tickets, and how does it affect the band's decision on ticket pricing?\",\n",
       " '27': 'Какие конкретные натуральные показатели наиболее часто используются в прогнозировании отраслевыми аналитиками для промышленных секторов?',\n",
       " '28': 'What are the main arguments against dumping in the context of trade protectionism?',\n",
       " '29': 'What is the primary difference between the income statement and the classified balance sheet?',\n",
       " '30': 'Какие преимущества имеет использование дебетовой карты или накопительного счета для накопления процентов?',\n",
       " '31': 'What is the inflation rate for the period from year 2 to year 3 based on the given price indices?',\n",
       " '32': \"How does a shift in perceived demand affect the firm's pricing strategy in a monopolistically competitive industry?\",\n",
       " '33': 'Какие специфические мультипликаторы наиболее часто используются для оценки компаний в сегменте Upstream нефтегазовой отрасли?',\n",
       " '34': 'What is the minimum period after which a valuer can be reappointed by an InvIT?',\n",
       " '35': 'What are the potential long-term effects of protectionist policies on the German and Czech economies, specifically in terms of job creation and economic growth?',\n",
       " '36': 'What measures can be taken to mitigate the increased moral hazard incentives for large financial institutions to take on greater risk?',\n",
       " '37': 'What are some ways to increase income to afford more of the things we want?',\n",
       " '38': 'What are the conditions under which SEBI can agree to a settlement proposal for defaults?',\n",
       " '39': 'Какие нефинансовые факторы наиболее важны для понимания истории развития компании?',\n",
       " '40': 'Какие действия могут привести к наказанию за незаконное перемещение носителей с государственной тайной за пределы России, если они не предусмотрены статьями 189, 226.1, 275, 276 и 283.1 Уголовного кодекса?',\n",
       " '41': 'Какое максимальное наказание предусмотрено за данное преступление?',\n",
       " '42': 'What is the difference between a listed and an unlisted depository receipt?',\n",
       " '43': 'How can government policies influence the supply of nurses in the healthcare sector?',\n",
       " '44': 'What are some common motives for day traders, and how do they differ from those of long-term investors?',\n",
       " '45': 'Какое наказание предусмотрено за изготовление и сбыт поддельных акцизных марок, специальных марок или знаков соответствия, защищенных от подделок?',\n",
       " '46': 'Какое наказание предусмотрено за деяния, предусмотренные частью первой статьи, совершенные в значительном размере?',\n",
       " '47': 'Какие риски связаны с инвестициями в корпоративные облигации по сравнению с государственными облигациями?',\n",
       " '48': 'Какой размер позиции следует использовать, чтобы минимизировать риски при высокой волатильности?',\n",
       " '49': 'Какие изменения в законодательстве были внесены Федеральным законом от 08.12.2003 N 162-ФЗ?',\n",
       " '50': 'What is the impact of capital mobility on the exchange rate in the foreign exchange market?',\n",
       " '51': 'Какие основные методы используются для определения уровней поддержки и сопротивления?',\n",
       " '52': 'What is the primary reason why the payback period method is criticized for not differentiating between Projects B and C?',\n",
       " '53': 'Какие месяцы считаются квартальными для расчетных фьючерсов?',\n",
       " '54': 'What is the cash conversion cycle for Clear Lake Sporting Goods?',\n",
       " '55': 'What does the downward slope of the aggregate demand curve in Figure 24.4 indicate about the responsiveness of aggregate demand to changes in price levels?',\n",
       " '56': 'What is the role of a merchant banker in the filing process of an offer document with SEBI?',\n",
       " '57': 'Какие темы в области инвестиций и трейдинга освещены в рубрике «Обучение» БКС Экспресс?',\n",
       " '58': 'Какие операции не относятся к льготному периоду и могут привести к уплате процентов?',\n",
       " '59': \"What are the key challenges in accurately determining the 'cost of production' for foreign firms exporting to the United States?\",\n",
       " '60': 'What is the GDP of the nation described in the first paragraph?',\n",
       " '61': 'How has the availability of technology, such as laptops, impacted the education sector in remote areas like Ghana?',\n",
       " '62': 'Как интуиция может помочь в принятии решений на биржевой торговле?',\n",
       " '63': 'Какое наказание предусмотрено для гражданина Российской Федерации, который выезжает из страны, несмотря на ограничения, связанные с государственной тайной?',\n",
       " '64': 'How does the increase in federal outlays for education, training, employment, and social services from 2014 to 2021 compare to the increase in total federal outlays for major physical capital investment during the same period?',\n",
       " '65': 'Какие действия могут освободить лицо от уголовной ответственности по статье, предусмотренной частями первой-четвертой?',\n",
       " '66': 'Какие должности в Российской Федерации устанавливаются для непосредственного исполнения полномочий государственных органов?',\n",
       " '67': 'How can governments and policymakers best regulate and invest in human capital to manage employment shifts caused by globalization and technological advancements?',\n",
       " '68': 'What is the primary reason why economic entities with a surplus in their budget are willing to lend money in the market for loanable funds?',\n",
       " '69': 'Какой брокер или управляющая компания (УК) лучше выбрать для открытия ИИС-3, если я хочу самостоятельно управлять своими инвестициями?',\n",
       " '70': 'Какое наказание предусмотрено за деяние, совершенное группой лиц по предварительному сговору?',\n",
       " '71': 'What types of investments do insurance companies typically make with their reserves?',\n",
       " '72': 'What is the marginal revenue for the monopolistic competitor at the point where the quantity is 80 units?',\n",
       " '73': 'Какие виды страхования вы считаете наиболее важными для защиты вашего здоровья и имущества?',\n",
       " '74': 'Как вы думаете, почему ребенок не хочет тратить свои деньги?',\n",
       " '75': \"What are the potential risks and challenges that could arise from Robinson's agreement with Friday, especially regarding the repayment of the loan?\",\n",
       " '76': 'What are the potential long-term economic impacts of the government bailout of Fannie Mae and Freddie Mac?',\n",
       " '77': 'What is the real interest rate for a specified period of time?',\n",
       " '78': 'What is the primary difference between commodity money and fiat money?',\n",
       " '79': 'Какие факторы могут влиять на скорректированную EPS в секторе consumer discretionary?',\n",
       " '80': 'What is the difference between absolute and comparative advantage in the context of international trade?',\n",
       " '81': 'How does the allocative inefficiency in a monopolistically competitive industry compare to that in a monopoly?',\n",
       " '82': 'Какие органы имеют право изменять обязанности условно осужденного в течение испытательного срока?',\n",
       " '83': 'What are the potential long-term effects of partial inflation indexing on the economy, particularly for small businesses and financially unsophisticated individuals?',\n",
       " '84': 'Какие меры могут быть приняты для предотвращения дачи взятки должностному лицу?',\n",
       " '85': 'How does the shift in the supply of unskilled labor affect the wages and employment opportunities for native-born workers in the United States?',\n",
       " '86': 'What are the two main reasons why inflation may persist over time according to the provided context?',\n",
       " '87': 'What is the effect on the domestic currency when domestic interest rates rise due to an expected increase in inflation, compared to when they rise due to a higher real interest rate?',\n",
       " '88': 'What is the difference between the marginal revenue product (MRPL) and the marginal revenue (MR) for the firm in the given context?',\n",
       " '89': 'Какую долю дивидендов вы рекомендуете выводить для новичков, чтобы они могли убедиться в правильности своей инвестиционной стратегии?',\n",
       " '90': 'How does the Midpoint Method for Elasticity account for changes in ticket prices at the Super Bowl?',\n",
       " '91': 'What is the initial tenure for the standardized contracts for lending/borrowing transactions?',\n",
       " '92': 'What is the cost of reducing garbage by the fifth five tons for Elm trees?',\n",
       " '93': 'What are the key responsibilities of the trustee or sponsor in the exit process of an AIF with a closed-ended scheme?',\n",
       " '94': 'What are the key quantitative and qualitative requirements that a company must continuously comply with to maintain its listing on the Stock Exchange, as specified by SEBI?',\n",
       " '95': 'How does the combination of underwriting and research in investment banks lead to potential conflicts of interest, and what are the primary incentives behind these conflicts?',\n",
       " '96': 'How can the government effectively incentivize flu vaccinations to achieve the socially optimal quantity?',\n",
       " '97': 'What are some specific public policies that have been implemented in the United States to reduce earnings gaps between men and women?',\n",
       " '98': 'What is the predicted monthly revenue for the Fortune 500 company if it spends $80,000 on advertising?',\n",
       " '99': 'Какое максимальное наказание предусмотрено за данное преступление?'}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('questions_for_metrics.json', 'r', encoding=\"utf-8\") as file:\n",
    "    questions = json.load(file)\n",
    "\n",
    "questions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Какие операции по карте считаются бесплатными, если оплачиваете покупку кредиткой?'"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "questions['1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Оплата покупок кредитной картой считается бесплатной операцией. Банк не возьмет за это комиссию.\\n\\nИсточник: {'Course theme': 'Что такое кредитная карта и зачем она вам', 'source': 'tinkoff'}\""
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output = pipeline.get_response(questions['1'])\n",
    "output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [33:48<00:00, 20.28s/it]\n"
     ]
    }
   ],
   "source": [
    "answers = {\n",
    "    \"questions\" :[],\n",
    "    \"answers\" : [],\n",
    "    \"match criteria\" : []\n",
    "}\n",
    "\n",
    "for q in tqdm(questions.values()):\n",
    "    # Получаем ответ от RAG пайплайна\n",
    "    output = pipeline.get_response(q)\n",
    "\n",
    "    # Получаем ответ от проверяющего\n",
    "    promt = f\"\"\" \n",
    "    Оцени правильность ответа на вопрос по шкале от 0 до 1. Где 1 - ответ полностью соответствует вопросу, 0.5 - ответ не полностью раскрывает вопрос и 0 - ответ не соответствует вопросу. \n",
    "    Ответ должен быть выдан в формате JSON.\n",
    "    {{\n",
    "        \"match criteria\": ... \n",
    "    }}\n",
    "\n",
    "    Вопрос: {q}\n",
    "    Ответ: {output}\n",
    "    \"\"\"\n",
    "\n",
    "    messages = [\n",
    "        {\n",
    "            \"role\": \"system\",\n",
    "            \"content\": \"Ты бот который определяет правильность ответа на вопрос.\",            \n",
    "        },\n",
    "        {\n",
    "            \"role\": \"user\",\n",
    "            \"content\": promt\n",
    "        },\n",
    "    ]\n",
    "\n",
    "    time.sleep(5)\n",
    "    chat_response = judge.chat.complete(\n",
    "        model = \"ministral-8b-latest\",\n",
    "        messages = messages,\n",
    "        response_format = {\n",
    "            \"type\": \"json_object\",\n",
    "        },\n",
    "        temperature = 0.5\n",
    "    )\n",
    "    \n",
    "    answers[\"questions\"].append(q)\n",
    "    answers[\"answers\"].append(output)\n",
    "    answers[\"match criteria\"].append(eval(chat_response.choices[0].message.content)[\"match criteria\"])\n",
    "    time.sleep(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>questions</th>\n",
       "      <th>answers</th>\n",
       "      <th>match criteria</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Какие моральные дилеммы сталкиваются герои в ф...</td>\n",
       "      <td>Я не знаю ответа на данный вопрос.</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Какие операции по карте считаются бесплатными,...</td>\n",
       "      <td>Оплата покупок по кредитной карте считается бе...</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What strategies could a monopolist employ to m...</td>\n",
       "      <td>To maintain its market dominance in the face o...</td>\n",
       "      <td>0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Какое наказание предусмотрено за искусственное...</td>\n",
       "      <td>За искусственное создание доказательств обвине...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What are some examples of how market forces ha...</td>\n",
       "      <td>One example provided in the text is the persis...</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>How does the combination of underwriting and r...</td>\n",
       "      <td>The combination of underwriting and research i...</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>How can the government effectively incentivize...</td>\n",
       "      <td>The government can effectively incentivize flu...</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>What are some specific public policies that ha...</td>\n",
       "      <td>Я не знаю ответа на данный вопрос.</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>What is the predicted monthly revenue for the ...</td>\n",
       "      <td>The predicted monthly revenue for the Fortune ...</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>Какое максимальное наказание предусмотрено за ...</td>\n",
       "      <td>Я не знаю ответа на данный вопрос. В предостав...</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            questions  \\\n",
       "0   Какие моральные дилеммы сталкиваются герои в ф...   \n",
       "1   Какие операции по карте считаются бесплатными,...   \n",
       "2   What strategies could a monopolist employ to m...   \n",
       "3   Какое наказание предусмотрено за искусственное...   \n",
       "4   What are some examples of how market forces ha...   \n",
       "..                                                ...   \n",
       "95  How does the combination of underwriting and r...   \n",
       "96  How can the government effectively incentivize...   \n",
       "97  What are some specific public policies that ha...   \n",
       "98  What is the predicted monthly revenue for the ...   \n",
       "99  Какое максимальное наказание предусмотрено за ...   \n",
       "\n",
       "                                              answers  match criteria  \n",
       "0                  Я не знаю ответа на данный вопрос.             0.0  \n",
       "1   Оплата покупок по кредитной карте считается бе...             0.8  \n",
       "2   To maintain its market dominance in the face o...             0.9  \n",
       "3   За искусственное создание доказательств обвине...             1.0  \n",
       "4   One example provided in the text is the persis...             0.8  \n",
       "..                                                ...             ...  \n",
       "95  The combination of underwriting and research i...             0.8  \n",
       "96  The government can effectively incentivize flu...             0.8  \n",
       "97                 Я не знаю ответа на данный вопрос.             0.0  \n",
       "98  The predicted monthly revenue for the Fortune ...             0.5  \n",
       "99  Я не знаю ответа на данный вопрос. В предостав...             0.5  \n",
       "\n",
       "[100 rows x 3 columns]"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answers_df = pd.DataFrame(answers)\n",
    "answers_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: >"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAh8AAAGdCAYAAACyzRGfAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAH5hJREFUeJzt3Xts1fX9x/HXaTk9pdLTCgilo8XbtHhBszLgePkprNCoQR1NZGIYEqZbrGTSzAtTbPEGGgNupuDmELbEDocRNhSBrg4M0qpUSEBdJ4IDLa1DpQfoOD20n98fSw/WVugp57y70z0fyfnjfM+3n/Ppu6ftk9Me6nHOOQEAABhJ6u0NAACA/y3EBwAAMEV8AAAAU8QHAAAwRXwAAABTxAcAADBFfAAAAFPEBwAAMNWvtzfwTW1tbaqvr1d6ero8Hk9vbwcAAHSDc06HDx9Wdna2kpJO/tzGf1181NfXKycnp7e3AQAAemD//v0aPnz4Sc/5r4uP9PR0Sf/ZvN/vj+na4XBYGzdu1KRJk+T1emO6Nk5gzjaYsw3mbIdZ24jXnIPBoHJyciLfx0/mvy4+2n/U4vf74xIfaWlp8vv9PLDjiDnbYM42mLMdZm0j3nPuzq9M8AunAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABM9evtDQAA8HWXlG1QqPXUf5Y9Gp8svCGm6+H08MwHAAAwRXwAAABTxAcAADBFfAAAAFPEBwAAMEV8AAAAU8QHAAAwRXwAAABTxAcAADBFfAAAAFPEBwAAMEV8AAAAU8QHAAAwRXwAAABTxAcAADAVVXyUlZXJ4/F0uOTl5UVuP3bsmIqLizVo0CANGDBARUVFamxsjPmmAQBA4or6mY+LL75YBw4ciFy2bNkSuW3OnDlau3atVq1apc2bN6u+vl5TpkyJ6YYBAEBi6xf1G/Trp6ysrE7Hm5qatGzZMlVUVGjChAmSpOXLl2vkyJGqqanRuHHjTn+3AAAg4UUdHx999JGys7OVmpqqQCCgBQsWKDc3V7W1tQqHwyooKIicm5eXp9zcXFVXV39rfIRCIYVCocj1YDAoSQqHwwqHw9Fu76Ta14v1uuiIOdtgzjaYs532GfuSXNzWRvwe09Gs53HOdfuj/Prrr+vIkSO68MILdeDAAc2fP1+fffaZdu3apbVr12rmzJkdQkKSxowZo/Hjx+vJJ5/scs2ysjLNnz+/0/GKigqlpaV1+x0BAAC9p7m5WdOmTVNTU5P8fv9Jz40qPr7p0KFDGjFihBYtWqT+/fv3KD66euYjJydHBw8ePOXmoxUOh1VZWamJEyfK6/XGdG2cwJxtMGcbzNlO+6znbUtSqM0T07V3lRXGdL1EFq/HdDAY1ODBg7sVH1H/2OXrMjMzdcEFF2j37t2aOHGiWlpadOjQIWVmZkbOaWxs7PJ3RNr5fD75fL5Ox71eb9w+0eO5Nk5gzjaYsw3mbCfU5lGoNbbxwceus1g/pqNZ67T+n48jR47o448/1rBhw5Sfny+v16uqqqrI7XV1ddq3b58CgcDp3A0AAOhDonrm4xe/+IUmT56sESNGqL6+XqWlpUpOTtatt96qjIwMzZo1SyUlJRo4cKD8fr9mz56tQCDAK10AAEBEVPHx6aef6tZbb9UXX3yhs846S1dddZVqamp01llnSZIWL16spKQkFRUVKRQKqbCwUEuWLInLxgEAQGKKKj5Wrlx50ttTU1NVXl6u8vLy09oUAADou/jbLgAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwBTxAQAATBEfAADAFPEBAABMER8AAMAU8QEAAEwRHwAAwNRpxcfChQvl8Xh0zz33RI4dO3ZMxcXFGjRokAYMGKCioiI1Njae7j4BAEAf0eP4ePfdd/Wb3/xGo0aN6nB8zpw5Wrt2rVatWqXNmzervr5eU6ZMOe2NAgCAvqFH8XHkyBHddtttev7553XmmWdGjjc1NWnZsmVatGiRJkyYoPz8fC1fvlxbt25VTU1NzDYNAAASV4/io7i4WDfccIMKCgo6HK+trVU4HO5wPC8vT7m5uaqurj69nQIAgD6hX7RvsHLlSr333nt69913O93W0NCglJQUZWZmdjg+dOhQNTQ0dLleKBRSKBSKXA8Gg5KkcDiscDgc7fZOqn29WK+LjpizDeZsgznbaZ+xL8nFbW3E7zEdzXpRxcf+/fv185//XJWVlUpNTY16Y11ZsGCB5s+f3+n4xo0blZaWFpP7+KbKysq4rIuOmLMN5myDOdt5dHRbzNdct25dzNdMdLF+TDc3N3f7XI9zrtuJuWbNGv3whz9UcnJy5Fhra6s8Ho+SkpK0YcMGFRQU6Kuvvurw7MeIESN0zz33aM6cOZ3W7OqZj5ycHB08eFB+v7/b70h3hMNhVVZWauLEifJ6vTFdGycwZxvM2QZzttM+63nbkhRq88R07V1lhTFdL5HF6zEdDAY1ePBgNTU1nfL7d1TPfPzgBz/Qzp07OxybOXOm8vLydP/99ysnJ0der1dVVVUqKiqSJNXV1Wnfvn0KBAJdrunz+eTz+Tod93q9cftEj+faOIE522DONpiznVCbR6HW2MYHH7vOYv2YjmatqOIjPT1dl1xySYdjZ5xxhgYNGhQ5PmvWLJWUlGjgwIHy+/2aPXu2AoGAxo0bF81dAQCAPirqXzg9lcWLFyspKUlFRUUKhUIqLCzUkiVLYn03AAAgQZ12fGzatKnD9dTUVJWXl6u8vPx0lwYAAH0Qf9sFAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAICpqOJj6dKlGjVqlPx+v/x+vwKBgF5//fXI7ceOHVNxcbEGDRqkAQMGqKioSI2NjTHfNAAASFxRxcfw4cO1cOFC1dbWatu2bZowYYJuuukmvf/++5KkOXPmaO3atVq1apU2b96s+vp6TZkyJS4bBwAAialfNCdPnjy5w/XHH39cS5cuVU1NjYYPH65ly5apoqJCEyZMkCQtX75cI0eOVE1NjcaNGxe7XQMAgIQVVXx8XWtrq1atWqWjR48qEAiotrZW4XBYBQUFkXPy8vKUm5ur6urqb42PUCikUCgUuR4MBiVJ4XBY4XC4p9vrUvt6sV4XHTFnG8zZBnO20z5jX5KL29qI32M6mvWijo+dO3cqEAjo2LFjGjBggFavXq2LLrpIO3bsUEpKijIzMzucP3ToUDU0NHzregsWLND8+fM7Hd+4caPS0tKi3V63VFZWxmVddMScbTBnG8zZzqOj22K+5rp162K+ZqKL9WO6ubm52+dGHR8XXnihduzYoaamJr388suaMWOGNm/eHO0yEXPnzlVJSUnkejAYVE5OjiZNmiS/39/jdbsSDodVWVmpiRMnyuv1xnRtnMCcbTBnG8zZTvus521LUqjNE9O1d5UVxnS9RBavx3T7Ty66I+r4SElJ0fnnny9Jys/P17vvvqtf/epXmjp1qlpaWnTo0KEOz340NjYqKyvrW9fz+Xzy+Xydjnu93rh9osdzbZzAnG0wZxvM2U6ozaNQa2zjg49dZ7F+TEez1mn/Px9tbW0KhULKz8+X1+tVVVVV5La6ujrt27dPgUDgdO8GAAD0EVE98zF37lxdd911ys3N1eHDh1VRUaFNmzZpw4YNysjI0KxZs1RSUqKBAwfK7/dr9uzZCgQCvNIFAABERBUfn3/+uX784x/rwIEDysjI0KhRo7RhwwZNnDhRkrR48WIlJSWpqKhIoVBIhYWFWrJkSVw2DgAAElNU8bFs2bKT3p6amqry8nKVl5ef1qYAAEDfxd92AQAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApqKKjwULFuj73/++0tPTNWTIEN18882qq6vrcM6xY8dUXFysQYMGacCAASoqKlJjY2NMNw0AABJXVPGxefNmFRcXq6amRpWVlQqHw5o0aZKOHj0aOWfOnDlau3atVq1apc2bN6u+vl5TpkyJ+cYBAEBi6hfNyevXr+9wfcWKFRoyZIhqa2v1f//3f2pqatKyZctUUVGhCRMmSJKWL1+ukSNHqqamRuPGjYvdzgEAQEKKKj6+qampSZI0cOBASVJtba3C4bAKCgoi5+Tl5Sk3N1fV1dVdxkcoFFIoFIpcDwaDkqRwOKxwOHw62+ukfb1Yr4uOmLMN5myDOdtpn7EvycVtbcTvMR3Neh7nXI8+ym1tbbrxxht16NAhbdmyRZJUUVGhmTNndogJSRozZozGjx+vJ598stM6ZWVlmj9/fqfjFRUVSktL68nWAACAsebmZk2bNk1NTU3y+/0nPbfHz3wUFxdr165dkfDoqblz56qkpCRyPRgMKicnR5MmTTrl5qMVDodVWVmpiRMnyuv1xnRtnMCcbTBnG8zZTvus521LUqjNE9O1d5UVxnS9RBavx3T7Ty66o0fxcffdd+vVV1/Vm2++qeHDh0eOZ2VlqaWlRYcOHVJmZmbkeGNjo7Kysrpcy+fzyefzdTru9Xrj9okez7VxAnO2wZxtMGc7oTaPQq2xjQ8+dp3F+jEdzVpRvdrFOae7775bq1ev1htvvKFzzjmnw+35+fnyer2qqqqKHKurq9O+ffsUCASiuSsAANBHRfXMR3FxsSoqKvTnP/9Z6enpamhokCRlZGSof//+ysjI0KxZs1RSUqKBAwfK7/dr9uzZCgQCvNIFAABIijI+li5dKkm69tprOxxfvny5br/9dknS4sWLlZSUpKKiIoVCIRUWFmrJkiUx2SwAAEh8UcVHd14Yk5qaqvLycpWXl/d4UwAAoO86rf/nI1FdUrYh5r/M9MnCG2K6HgAAfRV/WA4AAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmCI+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgKl+vb0BAP9bzn7gtbit/cnCG+K2NoDY4ZkPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgql9vbwAAAHR29gOvxWVdX7LTU2PisnS38cwHAAAwRXwAAABTUcfHm2++qcmTJys7O1sej0dr1qzpcLtzTg8//LCGDRum/v37q6CgQB999FGs9gsAABJc1PFx9OhRXXbZZSovL+/y9qeeekq//vWv9dxzz+ntt9/WGWecocLCQh07duy0NwsAABJf1L9wet111+m6667r8jbnnJ555hk99NBDuummmyRJf/jDHzR06FCtWbNGP/rRj05vtwAAIOHF9NUue/fuVUNDgwoKCiLHMjIyNHbsWFVXV3cZH6FQSKFQKHI9GAxKksLhsMLhcCy3F1nPl+Riuu7X18aJWTCT+ErUOfuSY//51y4es0jUOScivkZ3FK/Plfb5xut7bHd4nHM9fu88Ho9Wr16tm2++WZK0detWXXnllaqvr9ewYcMi591yyy3yeDx66aWXOq1RVlam+fPndzpeUVGhtLS0nm4NAAAYam5u1rRp09TU1CS/33/Sc3v9//mYO3euSkpKIteDwaBycnI0adKkU24+WuFwWJWVlZq3LUmhNk9M195VVhjT9RJZ+5wnTpwor9fb29vpsxJ1zpeUbYjb2vH4PEzUOScivkZ3FK/PFV+S06Oj22L+mG7/yUV3xDQ+srKyJEmNjY0dnvlobGzU5Zdf3uXb+Hw++Xy+Tse9Xm/cPtFDbR6FWmP7wOaLUmfx/BjihESbc6w/974unnNItDknMr5G/0c8P1ek2D+mo1krpv/PxznnnKOsrCxVVVVFjgWDQb399tsKBAKxvCsAAJCgon7m48iRI9q9e3fk+t69e7Vjxw4NHDhQubm5uueee/TYY4/pu9/9rs455xzNmzdP2dnZkd8LAQAA/9uijo9t27Zp/Pjxkevtv68xY8YMrVixQvfdd5+OHj2qO++8U4cOHdJVV12l9evXKzU1NXa7BgAACSvq+Lj22mt1shfIeDwePfLII3rkkUdOa2MAAKBv4m+7AAAAU8QHAAAwRXwAAABTxAcAADBFfAAAAFPEBwAAMEV8AAAAU8QHAAAwRXwAAABTxAcAADBFfAAAAFPEBwAAMEV8AAAAU8QHAAAwRXwAAABTxAcAADBFfAAAAFPEBwAAMEV8AAAAU8QHAAAwRXwAAABTxAcAADBFfAAAAFPEBwAAMEV8AAAAU8QHAAAwRXwAAABTxAcAADBFfAAAAFPEBwAAMEV8AAAAU8QHAAAwRXwAAABTxAcAADBFfAAAAFPEBwAAMEV8AAAAU8QHAAAwRXwAAABTxAcAADBFfAAAAFPEBwAAMEV8AAAAU8QHAAAwRXwAAABTxAcAADDVr7c3AABAIjv7gdd6ewsJh2c+AACAKeIDAACYIj4AAIAp4gMAAJgiPgAAgCniAwAAmOKltgDQi+L5Ms1PFt4Qt7WB08EzHwAAwFTc4qO8vFxnn322UlNTNXbsWL3zzjvxuisAAJBA4hIfL730kkpKSlRaWqr33ntPl112mQoLC/X555/H4+4AAEACiUt8LFq0SHfccYdmzpypiy66SM8995zS0tL0wgsvxOPuAABAAon5L5y2tLSotrZWc+fOjRxLSkpSQUGBqqurO50fCoUUCoUi15uamiRJX375pcLhcEz3Fg6H1dzcrH7hJLW2eWK69hdffBHT9RJZ+5y/+OILeb3e3t5On5Woc+53/Gjc1o7H52G855xo84inRP0aHc+PYTz0a3Nqbm6L+WP68OHDkiTn3KlPdjH22WefOUlu69atHY7fe++9bsyYMZ3OLy0tdZK4cOHChQsXLn3gsn///lO2Qq+/1Hbu3LkqKSmJXG9ra9OXX36pQYMGyeOJbfkGg0Hl5ORo//798vv9MV0bJzBnG8zZBnO2w6xtxGvOzjkdPnxY2dnZpzw35vExePBgJScnq7GxscPxxsZGZWVldTrf5/PJ5/N1OJaZmRnrbXXg9/t5YBtgzjaYsw3mbIdZ24jHnDMyMrp1Xsx/4TQlJUX5+fmqqqqKHGtra1NVVZUCgUCs7w4AACSYuPzYpaSkRDNmzNDo0aM1ZswYPfPMMzp69KhmzpwZj7sDAAAJJC7xMXXqVP3rX//Sww8/rIaGBl1++eVav369hg4dGo+76zafz6fS0tJOP+ZBbDFnG8zZBnO2w6xt/DfM2eNcd14TAwAAEBv8bRcAAGCK+AAAAKaIDwAAYIr4AAAApvpcfJSXl+vss89Wamqqxo4dq3feeeek569atUp5eXlKTU3VpZdeqnXr1hntNLFFM+fnn39eV199tc4880ydeeaZKigoOOXHBf8R7eO53cqVK+XxeHTzzTfHd4N9RLRzPnTokIqLizVs2DD5fD5dcMEFfO3ohmjn/Mwzz+jCCy9U//79lZOTozlz5ujYsWNGu01Mb775piZPnqzs7Gx5PB6tWbPmlG+zadMmfe9735PP59P555+vFStWxH2fMf/bLr1p5cqVLiUlxb3wwgvu/fffd3fccYfLzMx0jY2NXZ7/1ltvueTkZPfUU0+5Dz74wD300EPO6/W6nTt3Gu88sUQ752nTprny8nK3fft29+GHH7rbb7/dZWRkuE8//dR454kl2jm327t3r/vOd77jrr76anfTTTfZbDaBRTvnUCjkRo8e7a6//nq3ZcsWt3fvXrdp0ya3Y8cO450nlmjn/OKLLzqfz+defPFFt3fvXrdhwwY3bNgwN2fOHOOdJ5Z169a5Bx980L3yyitOklu9evVJz9+zZ49LS0tzJSUl7oMPPnDPPvusS05OduvXr4/rPvtUfIwZM8YVFxdHrre2trrs7Gy3YMGCLs+/5ZZb3A033NDh2NixY91Pf/rTuO4z0UU75286fvy4S09Pd7///e/jtcU+oSdzPn78uLviiivc7373OzdjxgzioxuinfPSpUvdueee61paWqy22CdEO+fi4mI3YcKEDsdKSkrclVdeGdd99iXdiY/77rvPXXzxxR2OTZ061RUWFsZxZ871mR+7tLS0qLa2VgUFBZFjSUlJKigoUHV1dZdvU11d3eF8SSosLPzW89GzOX9Tc3OzwuGwBg4cGK9tJryezvmRRx7RkCFDNGvWLIttJryezPkvf/mLAoGAiouLNXToUF1yySV64okn1NraarXthNOTOV9xxRWqra2N/Ghmz549Wrduna6//nqTPf+v6K3vg73+V21j5eDBg2ptbe30v6gOHTpUf//737t8m4aGhi7Pb2hoiNs+E11P5vxN999/v7Kzszs94HFCT+a8ZcsWLVu2TDt27DDYYd/Qkznv2bNHb7zxhm677TatW7dOu3fv1l133aVwOKzS0lKLbSecnsx52rRpOnjwoK666io553T8+HH97Gc/0y9/+UuLLf/P+Lbvg8FgUP/+97/Vv3//uNxvn3nmA4lh4cKFWrlypVavXq3U1NTe3k6fcfjwYU2fPl3PP/+8Bg8e3Nvb6dPa2to0ZMgQ/fa3v1V+fr6mTp2qBx98UM8991xvb61P2bRpk5544gktWbJE7733nl555RW99tprevTRR3t7a4iBPvPMx+DBg5WcnKzGxsYOxxsbG5WVldXl22RlZUV1Pno253ZPP/20Fi5cqL/+9a8aNWpUPLeZ8KKd88cff6xPPvlEkydPjhxra2uTJPXr1091dXU677zz4rvpBNSTx/OwYcPk9XqVnJwcOTZy5Eg1NDSopaVFKSkpcd1zIurJnOfNm6fp06frJz/5iSTp0ksv1dGjR3XnnXfqwQcfVFIS/3aOhW/7Puj3++P2rIfUh575SElJUX5+vqqqqiLH2traVFVVpUAg0OXbBAKBDudLUmVl5beej57NWZKeeuopPfroo1q/fr1Gjx5tsdWEFu2c8/LytHPnTu3YsSNyufHGGzV+/Hjt2LFDOTk5lttPGD15PF955ZXavXt3JO4k6R//+IeGDRtGeHyLnsy5ubm5U2C0B5/jT5LFTK99H4zrr7MaW7lypfP5fG7FihXugw8+cHfeeafLzMx0DQ0Nzjnnpk+f7h544IHI+W+99Zbr16+fe/rpp92HH37oSktLealtN0Q754ULF7qUlBT38ssvuwMHDkQuhw8f7q13ISFEO+dv4tUu3RPtnPft2+fS09Pd3Xff7erq6tyrr77qhgwZ4h577LHeehcSQrRzLi0tdenp6e6Pf/yj27Nnj9u4caM777zz3C233NJb70JCOHz4sNu+fbvbvn27k+QWLVrktm/f7v75z38655x74IEH3PTp0yPnt7/U9t5773UffvihKy8v56W2PfHss8+63Nxcl5KS4saMGeNqamoit11zzTVuxowZHc7/05/+5C644AKXkpLiLr74Yvfaa68Z7zgxRTPnESNGOEmdLqWlpfYbTzDRPp6/jvjovmjnvHXrVjd27Fjn8/ncueee6x5//HF3/Phx410nnmjmHA6HXVlZmTvvvPNcamqqy8nJcXfddZf76quv7DeeQP72t791+fW2fbYzZsxw11xzTae3ufzyy11KSoo799xz3fLly+O+T49zPH8FAADs9Jnf+QAAAImB+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACmiA8AAGCK+AAAAKaIDwAAYIr4AAAApogPAABgivgAAACm/h+yOCcCBRJ24wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "answers_df[\"match criteria\"].hist(bins = 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>questions</th>\n",
       "      <th>answers</th>\n",
       "      <th>match criteria</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Какие моральные дилеммы сталкиваются герои в ф...</td>\n",
       "      <td>Я не знаю ответа на данный вопрос.</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Какие операции по карте считаются бесплатными,...</td>\n",
       "      <td>Оплата покупок по кредитной карте считается бе...</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What strategies could a monopolist employ to m...</td>\n",
       "      <td>To maintain its market dominance in the face o...</td>\n",
       "      <td>0.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Какое наказание предусмотрено за искусственное...</td>\n",
       "      <td>За искусственное создание доказательств обвине...</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What are some examples of how market forces ha...</td>\n",
       "      <td>One example provided in the text is the persis...</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           questions  \\\n",
       "0  Какие моральные дилеммы сталкиваются герои в ф...   \n",
       "1  Какие операции по карте считаются бесплатными,...   \n",
       "2  What strategies could a monopolist employ to m...   \n",
       "3  Какое наказание предусмотрено за искусственное...   \n",
       "4  What are some examples of how market forces ha...   \n",
       "\n",
       "                                             answers  match criteria  \n",
       "0                 Я не знаю ответа на данный вопрос.             0.0  \n",
       "1  Оплата покупок по кредитной карте считается бе...             0.8  \n",
       "2  To maintain its market dominance in the face o...             0.9  \n",
       "3  За искусственное создание доказательств обвине...             1.0  \n",
       "4  One example provided in the text is the persis...             0.8  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answers_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "answers_df.to_csv(\"metrics_response_vs_input.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6289999999999999"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answers_df[\"match criteria\"].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Определяем документы для тестирования"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['**Figure 1.1 Finance is the linchpin that connects and directs many parts of a business or organization. (credit: modification of work**\\n“Finance behind the Glass” by Max London/flickr, CC BY 2.0)',\n",
       " 'By the end of this section, you will be able to:',\n",
       " 'Finance is the study of the management, movement, and raising of money. The word finance can be used as a\\nverb, such as when the First National Bank agrees to finance your home mortgage loan. It can also be used as\\na noun referring to an entire industry. At its essence, the study of finance is about understanding the uses and\\ndecision makers.',\n",
       " 'Finance is divided into three primary areas in the domestic market: business finance, investments, and\\nfinancial markets and institutions (see Figure 1.2). We look at each here in turn.\\n**Figure 1.2 The Three Basic Areas of Study in Finance**\\n**Business Finance**\\n**Business finance looks at how managers can apply financial principles to maximize the value of a firm in a**\\nrisky environment. Businesses have many stakeholders. In the case of corporations, the shareholders own the\\ncompany, and they hire managers to run the company with the intent to maximize shareholder wealth.\\nConsequently, all management decisions should run through the filter of these questions: “How does this\\ndecision impact the wealth of the shareholders?” and “Is this the best decision to be made for shareholders?”\\nIn business finance, managers focus on three broad areas (see Figure 1.3).\\nThe chief financial officer (CFO) and the finance team are responsible for establishing company policy for\\nhow to manage WCM. The finance department determines credit policy, establishes minimum criteria for\\nthe extension of credit to clients, terms of lending, when to extend, and when to take advantage of shortterm creditor financing. The accounting department basically implements the finance department’s\\npolicies. In many firms, the accounting and finance functions operate in the same department; in others,\\nthey are separate.',\n",
       " 'policies. In many firms, the accounting and finance functions operate in the same department; in others,\\nthey are separate.\\nmaximize shareholder value. Capital budgeting decisions add the greatest value to a firm. As such, capital\\nbudgeting is thought to be one of the most important financial functions within a firm. The capital\\nbudgeting process consists of estimating the value of potential investments by forecasting the size,\\ntiming, and risk of cash flows associated with the investments. The finance department develops and\\ncompiles cash flow estimates with input from the marketing, operations, accounting, human resources,\\nand economics departments to develop a portfolio of investment projects that collectively maximize the\\nvalue of the firm.\\nincreasing shareholder wealth. Capital structure questions require financial managers to work with\\neconomists, lenders, underwriters, investment bankers, and other sources of external financial\\ninformation and financial capital. When Bacon Signs developed its financial plan, the executives included\\neach of these three aspects of business finance into the plan.\\nand equity financing.\\n**Investments**\\n**Investments are products and processes used to create and grow wealth. Most commonly, investment topics**\\ninclude the discussion and application of the different types of financial instruments, delivery vehicles,\\nsecurities such as futures and options. A broad coverage of investment instruments would include mutual']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents = vector_store.docstore.__dict__[\"_dict\"]\n",
    "documents = [documents[index].page_content for index in documents]\n",
    "documents[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'**Figure 1.1 Finance is the linchpin that connects and directs many parts of a business or organization. (credit: modification of work**\\n“Finance behind the Glass” by Max London/flickr, CC BY 2.0)'"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "documents[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Выбираем 100 рандомных документов для тестирования"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [17:48<00:00, 10.69s/it]\n"
     ]
    }
   ],
   "source": [
    "questions = []\n",
    "\n",
    "for doc in tqdm(random.sample(documents, 100)):\n",
    "    promt = f\"\"\" \n",
    "    Придумай 1 вопрос исходя из контекста ниже. Ответ должен быть выдан в формате JSON.\n",
    "    {{\n",
    "        \"question\": \"...\"\n",
    "    }}\n",
    "\n",
    "    Контекст:\n",
    "    {doc}\n",
    "    \"\"\"\n",
    "\n",
    "    messages = [\n",
    "        {\n",
    "            \"role\": \"system\",\n",
    "            \"content\": \"Ты бот который помогает придумывать вопросы исходя из контекста.\",            \n",
    "        },\n",
    "        {\n",
    "            \"role\": \"user\",\n",
    "            \"content\": promt\n",
    "        },\n",
    "    ]\n",
    "\n",
    "    chat_response = judge.chat.complete(\n",
    "        model = \"ministral-8b-latest\",\n",
    "        messages = messages,\n",
    "        response_format = {\n",
    "            \"type\": \"json_object\",\n",
    "        },\n",
    "        temperature = 0.5\n",
    "    )\n",
    "\n",
    "    q = eval(chat_response.choices[0].message.content)\n",
    "\n",
    "    questions.append(q[\"question\"])\n",
    "    time.sleep(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"questions_for_metrics.json\", \"w\", encoding=\"utf-8\") as f:\n",
    "    json.dump({i:questions[i]  for i in range(len(questions))}, f, ensure_ascii = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}