prepare_subset_of_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "from allennlp.predictors.predictor import Predictor\n",
    "import allennlp_models.pair_classification\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "import json\n",
    "import requests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "bot_name = 'curekart'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Did not use initialization regex that was passed: .*token_embedder_tokens\\._projection.*weight\n"
     ]
    }
   ],
   "source": [
    "predictor = Predictor.from_path(\"https://storage.googleapis.com/allennlp-public-models/decomposable-attention-elmo-2020.04.09.tar.gz\", \"textual_entailment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_entail(premise, hypothesis):\n",
    "    premise = premise.lower()\n",
    "    hypothesis = hypothesis.lower()\n",
    "    res = predictor.predict(hypothesis=hypothesis, premise=premise)\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CALL_CENTER</td>\n",
       "      <td>What time is your call centre operational duri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CALL_CENTER</td>\n",
       "      <td>is the call center still functioning during lo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CALL_CENTER</td>\n",
       "      <td>what are the working hours of your call center...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CALL_CENTER</td>\n",
       "      <td>does covid affext your call center time</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CALL_CENTER</td>\n",
       "      <td>is your call center working during covid?</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         label                                           sentence\n",
       "0  CALL_CENTER  What time is your call centre operational duri...\n",
       "1  CALL_CENTER  is the call center still functioning during lo...\n",
       "2  CALL_CENTER  what are the working hours of your call center...\n",
       "3  CALL_CENTER            does covid affext your call center time\n",
       "4  CALL_CENTER          is your call center working during covid?"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_train = pd.read_csv(f'train/{bot_name}_train.csv')\n",
    "df_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>I am confused about what to buy since there ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>I have been trying to maintain a healthy lifes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>Can you help me with building an athletic body...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>I need some hair care products since I have be...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>I'm here to browse some products because my fr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>I want help to reach my weight goal, can you s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>Can you please suggest me some products specif...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>give me some other products regarding building...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>I am aiming to get a lean body, can you help me?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>What is the best pre workout protein to boost ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               label                                           sentence\n",
       "0  RECOMMEND_PRODUCT  I am confused about what to buy since there ar...\n",
       "1  RECOMMEND_PRODUCT  I have been trying to maintain a healthy lifes...\n",
       "2  RECOMMEND_PRODUCT  Can you help me with building an athletic body...\n",
       "3  RECOMMEND_PRODUCT  I need some hair care products since I have be...\n",
       "4  RECOMMEND_PRODUCT  I'm here to browse some products because my fr...\n",
       "5  RECOMMEND_PRODUCT  I want help to reach my weight goal, can you s...\n",
       "6  RECOMMEND_PRODUCT  Can you please suggest me some products specif...\n",
       "7  RECOMMEND_PRODUCT  give me some other products regarding building...\n",
       "8  RECOMMEND_PRODUCT   I am aiming to get a lean body, can you help me?\n",
       "9  RECOMMEND_PRODUCT  What is the best pre workout protein to boost ..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_labels = list(set(df_train['label']))\n",
    "correct_order_sens = []\n",
    "correct_order_labels = []\n",
    "for label in all_labels:\n",
    "    sens = list(df_train[df_train['label']==label]['sentence'])\n",
    "    sens = sorted(sens, key = lambda x: len(str(x).split()), reverse=True)\n",
    "    correct_order_sens.extend(sens)\n",
    "    correct_order_labels.extend([label]*len(sens))\n",
    "df_correct_order = pd.DataFrame({'label': correct_order_labels, 'sentence': correct_order_sens})\n",
    "df_correct_order.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 600/600 [1:14:42<00:00,  7.47s/it]\n"
     ]
    }
   ],
   "source": [
    "# dict of {node_name: list of parent sens}\n",
    "parent_sens = defaultdict(list)\n",
    "# dict of {node_name: {parent: list of child}}\n",
    "child_sens = defaultdict(lambda: defaultdict(list))\n",
    "# raw output of entail\n",
    "entail_output = {}\n",
    "for index, row in tqdm(df_correct_order.iterrows(), total=df_correct_order.shape[0]):\n",
    "    is_parent = True\n",
    "    for psen in parent_sens[row['label']]:\n",
    "        entail_forward = get_entail(psen, row['sentence'])\n",
    "        entail_backward = get_entail(row['sentence'], psen)\n",
    "        entail_output[(psen, row['sentence'])] = entail_forward\n",
    "        entail_output[(row['sentence'], psen)] = entail_backward\n",
    "        if entail_forward['label_probs'][0] > 0.6 and entail_backward['label_probs'][0] > 0.6:\n",
    "            is_parent = False\n",
    "            child_sens[row['label']][psen].append(row['sentence'])\n",
    "            break\n",
    "    if is_parent:\n",
    "        parent_sens[row['label']].append(row['sentence'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(parent_sens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>I am confused about what to buy since there ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>I have been trying to maintain a healthy lifes...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>Can you help me with building an athletic body...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>I need some hair care products since I have be...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>I'm here to browse some products because my fr...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               label                                           sentence\n",
       "0  RECOMMEND_PRODUCT  I am confused about what to buy since there ar...\n",
       "1  RECOMMEND_PRODUCT  I have been trying to maintain a healthy lifes...\n",
       "2  RECOMMEND_PRODUCT  Can you help me with building an athletic body...\n",
       "3  RECOMMEND_PRODUCT  I need some hair care products since I have be...\n",
       "4  RECOMMEND_PRODUCT  I'm here to browse some products because my fr..."
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_nodes = []\n",
    "all_sens = []\n",
    "for node, sens in parent_sens.items():\n",
    "    sens = [str(s) for s in sens]\n",
    "    all_sens.extend(sens)\n",
    "    all_nodes.extend([node]*len(sens))\n",
    "assert(len(all_nodes) == len(all_sens))\n",
    "df_subset_train = pd.DataFrame({'label': all_nodes, 'sentence': all_sens})\n",
    "df_subset_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(413, 2)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_subset_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>parents</th>\n",
       "      <th>child</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>Which products can you help me?</td>\n",
       "      <td>Can you suggest me some products?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>I want get muscle mass and lean mass</td>\n",
       "      <td>I want to gain more muscles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>what products would you suggest to lose weight?</td>\n",
       "      <td>I want to gain my weight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>i mean i want protein supplement</td>\n",
       "      <td>Wanted to get Protein powder</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>RECOMMEND_PRODUCT</td>\n",
       "      <td>I want to get the product</td>\n",
       "      <td>I want to buy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               label                                          parents  \\\n",
       "0  RECOMMEND_PRODUCT                  Which products can you help me?   \n",
       "1  RECOMMEND_PRODUCT             I want get muscle mass and lean mass   \n",
       "2  RECOMMEND_PRODUCT  what products would you suggest to lose weight?   \n",
       "3  RECOMMEND_PRODUCT                 i mean i want protein supplement   \n",
       "4  RECOMMEND_PRODUCT                        I want to get the product   \n",
       "\n",
       "                               child  \n",
       "0  Can you suggest me some products?  \n",
       "1        I want to gain more muscles  \n",
       "2           I want to gain my weight  \n",
       "3       Wanted to get Protein powder  \n",
       "4                      I want to buy  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# all children\n",
    "node_names = []\n",
    "parents = []\n",
    "childs = []\n",
    "for node, pc in child_sens.items():\n",
    "    for par, children in pc.items():\n",
    "        node_names.extend([node]*len(children))\n",
    "        parents.extend([par]*len(children))\n",
    "        childs.extend(children)\n",
    "    assert len(node_names) == len(parents) == len(childs)\n",
    "df_node_parent_child = pd.DataFrame({'label': node_names, 'parents': parents, 'child': childs})\n",
    "df_node_parent_child.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_subset_train.to_csv(f'train/{bot_name}_subset_train.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_node_parent_child.to_csv(f'assets/{bot_name}_allennlp_node_parent_child')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "with open(f'assets/{bot_name}_allennlp_raw_output.pkl', 'wb') as f:\n",
    "    pickle.dump(entail_output, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (py36)",
   "language": "python",
   "name": "py36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}