ngram_inference.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classifying OUV using NGram features and MLP"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/opt/miniconda3/bin/python'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import sys\n",
    "sys.executable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from argparse import Namespace\n",
    "from collections import Counter\n",
    "import json\n",
    "import os\n",
    "import re\n",
    "import string\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from scipy.special import softmax\n",
    "\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch.autograd.profiler as profiler\n",
    "\n",
    "import torchtext\n",
    "from torchtext.data import get_tokenizer\n",
    "tokenizer = get_tokenizer('spacy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PyTorch version 1.7.0\n",
      "GPU-enabled installation? False\n"
     ]
    }
   ],
   "source": [
    "print(\"PyTorch version {}\".format(torch.__version__))\n",
    "print(\"GPU-enabled installation? {}\".format(torch.cuda.is_available()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cpu\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Vectorization Classes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Vocabulary(object):\n",
    "    \"\"\"Class to process text and extract vocabulary for mapping\"\"\"\n",
    "\n",
    "    def __init__(self, token_to_idx=None, add_unk=True, unk_token=\"<UNK>\"):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            token_to_idx (dict): a pre-existing map of tokens to indices\n",
    "            add_unk (bool): a flag that indicates whether to add the UNK token\n",
    "            unk_token (str): the UNK token to add into the Vocabulary\n",
    "        \"\"\"\n",
    "\n",
    "        if token_to_idx is None:\n",
    "            token_to_idx = {}\n",
    "        self._token_to_idx = token_to_idx\n",
    "\n",
    "        self._idx_to_token = {idx: token \n",
    "                              for token, idx in self._token_to_idx.items()}\n",
    "        \n",
    "        self._add_unk = add_unk\n",
    "        self._unk_token = unk_token\n",
    "        \n",
    "        self.unk_index = -1\n",
    "        if add_unk:\n",
    "            self.unk_index = self.add_token(unk_token) \n",
    "        \n",
    "        \n",
    "    def to_serializable(self):\n",
    "        \"\"\" returns a dictionary that can be serialized \"\"\"\n",
    "        return {'token_to_idx': self._token_to_idx, \n",
    "                'add_unk': self._add_unk, \n",
    "                'unk_token': self._unk_token}\n",
    "\n",
    "    @classmethod\n",
    "    def from_serializable(cls, contents):\n",
    "        \"\"\" instantiates the Vocabulary from a serialized dictionary \"\"\"\n",
    "        return cls(**contents)\n",
    "\n",
    "    def add_token(self, token):\n",
    "        \"\"\"Update mapping dicts based on the token.\n",
    "\n",
    "        Args:\n",
    "            token (str): the item to add into the Vocabulary\n",
    "        Returns:\n",
    "            index (int): the integer corresponding to the token\n",
    "        \"\"\"\n",
    "        if token in self._token_to_idx:\n",
    "            index = self._token_to_idx[token]\n",
    "        else:\n",
    "            index = len(self._token_to_idx)\n",
    "            self._token_to_idx[token] = index\n",
    "            self._idx_to_token[index] = token\n",
    "        return index\n",
    "    \n",
    "    def add_many(self, tokens):\n",
    "        \"\"\"Add a list of tokens into the Vocabulary\n",
    "        \n",
    "        Args:\n",
    "            tokens (list): a list of string tokens\n",
    "        Returns:\n",
    "            indices (list): a list of indices corresponding to the tokens\n",
    "        \"\"\"\n",
    "        return [self.add_token(token) for token in tokens]\n",
    "\n",
    "    def lookup_token(self, token):\n",
    "        \"\"\"Retrieve the index associated with the token \n",
    "          or the UNK index if token isn't present.\n",
    "        \n",
    "        Args:\n",
    "            token (str): the token to look up \n",
    "        Returns:\n",
    "            index (int): the index corresponding to the token\n",
    "        Notes:\n",
    "            `unk_index` needs to be >=0 (having been added into the Vocabulary) \n",
    "              for the UNK functionality \n",
    "        \"\"\"\n",
    "        if self.unk_index >= 0:\n",
    "            return self._token_to_idx.get(token, self.unk_index)\n",
    "        else:\n",
    "            return self._token_to_idx[token]\n",
    "\n",
    "    def lookup_index(self, index):\n",
    "        \"\"\"Return the token associated with the index\n",
    "        \n",
    "        Args: \n",
    "            index (int): the index to look up\n",
    "        Returns:\n",
    "            token (str): the token corresponding to the index\n",
    "        Raises:\n",
    "            KeyError: if the index is not in the Vocabulary\n",
    "        \"\"\"\n",
    "        if index not in self._idx_to_token:\n",
    "            raise KeyError(\"the index (%d) is not in the Vocabulary\" % index)\n",
    "        return self._idx_to_token[index]\n",
    "\n",
    "    def __str__(self):\n",
    "        return \"<Vocabulary(size=%d)>\" % len(self)\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self._token_to_idx)\n",
    "\n",
    "\"\"\"### The Vectorizer\"\"\"\n",
    "def sparse_to_tensor(M):\n",
    "    \"\"\"\n",
    "    input: M is Scipy sparse matrix\n",
    "    output: pytorch sparse tensor in GPU \n",
    "    \"\"\"\n",
    "    M = M.tocoo().astype(np.float32)\n",
    "    indices = torch.from_numpy(np.vstack((M.row, M.col))).long()\n",
    "    values = torch.from_numpy(M.data)\n",
    "    shape = torch.Size(M.shape)\n",
    "    Ms = torch.sparse.FloatTensor(indices, values, shape)\n",
    "    return Ms.to_dense().to(args.device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ngrams_iterator(token_list, ngrams):\n",
    "    \"\"\"Return an iterator that yields the given tokens and their ngrams.\n",
    "\n",
    "    Arguments:\n",
    "        token_list: A list of tokens\n",
    "        ngrams: the number of ngrams.\n",
    "\n",
    "    Examples:\n",
    "        >>> token_list = ['here', 'we', 'are']\n",
    "        >>> list(ngrams_iterator(token_list, 2))\n",
    "        >>> ['here', 'here we', 'we', 'we are', 'are']\n",
    "    \"\"\"\n",
    "\n",
    "    def _get_ngrams(n):\n",
    "        return zip(*[token_list[i:] for i in range(n)])\n",
    "\n",
    "    for x in token_list:\n",
    "        yield x\n",
    "    for n in range(2, ngrams + 1):\n",
    "        for x in _get_ngrams(n):\n",
    "            yield ' '.join(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Vectorization parameters\n",
    "# Range (inclusive) of n-gram sizes for tokenizing text.\n",
    "NGRAM_RANGE = (1, 2)\n",
    "\n",
    "# Limit on the number of features. We use the top 20K features.\n",
    "TOP_K = 20000\n",
    "\n",
    "# Whether text should be split into word or character n-grams.\n",
    "# One of 'word', 'char'.\n",
    "TOKEN_MODE = 'word'\n",
    "\n",
    "# Minimum document/corpus frequency below which a token will be discarded.\n",
    "MIN_DOCUMENT_FREQUENCY = 2\n",
    "\n",
    "def sparse_to_tensor(M):\n",
    "    \"\"\"\n",
    "    input: M is Scipy sparse matrix\n",
    "    output: pytorch sparse tensor in GPU \n",
    "    \"\"\"\n",
    "    M = M.tocoo().astype(np.float32)\n",
    "    indices = torch.from_numpy(np.vstack((M.row, M.col))).long()\n",
    "    values = torch.from_numpy(M.data)\n",
    "    shape = torch.Size(M.shape)\n",
    "    Ms = torch.sparse.FloatTensor(indices, values, shape)\n",
    "    return Ms.to_dense().to(args.device)\n",
    "\n",
    "class OuvVectorizer(object):\n",
    "    \"\"\" The Vectorizer which coordinates the Vocabularies and puts them to use\"\"\"\n",
    "    def __init__(self, ouv_vocab, ngrams, vectorizer):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            review_vocab (Vocabulary): maps words to integers\n",
    "        \"\"\"\n",
    "        self.ouv_vocab = ouv_vocab\n",
    "        self.ngrams = ngrams\n",
    "        self.vectorizer = vectorizer\n",
    "        \n",
    "    def vectorize(self, data):\n",
    "        \"\"\"Create a tf_idf vector for the ouv data\n",
    "        \n",
    "        Args:\n",
    "            data (str): the ouv description data\n",
    "            ngrams (int): the maximum ngram value\n",
    "        Returns:\n",
    "            tf_idf (np.ndarray): the tf-idf encoding \n",
    "        \"\"\"\n",
    "        data = [data]\n",
    "        tf_idf = self.vectorizer.transform(data)\n",
    "        \n",
    "        return sparse_to_tensor(tf_idf)[0]\n",
    "\n",
    "    @classmethod\n",
    "    def from_dataframe(cls, ouv_df, ngrams, cutoff=5):\n",
    "        \"\"\"Instantiate the vectorizer from the dataset dataframe\n",
    "        \n",
    "        Args:\n",
    "            ouv_df (pandas.DataFrame): the ouv dataset\n",
    "            cutoff (int): the parameter for frequency-based filtering\n",
    "            ngrams (int): the maximum ngram value\n",
    "        Returns:\n",
    "            an instance of the OuvVectorizer\n",
    "        \"\"\"\n",
    "        ouv_vocab = Vocabulary(add_unk=True)\n",
    "        corpus=[]\n",
    "        \n",
    "        # Add top words if count > provided count\n",
    "        word_counts = Counter()\n",
    "        for data in ouv_df.data:\n",
    "            corpus.append(data)\n",
    "            for word in ngrams_iterator(data.split(' '),ngrams=ngrams):\n",
    "                if word not in string.punctuation:\n",
    "                    word_counts[word] += 1\n",
    "                     \n",
    "        for word, count in word_counts.items():\n",
    "            if count > cutoff:\n",
    "                ouv_vocab.add_token(word)\n",
    "        \n",
    "        # Create keyword arguments to pass to the 'tf-idf' vectorizer.\n",
    "        kwargs = {\n",
    "                'ngram_range': NGRAM_RANGE,  # Use 1-grams + 2-grams.\n",
    "                'dtype': 'int32',\n",
    "                'strip_accents': 'unicode',\n",
    "                'decode_error': 'replace',\n",
    "                'analyzer': TOKEN_MODE,  # Split text into word tokens.\n",
    "                'min_df': MIN_DOCUMENT_FREQUENCY,\n",
    "        }\n",
    "        vectorizer = TfidfVectorizer(**kwargs)\n",
    "\n",
    "        # Learn vocabulary from training texts and vectorize training texts.\n",
    "        vectorizer.fit_transform(corpus).astype('float32')\n",
    "\n",
    "        return cls(ouv_vocab, ngrams, vectorizer)\n",
    "\n",
    "    @classmethod\n",
    "    def from_serializable(cls, contents, ngrams, vectorizer):\n",
    "        \"\"\"Instantiate a OuvVectorizer from a serializable dictionary\n",
    "        \n",
    "        Args:\n",
    "            contents (dict): the serializable dictionary\n",
    "        Returns:\n",
    "            an instance of the OuvVectorizer class\n",
    "        \"\"\"\n",
    "        ouv_vocab = Vocabulary.from_serializable(contents['ouv_vocab'])\n",
    "        \n",
    "        return cls(ouv_vocab=ouv_vocab, ngrams=ngrams, vectorizer = vectorizer)\n",
    "\n",
    "    def to_serializable(self):\n",
    "        \"\"\"Create the serializable dictionary for caching\n",
    "        \n",
    "        Returns:\n",
    "            contents (dict): the serializable dictionary\n",
    "        \"\"\"\n",
    "        return {'ouv_vocab': self.ouv_vocab.to_serializable()}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OuvDataset(Dataset):\n",
    "    def __init__(self, ouv_df, vectorizer):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            ouv_df (pandas.DataFrame): the dataset\n",
    "            vectorizer (ReviewVectorizer): vectorizer instantiated from dataset\n",
    "        \"\"\"\n",
    "        self.ouv_df = ouv_df\n",
    "        self._vectorizer = vectorizer\n",
    "\n",
    "        self.train_df = self.ouv_df[self.ouv_df.split=='train']\n",
    "        self.train_size = len(self.train_df)\n",
    "\n",
    "        self.val_df = self.ouv_df[self.ouv_df.split=='dev']\n",
    "        self.validation_size = len(self.val_df)\n",
    "\n",
    "        self.test_df = self.ouv_df[self.ouv_df.split=='test']\n",
    "        self.test_size = len(self.test_df)\n",
    "\n",
    "        self._lookup_dict = {'train': (self.train_df, self.train_size),\n",
    "                             'val': (self.val_df, self.validation_size),\n",
    "                             'test': (self.test_df, self.test_size)}\n",
    "\n",
    "        self.set_split('train')\n",
    "\n",
    "    @classmethod\n",
    "    def load_dataset_and_make_vectorizer(cls, ouv_csv, ngrams, cutoff):\n",
    "        \"\"\"Load dataset and make a new vectorizer from scratch\n",
    "        \n",
    "        Args:\n",
    "            ouv_csv (str): location of the dataset\n",
    "        Returns:\n",
    "            an instance of OuvDataset\n",
    "        \"\"\"\n",
    "        ouv_df = pd.read_csv(ouv_csv)\n",
    "        train_ouv_df = ouv_df[ouv_df.split=='train']\n",
    "        return cls(ouv_df, OuvVectorizer.from_dataframe(train_ouv_df,ngrams=ngrams, cutoff=cutoff))\n",
    "    \n",
    "    @classmethod\n",
    "    def load_dataset_and_load_vectorizer(cls, ouv_csv, vectorizer_filepath, ngrams, vectorizer):\n",
    "        \"\"\"Load dataset and the corresponding vectorizer. \n",
    "        Used in the case in the vectorizer has been cached for re-use\n",
    "        \n",
    "        Args:\n",
    "            ouv_csv (str): location of the dataset\n",
    "            vectorizer_filepath (str): location of the saved vectorizer\n",
    "        Returns:\n",
    "            an instance of OuvDataset\n",
    "        \"\"\"\n",
    "        ouv_df = pd.read_csv(ouv_csv)\n",
    "        vectorizer = cls.load_vectorizer_only(vectorizer_filepath, ngrams=ngrams, vectorizer=vectorizer)\n",
    "        return cls(ouv_df, vectorizer)\n",
    "\n",
    "    @staticmethod\n",
    "    def load_vectorizer_only(vectorizer_filepath,ngrams, vectorizer):\n",
    "        \"\"\"a static method for loading the vectorizer from file\n",
    "        \n",
    "        Args:\n",
    "            vectorizer_filepath (str): the location of the serialized vectorizer\n",
    "        Returns:\n",
    "            an instance of ReviewVectorizer\n",
    "        \"\"\"\n",
    "        with open(vectorizer_filepath) as fp:\n",
    "            return OuvVectorizer.from_serializable(json.load(fp),ngrams=ngrams, vectorizer=vectorizer)\n",
    "\n",
    "    def save_vectorizer(self, vectorizer_filepath):\n",
    "        \"\"\"saves the vectorizer to disk using json\n",
    "        \n",
    "        Args:\n",
    "            vectorizer_filepath (str): the location to save the vectorizer\n",
    "        \"\"\"\n",
    "        with open(vectorizer_filepath, \"w\") as fp:\n",
    "            json.dump(self._vectorizer.to_serializable(), fp)\n",
    "\n",
    "    def get_vectorizer(self):\n",
    "        \"\"\" returns the vectorizer \"\"\"\n",
    "        return self._vectorizer\n",
    "\n",
    "    def set_split(self, split=\"train\"):\n",
    "        \"\"\" selects the splits in the dataset using a column in the dataframe \n",
    "        \n",
    "        Args:\n",
    "            split (str): one of \"train\", \"val\", or \"test\"\n",
    "        \"\"\"\n",
    "        self._target_split = split\n",
    "        self._target_df, self._target_size = self._lookup_dict[split]\n",
    "\n",
    "    def __len__(self):\n",
    "        return self._target_size\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        \"\"\"the primary entry point method for PyTorch datasets\n",
    "        \n",
    "        Args:\n",
    "            index (int): the index to the data point \n",
    "        Returns:\n",
    "            a dictionary holding the data point's features (x_data) and component for labels (y_target and y_fuzzy)\n",
    "        \"\"\"\n",
    "        row = self._target_df.iloc[index]\n",
    "\n",
    "        ouv_vector = \\\n",
    "            self._vectorizer.vectorize(row.data)\n",
    "\n",
    "        true_label = \\\n",
    "            np.fromstring(row.true[1:-1],dtype=float, sep=' ')\n",
    "        if len(true_label)==10:\n",
    "            true_label = np.append(true_label,0.0)\n",
    "        \n",
    "        fuzzy_label = \\\n",
    "            np.fromstring(row.fuzzy[1:-1],dtype=float, sep=' ')\n",
    "\n",
    "        return {'x_data': ouv_vector,\n",
    "                'y_target': true_label,\n",
    "                'y_fuzzy': fuzzy_label\n",
    "               }\n",
    "\n",
    "    def get_num_batches(self, batch_size):\n",
    "        \"\"\"Given a batch size, return the number of batches in the dataset\n",
    "        \n",
    "        Args:\n",
    "            batch_size (int)\n",
    "        Returns:\n",
    "            number of batches in the dataset\n",
    "        \"\"\"\n",
    "        return len(self) // batch_size  \n",
    "    \n",
    "def generate_batches(dataset, batch_size, shuffle=True,\n",
    "                     drop_last=True, device=\"cpu\"):\n",
    "    \"\"\"\n",
    "    A generator function which wraps the PyTorch DataLoader. It will \n",
    "      ensure each tensor is on the write device location.\n",
    "    \"\"\"\n",
    "    dataloader = DataLoader(dataset=dataset, batch_size=batch_size,\n",
    "                            shuffle=shuffle, drop_last=drop_last)\n",
    "\n",
    "    for data_dict in dataloader:\n",
    "        out_data_dict = {}\n",
    "        for name, tensor in data_dict.items():\n",
    "            out_data_dict[name] = data_dict[name].to(device)\n",
    "        yield out_data_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The Model: Naive_Bayers_Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MLPClassifier(nn.Module):\n",
    "    \n",
    "    def __init__(self, embedding_size, hidden_dim, num_classes, dropout_p, \n",
    "                 pretrained_embeddings=None, padding_idx=0):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            embedding_size (int): size of the embedding vectors\n",
    "            num_embeddings (int): number of embedding vectors\n",
    "            hidden_dim (int): the size of the hidden dimension\n",
    "            num_classes (int): the number of classes in classification\n",
    "            dropout_p (float): a dropout parameter \n",
    "            pretrained_embeddings (numpy.array): previously trained word embeddings\n",
    "                default is None. If provided, \n",
    "            padding_idx (int): an index representing a null position\n",
    "        \"\"\"\n",
    "        super(MLPClassifier, self).__init__()\n",
    "\n",
    "        self._dropout_p = dropout_p\n",
    "        self.dropout = nn.Dropout(dropout_p)\n",
    "        self.fc1 = nn.Linear(embedding_size, hidden_dim)\n",
    "        self.fc2 = nn.Linear(hidden_dim, num_classes)\n",
    "\n",
    "    def forward(self, x_in, apply_softmax=False):\n",
    "        \"\"\"The forward pass of the classifier\n",
    "        \n",
    "        Args:\n",
    "            x_in (torch.Tensor): an input data tensor. \n",
    "                x_in.shape should be (batch, dataset._max_seq_length)\n",
    "            apply_softmax (bool): a flag for the softmax activation\n",
    "                should be false if used with the Cross Entropy losses\n",
    "        Returns:\n",
    "            the resulting tensor. tensor.shape should be (batch, num_classes)\n",
    "        \"\"\"\n",
    "        \n",
    "        intermediate_vector = F.relu(self.dropout(self.fc1(x_in)))\n",
    "        prediction_vector = self.fc2(intermediate_vector)\n",
    "        \n",
    "        if apply_softmax:\n",
    "            prediction_vector = F.softmax(prediction_vector, dim=1)\n",
    "\n",
    "        return prediction_vector"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Routine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_train_state(args):\n",
    "    return {'stop_early': False,\n",
    "            'early_stopping_step': 0,\n",
    "            'early_stopping_best_k_acc_val': 0,\n",
    "            'learning_rate': args.learning_rate,\n",
    "            'epoch_index': 0,\n",
    "            'train_loss': [],\n",
    "            'train_1_acc': [],\n",
    "            'train_k_acc': [],\n",
    "            'train_k_jac': [],\n",
    "            'val_loss': [],\n",
    "            'val_1_acc': [],\n",
    "            'val_k_acc': [],\n",
    "            'val_k_jac': [],\n",
    "            'test_loss': -1,\n",
    "            'test_1_acc': -1,\n",
    "            'test_k_acc':-1,\n",
    "            'test_k_jac':-1,\n",
    "            'model_filename': args.model_state_file}\n",
    "\n",
    "def update_train_state(args, model, train_state):\n",
    "    \"\"\"Handle the training state updates.\n",
    "\n",
    "    Components:\n",
    "     - Early Stopping: Prevent overfitting.\n",
    "     - Model Checkpoint: Model is saved if the model is better\n",
    "\n",
    "    :param args: main arguments\n",
    "    :param model: model to train\n",
    "    :param train_state: a dictionary representing the training state values\n",
    "    :returns:\n",
    "        a new train_state\n",
    "    \"\"\"\n",
    "\n",
    "    # Save one model at least\n",
    "    if train_state['epoch_index'] == 0:\n",
    "        torch.save(model.state_dict(), train_state['model_filename'])\n",
    "        train_state['stop_early'] = False\n",
    "\n",
    "    # Save model if performance improved\n",
    "    elif train_state['epoch_index'] >= 1:\n",
    "        acc_tm1, acc_t = train_state['val_k_acc'][-2:]\n",
    "\n",
    "        # If accuracy worsened\n",
    "        if acc_t <= train_state['early_stopping_best_k_acc_val']:\n",
    "            # Update step\n",
    "            train_state['early_stopping_step'] += 1\n",
    "        # Loss decreased\n",
    "        else:\n",
    "            # Save the best model from sklearn\n",
    "            if acc_t > train_state['early_stopping_best_k_acc_val']:\n",
    "                train_state['early_stopping_best_k_acc_val'] = acc_t\n",
    "                torch.save(model.state_dict(), train_state['model_filename'])\n",
    "                \n",
    "            # Reset early stopping step\n",
    "            train_state['early_stopping_step'] = 0\n",
    "\n",
    "        # Stop early ?\n",
    "        train_state['stop_early'] = \\\n",
    "            train_state['early_stopping_step'] >= args.early_stopping_criteria\n",
    "\n",
    "    return train_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_cross_entropy(y_pred, y_target):\n",
    "    y_target = y_target.cpu().float()\n",
    "    y_pred = y_pred.cpu().float()\n",
    "    criterion = nn.BCEWithLogitsLoss()\n",
    "    return criterion(y_target, y_pred)\n",
    "\n",
    "def compute_1_accuracy(y_pred, y_target):\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    n_correct = torch.eq(y_pred_indices, y_target_indices).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_accuracy(y_pred, y_target, k=3):\n",
    "    y_pred_indices = y_pred.topk(k, dim=1)[1]\n",
    "    y_target_indices = y_target.max(dim=1)[1]\n",
    "    n_correct = torch.tensor([y_pred_indices[i] in y_target_indices[i] for i in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_k_jaccard_index(y_pred, y_target, k=3):\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "    y_pred_indices = y_pred.max(dim=1)[1]\n",
    "    jaccard = torch.tensor([len(np.intersect1d(y_target_indices[i], y_pred_indices[i]))/\n",
    "                            len(np.union1d(y_target_indices[i], y_pred_indices[i]))\n",
    "                            for i in range(len(y_pred))]).sum().item()\n",
    "    return jaccard / len(y_pred_indices)\n",
    "\n",
    "def compute_jaccard_index(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    threshold = 1.0/(k+1)\n",
    "    threshold_2 = 0.5\n",
    "    \n",
    "    if multilabel:\n",
    "        y_pred_indices = y_pred.gt(threshold_2)\n",
    "    else:\n",
    "        y_pred_indices = y_pred.gt(threshold)\n",
    "    \n",
    "    y_target_indices = y_target.gt(threshold)\n",
    "        \n",
    "    jaccard = ((y_target_indices*y_pred_indices).sum(axis=1)/((y_target_indices+y_pred_indices).sum(axis=1)+1e-8)).sum().item()\n",
    "    return jaccard / len(y_pred_indices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax_sensitive(T):\n",
    "    T = np.exp(T) - np.exp(0) + 1e-9\n",
    "    if len(T.shape)==1:\n",
    "        return T/T.sum()\n",
    "    return  T/(T.sum(axis=1).unsqueeze(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cross_entropy(pred, soft_targets):\n",
    "    logsoftmax = nn.LogSoftmax(dim=1)\n",
    "    return torch.mean(torch.sum(- soft_targets * logsoftmax(pred), 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert a df to tensor to be used in pytorch\n",
    "def df_to_tensor(df):\n",
    "    device = args.device\n",
    "    return torch.from_numpy(df.values).float().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_prior():\n",
    "    prior = pd.read_csv(args.prior_csv,sep=';',names=classes[:-1], skiprows=1)\n",
    "    prior['Others'] = 1\n",
    "    prior = prior.T\n",
    "    prior['Others'] = 1\n",
    "    prior = df_to_tensor(prior)\n",
    "    return prior"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_fuzzy_label(y_target, y_fuzzy, fuzzy=False, how='uni', lbd=0):\n",
    "    '''\n",
    "    Using two sets of prediction labels and fuzziness parameters to compute the fuzzy label in the form as \n",
    "    a distribution over classes\n",
    "    \n",
    "    Args:\n",
    "    y_target (torch.Tensor) of shape (n_batch, n_classes): the true label of the ouv description\n",
    "    y_fuzzy (torch.Tensor) of shape (n_batch, n_classes): the fuzzy label of the ouv description\n",
    "    fuzzy (bool): whether or not to turn on the fuzziness option\n",
    "    how (string): the way fuzziness weights are used, one of the options in {'uni', 'prior'}\n",
    "    lbd (float): the scaler applied to the fuzziness of the label\n",
    "    \n",
    "    Returns:\n",
    "    A pytorch Tensor of shape (n_batch, n_classes): The processed label in the form of distribution that add to 1\n",
    "    '''\n",
    "    assert y_target.shape == y_fuzzy.shape, 'target labels must have the same size'\n",
    "    assert how in {'uni', 'prior', 'origin'}, '''how must be one of the two options in {'uni', 'prior', 'origin'}'''\n",
    "    \n",
    "    if not fuzzy:\n",
    "        return softmax_sensitive(y_target)\n",
    "    \n",
    "    if how == 'uni':\n",
    "        y_label = y_target + lbd * y_fuzzy\n",
    "        return softmax_sensitive(y_label)\n",
    "    \n",
    "    ### TO DO ###\n",
    "    elif how == 'prior':\n",
    "        prior = get_prior()\n",
    "        y_inter = torch.matmul(y_target.float(),prior)\n",
    "        y_inter = y_inter/(y_inter.max(dim=1, keepdim=True)[0])\n",
    "        y_label = y_target + lbd * y_fuzzy * y_inter\n",
    "        return softmax_sensitive(y_label)\n",
    "    \n",
    "    else:\n",
    "        y_label = y_target + lbd\n",
    "        return softmax_sensitive(y_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sparse_to_tensor(M):\n",
    "    \"\"\"\n",
    "    input: M is Scipy sparse matrix\n",
    "    output: pytorch sparse tensor in GPU \n",
    "    \"\"\"\n",
    "    M = M.tocoo().astype(np.float32)\n",
    "    indices = torch.from_numpy(np.vstack((M.row, M.col))).long()\n",
    "    values = torch.from_numpy(M.data)\n",
    "    shape = torch.Size(M.shape)\n",
    "    Ms = torch.sparse.FloatTensor(indices, values, shape, device=args.device)\n",
    "    return Ms.to_dense()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### General Utilities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed_everywhere(seed, cuda):\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    if cuda:\n",
    "        torch.cuda.manual_seed_all(seed)\n",
    "\n",
    "def handle_dirs(dirpath):\n",
    "    if not os.path.exists(dirpath):\n",
    "        os.makedirs(dirpath)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Settings and Some Prep Work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Expanded filepaths: \n",
      "\tmodel_storage/ngram/vectorizer.json\n",
      "\tmodel_storage/ngram/model.pth\n",
      "Using CUDA: False\n"
     ]
    }
   ],
   "source": [
    "args = Namespace(\n",
    "    # Data and Path information\n",
    "    frequency_cutoff=1,\n",
    "    model_state_file='model.pth',\n",
    "    ouv_csv='Data/ouv_with_splits_full.csv',\n",
    "    #ouv_csv='Data/all_with_splits_full.csv',\n",
    "    prior_csv = 'Data/Coappearance_matrix.csv',\n",
    "    save_dir='model_storage/ngram/',\n",
    "    vectorizer_file='vectorizer.json',\n",
    "    # Model hyper parameters\n",
    "    ngrams=2,\n",
    "    hidden_dim=200, \n",
    "    # Training hyper parameters\n",
    "    batch_size=128,\n",
    "    early_stopping_criteria=5,\n",
    "    learning_rate=0.0002,\n",
    "    l2 = 1e-5,\n",
    "    dropout_p=0.5,\n",
    "    k = 3,\n",
    "    fuzzy = True,\n",
    "    fuzzy_how = 'uni',\n",
    "    fuzzy_lambda = 0.1,\n",
    "    num_epochs=100,\n",
    "    seed=1337,\n",
    "    # Runtime options\n",
    "    catch_keyboard_interrupt=True,\n",
    "    cuda=True,\n",
    "    expand_filepaths_to_save_dir=True,\n",
    "    reload_from_files=False,\n",
    ")\n",
    "\n",
    "classes = ['Criteria i', 'Criteria ii', 'Criteria iii', 'Criteria iv', 'Criteria v', 'Criteria vi', \n",
    "              'Criteria vii', 'Criteria viii', 'Criteria ix', 'Criteria x', 'Others']\n",
    "\n",
    "if args.expand_filepaths_to_save_dir:\n",
    "    args.vectorizer_file = os.path.join(args.save_dir,\n",
    "                                        args.vectorizer_file)\n",
    "\n",
    "    args.model_state_file = os.path.join(args.save_dir,\n",
    "                                         args.model_state_file)\n",
    "    \n",
    "    print(\"Expanded filepaths: \")\n",
    "    print(\"\\t{}\".format(args.vectorizer_file))\n",
    "    print(\"\\t{}\".format(args.model_state_file))\n",
    "    \n",
    "# Check CUDA\n",
    "if not torch.cuda.is_available():\n",
    "    args.cuda = False\n",
    "\n",
    "print(\"Using CUDA: {}\".format(args.cuda))\n",
    "\n",
    "args.device = torch.device(\"cuda\" if args.cuda else \"cpu\")\n",
    "\n",
    "# Set seed for reproducibility\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "\n",
    "# handle dirs\n",
    "handle_dirs(args.save_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/miniconda3/lib/python3.7/site-packages/sklearn/feature_extraction/text.py:1799: UserWarning: Only (<class 'numpy.float64'>, <class 'numpy.float32'>, <class 'numpy.float16'>) 'dtype' should be used. int32 'dtype' will be converted to np.float64.\n",
      "  UserWarning)\n"
     ]
    }
   ],
   "source": [
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "if args.reload_from_files:\n",
    "    # training from a checkpoint\n",
    "    dataset = OuvDataset.load_dataset_and_load_vectorizer(args.ouv_csv, args.vectorizer_file)\n",
    "\n",
    "else:\n",
    "    # create dataset and vectorizer\n",
    "    dataset = OuvDataset.load_dataset_and_make_vectorizer(args.ouv_csv, \n",
    "                                                          cutoff=args.frequency_cutoff, ngrams=args.ngrams)\n",
    "    dataset.save_vectorizer(args.vectorizer_file)    \n",
    "\n",
    "vectorizer = dataset.get_vectorizer()\n",
    "\n",
    "embedding_size = len(vectorizer.vectorizer.vocabulary_)\n",
    "\n",
    "classifier = MLPClassifier(embedding_size=embedding_size, \n",
    "                            hidden_dim=args.hidden_dim, \n",
    "                            num_classes=len(classes), \n",
    "                            dropout_p=args.dropout_p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19081"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "with profiler.profile(record_shapes=True) as prof:\n",
    "    with profiler.record_function(\"model_inference\"):\n",
    "        classifier(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
      "                   Name    Self CPU %      Self CPU   CPU total %     CPU total  CPU time avg    # of Calls  \n",
      "-----------------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
      "        model_inference         6.54%     224.467us        99.33%       3.408ms       3.408ms             1  \n",
      "            aten::addmm        81.37%       2.792ms        83.10%       2.852ms       1.426ms             2  \n",
      "          aten::dropout         0.48%      16.322us         7.10%     243.634us     243.634us             1  \n",
      "       aten::bernoulli_         3.59%     123.054us         3.63%     124.419us     124.419us             1  \n",
      "                aten::t         1.16%      39.824us         1.70%      58.427us      29.214us             2  \n",
      "             aten::div_         0.80%      27.575us         1.33%      45.740us      45.740us             1  \n",
      "              aten::mul         1.13%      38.764us         1.19%      40.791us      40.791us             1  \n",
      "            aten::copy_         1.08%      36.901us         1.15%      39.359us      13.120us             3  \n",
      "            aten::empty         1.00%      34.303us         1.00%      34.303us       4.288us             8  \n",
      "             aten::relu         0.43%      14.880us         0.78%      26.640us      26.640us             1  \n",
      "-----------------------  ------------  ------------  ------------  ------------  ------------  ------------  \n",
      "Self CPU time total: 3.431ms\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prof.key_averages().table(sort_by=\"cpu_time_total\", row_limit=10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Trained Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option 1 LS Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'hyperdict_fuzzy.p', 'rb') as fp:\n",
    "    hyperdict_fuzzy = pickle.load(fp)\n",
    "    train_state = hyperdict_fuzzy[('uni',0.1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLPClassifier(\n",
       "  (dropout): Dropout(p=0.5, inplace=False)\n",
       "  (fc1): Linear(in_features=19081, out_features=200, bias=True)\n",
       "  (fc2): Linear(in_features=200, out_features=11, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.load_state_dict(torch.load(args.save_dir+'1337/model.pth',map_location=torch.device('cpu')))\n",
    "classifier.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Option 2 Baseline w/o LS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 402,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(args.save_dir+'hyperdict_fuzzy.p', 'rb') as fp:\n",
    "    hyperdict_fuzzy = pickle.load(fp)\n",
    "    train_state = hyperdict_fuzzy[('uni',0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 403,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MLPClassifier(\n",
       "  (dropout): Dropout(p=0.5, inplace=False)\n",
       "  (fc1): Linear(in_features=19081, out_features=200, bias=True)\n",
       "  (fc2): Linear(in_features=200, out_features=11, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 403,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classifier.load_state_dict(torch.load(args.save_dir+'baseline/model.pth',map_location=torch.device('cpu')))\n",
    "classifier.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3818611"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "count_parameters(classifier)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the loss & accuracy on the test set using the best available model\n",
    "loss_func = cross_entropy\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "\n",
    "dataset.set_split('test')\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "running_loss = 0.\n",
    "running_1_acc = 0.\n",
    "running_k_acc = 0.\n",
    "running_k_jac = 0.\n",
    "classifier.eval()\n",
    "\n",
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    \n",
    "    # get the data compute fuzzy labels\n",
    "    X = batch_dict['x_data']\n",
    "\n",
    "    y_target = batch_dict['y_target']\n",
    "    y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "    # compute the output\n",
    "    with torch.no_grad():\n",
    "        y_pred = classifier(X)\n",
    "\n",
    "    # compute the loss\n",
    "    loss = loss_func(y_pred, Y)\n",
    "    loss_t = loss.item()\n",
    "    running_loss += (loss_t - running_loss) / (batch_index + 1)\n",
    "\n",
    "    # compute the accuracy\n",
    "    acc_1_t = compute_1_accuracy(y_pred, y_target)\n",
    "    acc_k_t = compute_k_accuracy(y_pred, y_target, args.k)\n",
    "    jac_k_t = compute_jaccard_index(y_pred, y_target, args.k)\n",
    "\n",
    "    running_1_acc += (acc_1_t - running_1_acc) / (batch_index + 1)\n",
    "    running_k_acc += (acc_k_t - running_k_acc) / (batch_index + 1)\n",
    "    running_k_jac += (jac_k_t - running_k_jac) / (batch_index + 1)\n",
    "\n",
    "train_state['test_loss'] = running_loss\n",
    "train_state['test_1_acc'] = running_1_acc\n",
    "train_state['test_k_acc'] = running_k_acc\n",
    "train_state['test_k_jac'] = running_k_jac"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 392,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': True,\n",
       " 'early_stopping_step': 5,\n",
       " 'early_stopping_best_k_acc_val': 91.2109375,\n",
       " 'learning_rate': 0.0002,\n",
       " 'epoch_index': 32,\n",
       " 'train_loss': [2.382710953165854,\n",
       "  2.3501047541321927,\n",
       "  2.2933758742039005,\n",
       "  2.2139634354483335,\n",
       "  2.1189518137611283,\n",
       "  2.015816979199149,\n",
       "  1.917336219988604,\n",
       "  1.8204204257401342,\n",
       "  1.725383685584271,\n",
       "  1.6366511834831206,\n",
       "  1.556673681326131,\n",
       "  1.472466073606517,\n",
       "  1.404638742030301,\n",
       "  1.3314902436662017,\n",
       "  1.2691519966329905,\n",
       "  1.206038874454092,\n",
       "  1.1506658358703707,\n",
       "  1.1005675252263447,\n",
       "  1.0470785168565697,\n",
       "  1.0026070142350851,\n",
       "  0.960046100542755,\n",
       "  0.9264518535723673,\n",
       "  0.8872131928046337,\n",
       "  0.8561477694927272,\n",
       "  0.8252839871102317,\n",
       "  0.7967185736870548,\n",
       "  0.7708332256268895,\n",
       "  0.7518785079131212,\n",
       "  0.7276363425709572,\n",
       "  0.7088139245532687,\n",
       "  0.6894438974322717,\n",
       "  0.6836571894768502,\n",
       "  0.6769948920907899],\n",
       " 'train_1_acc': [14.464285714285715,\n",
       "  17.142857142857142,\n",
       "  35.982142857142854,\n",
       "  45.22321428571428,\n",
       "  44.55357142857144,\n",
       "  44.598214285714285,\n",
       "  45.91517857142857,\n",
       "  50.379464285714285,\n",
       "  54.64285714285714,\n",
       "  59.30803571428571,\n",
       "  63.30357142857142,\n",
       "  68.30357142857146,\n",
       "  71.20535714285715,\n",
       "  74.15178571428574,\n",
       "  78.16964285714285,\n",
       "  80.04464285714286,\n",
       "  83.28125000000001,\n",
       "  84.50892857142856,\n",
       "  86.71875,\n",
       "  87.74553571428571,\n",
       "  89.375,\n",
       "  90.08928571428571,\n",
       "  91.58482142857144,\n",
       "  92.32142857142857,\n",
       "  93.19196428571429,\n",
       "  93.81696428571429,\n",
       "  94.50892857142858,\n",
       "  94.30803571428571,\n",
       "  95.37946428571426,\n",
       "  95.82589285714285,\n",
       "  96.62946428571428,\n",
       "  96.3392857142857,\n",
       "  96.69642857142854],\n",
       " 'train_k_acc': [35.06696428571428,\n",
       "  59.776785714285715,\n",
       "  62.410714285714285,\n",
       "  62.25446428571428,\n",
       "  65.00000000000001,\n",
       "  68.03571428571429,\n",
       "  71.4732142857143,\n",
       "  74.6875,\n",
       "  77.74553571428572,\n",
       "  81.07142857142857,\n",
       "  83.86160714285715,\n",
       "  87.79017857142858,\n",
       "  90.71428571428574,\n",
       "  92.61160714285715,\n",
       "  94.39732142857143,\n",
       "  96.04910714285714,\n",
       "  96.74107142857143,\n",
       "  97.63392857142858,\n",
       "  97.94642857142857,\n",
       "  98.70535714285715,\n",
       "  98.88392857142857,\n",
       "  99.24107142857143,\n",
       "  99.39732142857142,\n",
       "  99.66517857142858,\n",
       "  99.53124999999997,\n",
       "  99.64285714285715,\n",
       "  99.75446428571428,\n",
       "  99.84374999999997,\n",
       "  99.88839285714285,\n",
       "  99.79910714285714,\n",
       "  99.88839285714286,\n",
       "  99.88839285714288,\n",
       "  99.86607142857143],\n",
       " 'train_k_jac': [0.029910714285714287,\n",
       "  0.17371651998588017,\n",
       "  0.24218750979219167,\n",
       "  0.23267857943262374,\n",
       "  0.2335034136261259,\n",
       "  0.24313882631914951,\n",
       "  0.2577482019151961,\n",
       "  0.2747199296951294,\n",
       "  0.2851153424807957,\n",
       "  0.2949181650366102,\n",
       "  0.30268230353082926,\n",
       "  0.3046912338052477,\n",
       "  0.31046504122870305,\n",
       "  0.31397322331156047,\n",
       "  0.3173995648111615,\n",
       "  0.3224553661687033,\n",
       "  0.32888393657548076,\n",
       "  0.336875011239733,\n",
       "  0.34369421260697486,\n",
       "  0.3501078946249826,\n",
       "  0.3564211402620588,\n",
       "  0.36172619632312225,\n",
       "  0.36594867110252366,\n",
       "  0.3691220351627896,\n",
       "  0.379215042080198,\n",
       "  0.38420388187680926,\n",
       "  0.387939006941659,\n",
       "  0.3914062636239188,\n",
       "  0.3941369209970746,\n",
       "  0.4018712971891675,\n",
       "  0.40577381678989943,\n",
       "  0.4058221885136197,\n",
       "  0.40712426475116187],\n",
       " 'val_loss': [2.3740046571475357,\n",
       "  2.3356895547644547,\n",
       "  2.276732363664654,\n",
       "  2.2049634395923574,\n",
       "  2.118362051482241,\n",
       "  2.039938690502814,\n",
       "  1.9641622931637537,\n",
       "  1.8950441405169305,\n",
       "  1.8280354859148231,\n",
       "  1.761112046311037,\n",
       "  1.6823259767869072,\n",
       "  1.6555813113506523,\n",
       "  1.610288260098908,\n",
       "  1.5616392093222504,\n",
       "  1.5205270177621062,\n",
       "  1.487478349760449,\n",
       "  1.444392173059423,\n",
       "  1.4156991257813978,\n",
       "  1.3850257873989436,\n",
       "  1.3650875576406498,\n",
       "  1.3547780103419684,\n",
       "  1.3292809103446255,\n",
       "  1.3193960852246587,\n",
       "  1.2966049810232019,\n",
       "  1.274533575541608,\n",
       "  1.2511274484036017,\n",
       "  1.261535050335143,\n",
       "  1.2320019969770653,\n",
       "  1.2354885529879072,\n",
       "  1.2437077639862433,\n",
       "  1.2459560145194486,\n",
       "  1.2323465905531468,\n",
       "  1.2362594065073182],\n",
       " 'val_1_acc': [13.8671875,\n",
       "  23.2421875,\n",
       "  38.28125,\n",
       "  38.671875,\n",
       "  36.71875,\n",
       "  37.109375,\n",
       "  39.0625,\n",
       "  40.8203125,\n",
       "  45.5078125,\n",
       "  48.2421875,\n",
       "  52.734375,\n",
       "  53.90625,\n",
       "  56.25,\n",
       "  58.0078125,\n",
       "  58.984375,\n",
       "  58.984375,\n",
       "  61.71875,\n",
       "  62.109375,\n",
       "  63.8671875,\n",
       "  64.453125,\n",
       "  64.84375,\n",
       "  65.0390625,\n",
       "  65.0390625,\n",
       "  66.6015625,\n",
       "  66.9921875,\n",
       "  66.6015625,\n",
       "  66.2109375,\n",
       "  67.1875,\n",
       "  64.84375,\n",
       "  64.453125,\n",
       "  64.453125,\n",
       "  65.625,\n",
       "  65.0390625],\n",
       " 'val_k_acc': [40.0390625,\n",
       "  58.3984375,\n",
       "  58.203125,\n",
       "  58.59375,\n",
       "  61.9140625,\n",
       "  64.6484375,\n",
       "  66.9921875,\n",
       "  68.9453125,\n",
       "  69.7265625,\n",
       "  72.65625,\n",
       "  76.171875,\n",
       "  76.3671875,\n",
       "  78.515625,\n",
       "  80.46875,\n",
       "  83.0078125,\n",
       "  84.5703125,\n",
       "  86.5234375,\n",
       "  85.9375,\n",
       "  87.109375,\n",
       "  87.890625,\n",
       "  88.28125,\n",
       "  88.671875,\n",
       "  89.6484375,\n",
       "  89.453125,\n",
       "  90.625,\n",
       "  90.625,\n",
       "  89.84375,\n",
       "  91.2109375,\n",
       "  90.234375,\n",
       "  89.84375,\n",
       "  90.4296875,\n",
       "  90.4296875,\n",
       "  89.453125],\n",
       " 'val_k_jac': [0.130859375,\n",
       "  0.2109375037252903,\n",
       "  0.2275390736758709,\n",
       "  0.20628256350755692,\n",
       "  0.2108910046517849,\n",
       "  0.22278646379709244,\n",
       "  0.24189453572034836,\n",
       "  0.25192058086395264,\n",
       "  0.2537434957921505,\n",
       "  0.2596028745174408,\n",
       "  0.2784830778837204,\n",
       "  0.27281901240348816,\n",
       "  0.28079429268836975,\n",
       "  0.2767578288912773,\n",
       "  0.28404948115348816,\n",
       "  0.2873698025941849,\n",
       "  0.30380861461162567,\n",
       "  0.30227865278720856,\n",
       "  0.3101237043738365,\n",
       "  0.3172200545668602,\n",
       "  0.3207356929779053,\n",
       "  0.32109376043081284,\n",
       "  0.32880860567092896,\n",
       "  0.33444012701511383,\n",
       "  0.33899740129709244,\n",
       "  0.34257815033197403,\n",
       "  0.3405599147081375,\n",
       "  0.343912772834301,\n",
       "  0.3418945372104645,\n",
       "  0.34322917461395264,\n",
       "  0.34703777730464935,\n",
       "  0.3475911468267441,\n",
       "  0.3440755307674408],\n",
       " 'test_loss': 1.2922605559424614,\n",
       " 'test_1_acc': 59.5703125,\n",
       " 'test_k_acc': 89.6484375,\n",
       " 'test_k_jac': 0.3485351800918579,\n",
       " 'model_filename': 'model_storage/ngram/multigroup/model.pth'}"
      ]
     },
     "execution_count": 392,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Result of LS Model\n",
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': True,\n",
       " 'early_stopping_step': 5,\n",
       " 'early_stopping_best_k_acc_val': 90.8203125,\n",
       " 'learning_rate': 0.0002,\n",
       " 'epoch_index': 32,\n",
       " 'train_loss': [2.3822073093172866,\n",
       "  2.3483752532384057,\n",
       "  2.2898766432460684,\n",
       "  2.2077932670707567,\n",
       "  2.1081109083167813,\n",
       "  1.9974981981136184,\n",
       "  1.8890756032591272,\n",
       "  1.7814928880306273,\n",
       "  1.6739859497765983,\n",
       "  1.5714231084477355,\n",
       "  1.4770863319759284,\n",
       "  1.3769292246986862,\n",
       "  1.2949315339131129,\n",
       "  1.2054808567138633,\n",
       "  1.1291811494103217,\n",
       "  1.050834673680587,\n",
       "  0.9816362487981773,\n",
       "  0.9199762458194751,\n",
       "  0.8522468257719096,\n",
       "  0.7965927294180679,\n",
       "  0.7427864249104039,\n",
       "  0.6992514694804647,\n",
       "  0.6490499609507968,\n",
       "  0.608924275879578,\n",
       "  0.5695455983635607,\n",
       "  0.5315245828531986,\n",
       "  0.49746133108882906,\n",
       "  0.47098072782707195,\n",
       "  0.43962891003744736,\n",
       "  0.413331593465177,\n",
       "  0.3880896365500506,\n",
       "  0.37878601054829625,\n",
       "  0.36913309914749876],\n",
       " 'train_1_acc': [14.464285714285715,\n",
       "  17.366071428571427,\n",
       "  36.54017857142858,\n",
       "  46.339285714285715,\n",
       "  45.80357142857144,\n",
       "  45.624999999999986,\n",
       "  46.85267857142857,\n",
       "  51.22767857142858,\n",
       "  55.62499999999999,\n",
       "  60.491071428571416,\n",
       "  64.39732142857143,\n",
       "  69.50892857142857,\n",
       "  72.1651785714286,\n",
       "  74.91071428571429,\n",
       "  79.17410714285717,\n",
       "  80.78125,\n",
       "  83.61607142857143,\n",
       "  85.00000000000003,\n",
       "  87.00892857142857,\n",
       "  87.9017857142857,\n",
       "  89.62053571428574,\n",
       "  90.24553571428571,\n",
       "  91.49553571428571,\n",
       "  92.32142857142857,\n",
       "  93.08035714285715,\n",
       "  93.59375,\n",
       "  94.44196428571429,\n",
       "  94.24107142857144,\n",
       "  95.20089285714285,\n",
       "  95.53571428571429,\n",
       "  96.42857142857143,\n",
       "  96.18303571428571,\n",
       "  96.5401785714286],\n",
       " 'train_k_acc': [35.0892857142857,\n",
       "  59.75446428571428,\n",
       "  63.14732142857142,\n",
       "  62.54464285714286,\n",
       "  64.95535714285712,\n",
       "  67.41071428571428,\n",
       "  71.13839285714289,\n",
       "  74.6875,\n",
       "  78.05803571428572,\n",
       "  81.98660714285714,\n",
       "  85.0,\n",
       "  88.6607142857143,\n",
       "  91.40625000000001,\n",
       "  93.16964285714286,\n",
       "  94.64285714285715,\n",
       "  96.27232142857143,\n",
       "  96.96428571428571,\n",
       "  97.58928571428572,\n",
       "  98.05803571428571,\n",
       "  98.61607142857146,\n",
       "  98.86160714285714,\n",
       "  99.24107142857143,\n",
       "  99.41964285714285,\n",
       "  99.55357142857146,\n",
       "  99.50892857142856,\n",
       "  99.66517857142856,\n",
       "  99.73214285714288,\n",
       "  99.86607142857142,\n",
       "  99.82142857142856,\n",
       "  99.79910714285714,\n",
       "  99.88839285714286,\n",
       "  99.88839285714288,\n",
       "  99.82142857142856],\n",
       " 'train_k_jac': [0.025669642857142856,\n",
       "  0.17202381278787343,\n",
       "  0.2504315563610621,\n",
       "  0.23581102022102898,\n",
       "  0.23407845411981856,\n",
       "  0.2415747327463967,\n",
       "  0.2581016280821391,\n",
       "  0.27870058289596006,\n",
       "  0.2926302186080388,\n",
       "  0.3054560031209674,\n",
       "  0.3163318565913608,\n",
       "  0.32280507258006497,\n",
       "  0.32811756900378647,\n",
       "  0.3347991202558791,\n",
       "  0.34011533686092915,\n",
       "  0.34481771758624485,\n",
       "  0.35308036974498197,\n",
       "  0.36369792648724136,\n",
       "  0.3718378075531551,\n",
       "  0.37792039598737454,\n",
       "  0.3835714382784706,\n",
       "  0.39029763000352047,\n",
       "  0.3959561109542846,\n",
       "  0.39786087529999864,\n",
       "  0.408281260728836,\n",
       "  0.41591147269521433,\n",
       "  0.42022694860185894,\n",
       "  0.42375000800405227,\n",
       "  0.4259449550083706,\n",
       "  0.43533483658518113,\n",
       "  0.43881326062338694,\n",
       "  0.4406250144754138,\n",
       "  0.4449665299483708],\n",
       " 'val_loss': [2.3732982919808565,\n",
       "  2.3341410155039064,\n",
       "  2.2740343632587234,\n",
       "  2.2005690225972145,\n",
       "  2.1095736676315435,\n",
       "  2.025941343956679,\n",
       "  1.9434757097304416,\n",
       "  1.8696733544886426,\n",
       "  1.7922418712929722,\n",
       "  1.7167264135239886,\n",
       "  1.625244409823854,\n",
       "  1.5944101432224196,\n",
       "  1.5388586693728887,\n",
       "  1.480677290756429,\n",
       "  1.4316574686172245,\n",
       "  1.389944751141797,\n",
       "  1.3362726552920936,\n",
       "  1.3010356352428623,\n",
       "  1.2622163979347316,\n",
       "  1.2378541618166554,\n",
       "  1.222104190573985,\n",
       "  1.195046635664932,\n",
       "  1.175989711422298,\n",
       "  1.1485100545265405,\n",
       "  1.1204237505937236,\n",
       "  1.0921874821313367,\n",
       "  1.1039747963443949,\n",
       "  1.0672489788133404,\n",
       "  1.0678343459629076,\n",
       "  1.0801070382030575,\n",
       "  1.0816601419330698,\n",
       "  1.063108785419998,\n",
       "  1.0653782412169683],\n",
       " 'val_1_acc': [13.8671875,\n",
       "  23.828125,\n",
       "  38.4765625,\n",
       "  39.2578125,\n",
       "  36.9140625,\n",
       "  37.109375,\n",
       "  39.2578125,\n",
       "  41.2109375,\n",
       "  45.5078125,\n",
       "  49.0234375,\n",
       "  53.3203125,\n",
       "  54.1015625,\n",
       "  55.6640625,\n",
       "  58.0078125,\n",
       "  58.7890625,\n",
       "  58.7890625,\n",
       "  61.328125,\n",
       "  61.71875,\n",
       "  64.2578125,\n",
       "  65.234375,\n",
       "  64.6484375,\n",
       "  65.0390625,\n",
       "  65.4296875,\n",
       "  66.796875,\n",
       "  67.3828125,\n",
       "  67.3828125,\n",
       "  66.796875,\n",
       "  67.3828125,\n",
       "  65.625,\n",
       "  64.84375,\n",
       "  64.0625,\n",
       "  65.8203125,\n",
       "  65.4296875],\n",
       " 'val_k_acc': [39.84375,\n",
       "  58.7890625,\n",
       "  58.203125,\n",
       "  58.984375,\n",
       "  61.1328125,\n",
       "  64.0625,\n",
       "  66.40625,\n",
       "  68.1640625,\n",
       "  69.7265625,\n",
       "  72.4609375,\n",
       "  76.7578125,\n",
       "  77.734375,\n",
       "  78.90625,\n",
       "  81.25,\n",
       "  83.203125,\n",
       "  85.546875,\n",
       "  86.5234375,\n",
       "  85.9375,\n",
       "  87.3046875,\n",
       "  88.0859375,\n",
       "  88.4765625,\n",
       "  88.28125,\n",
       "  89.2578125,\n",
       "  89.84375,\n",
       "  90.4296875,\n",
       "  90.4296875,\n",
       "  89.6484375,\n",
       "  90.8203125,\n",
       "  89.6484375,\n",
       "  89.84375,\n",
       "  89.84375,\n",
       "  89.84375,\n",
       "  89.453125],\n",
       " 'val_k_jac': [0.126953125,\n",
       "  0.225260429084301,\n",
       "  0.2314453236758709,\n",
       "  0.20846354961395264,\n",
       "  0.21297433599829674,\n",
       "  0.21979167684912682,\n",
       "  0.23977865651249886,\n",
       "  0.2533203214406967,\n",
       "  0.26608074456453323,\n",
       "  0.2714843824505806,\n",
       "  0.2898763120174408,\n",
       "  0.2871093824505806,\n",
       "  0.2947591245174408,\n",
       "  0.29749350249767303,\n",
       "  0.3010091260075569,\n",
       "  0.3057291880249977,\n",
       "  0.3198242262005806,\n",
       "  0.32376303523778915,\n",
       "  0.3295247480273247,\n",
       "  0.3388020992279053,\n",
       "  0.3440755382180214,\n",
       "  0.34046225994825363,\n",
       "  0.34990233927965164,\n",
       "  0.35582683980464935,\n",
       "  0.36363933235406876,\n",
       "  0.3645833358168602,\n",
       "  0.3599283993244171,\n",
       "  0.36552736163139343,\n",
       "  0.3637044355273247,\n",
       "  0.3650716319680214,\n",
       "  0.36819662898778915,\n",
       "  0.3709309995174408,\n",
       "  0.3687174469232559],\n",
       " 'test_loss': 1.2968807102569224,\n",
       " 'test_1_acc': 59.9609375,\n",
       " 'test_k_acc': 88.8671875,\n",
       " 'test_k_jac': 0.36910808086395264,\n",
       " 'model_filename': 'model_storage/ngram/multigroup/model.pth'}"
      ]
     },
     "execution_count": 405,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Result of Baseline\n",
    "train_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"([.,!?])\", r\" \\1 \", text)\n",
    "    text = re.sub(r\"[^a-zA-Z.,!?]+\", r\" \", text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_rating(text, classifier, vectorizer, classes, k=1):\n",
    "    \"\"\"Predict the rating of a review\n",
    "    \n",
    "    Args:\n",
    "        text (str): the text of the description\n",
    "        classifier (ReviewClassifier): the trained model\n",
    "        vectorizer (ReviewVectorizer): the corresponding vectorizer\n",
    "        classes (list of str): The name of the ouv classes\n",
    "        k (int): show the largest k prediction, default to 1\n",
    "    \"\"\"\n",
    "    classifier.eval()\n",
    "    ouv = preprocess_text(text)\n",
    "    vectorized_ouv = vectorizer.vectorize(ouv)\n",
    "    X = vectorized_ouv.view(1,-1)\n",
    "    with torch.no_grad():\n",
    "        result = classifier(vectorized_ouv.unsqueeze(0), apply_softmax=True)\n",
    "    \n",
    "    if k==1:\n",
    "        pred_id = result.argmax().item()\n",
    "        return (classes[pred_id], result[0][pred_id])\n",
    "    else:\n",
    "        pred_indices = [i.item() for i in result.topk(k)[1][0]]\n",
    "        output = []\n",
    "        for pred_id in pred_indices:\n",
    "            output.append((classes[pred_id],result[0][pred_id].item()))\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "this is a very old building dating back to 13th century -> Criteria iv with a probability of 0.31\n"
     ]
    }
   ],
   "source": [
    "test_ouv = 'this is a very old building dating back to 13th century'\n",
    "\n",
    "prediction = predict_rating(test_ouv,classifier,vectorizer,classes)\n",
    "print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "this is a very old building dating back to 13th century -> Criteria iv with a probability of 0.31\n",
      "this is a very old building dating back to 13th century -> Criteria iii with a probability of 0.26\n",
      "this is a very old building dating back to 13th century -> Criteria i with a probability of 0.09\n"
     ]
    }
   ],
   "source": [
    "test_ouv = 'this is a very old building dating back to 13th century'\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "The particular layout of the complex is unique to this site -> Criteria iv with a probability of 0.27\n",
      "The particular layout of the complex is unique to this site -> Criteria iii with a probability of 0.24\n",
      "The particular layout of the complex is unique to this site -> Criteria ii with a probability of 0.16\n"
     ]
    }
   ],
   "source": [
    "test_ouv = 'The particular layout of the complex is unique to this site'\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world from \n",
      "torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden \n",
      "age are represented by monuments of incomparable beauty -> Criteria i with a probability of 0.32\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world from \n",
      "torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden \n",
      "age are represented by monuments of incomparable beauty -> Criteria iv with a probability of 0.17\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world from \n",
      "torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden \n",
      "age are represented by monuments of incomparable beauty -> Criteria iii with a probability of 0.12\n"
     ]
    }
   ],
   "source": [
    "test_ouv = '''the lagoon of venice also has one of the highest concentrations of masterpieces in the world from \n",
    "torcellos cathedral to the church of santa maria della salute . the years of the republics extraordinary golden \n",
    "age are represented by monuments of incomparable beauty'''\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world -> Criteria i with a probability of 0.26\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world -> Criteria x with a probability of 0.13\n",
      "the lagoon of venice also has one of the highest concentrations of masterpieces in the world -> Criteria vii with a probability of 0.13\n"
     ]
    }
   ],
   "source": [
    "test_ouv = '''the lagoon of venice also has one of the highest concentrations of masterpieces in the world'''\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "from torcellos cathedral to the church of santa maria della salute -> Criteria iv with a probability of 0.28\n",
      "from torcellos cathedral to the church of santa maria della salute -> Criteria vi with a probability of 0.16\n",
      "from torcellos cathedral to the church of santa maria della salute -> Criteria ii with a probability of 0.15\n"
     ]
    }
   ],
   "source": [
    "test_ouv = '''from torcellos cathedral to the church of santa maria della salute'''\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top 3 predictions:\n",
      "===================\n",
      "the years of the republics extraordinary golden age are represented by monuments of incomparable beauty -> Criteria iii with a probability of 0.20\n",
      "the years of the republics extraordinary golden age are represented by monuments of incomparable beauty -> Criteria iv with a probability of 0.20\n",
      "the years of the republics extraordinary golden age are represented by monuments of incomparable beauty -> Criteria i with a probability of 0.16\n"
     ]
    }
   ],
   "source": [
    "test_ouv = '''the years of the republics extraordinary golden age are represented by monuments of incomparable beauty'''\n",
    "k=3\n",
    "predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k)\n",
    "\n",
    "print(\"Top {} predictions:\".format(k))\n",
    "print(\"===================\")\n",
    "for prediction in predictions:\n",
    "    print('{} -> {} with a probability of {:0.2f}'.format(test_ouv, prediction[0],prediction[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elapsed: 0.0027370452880859375\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "class Timer(object):\n",
    "    def __init__(self, name=None):\n",
    "        self.name = name\n",
    "\n",
    "    def __enter__(self):\n",
    "        self.tstart = time.time()\n",
    "\n",
    "    def __exit__(self, type, value, traceback):\n",
    "        if self.name:\n",
    "            print('[%s]' % self.name,)\n",
    "        print('Elapsed: %s' % (time.time() - self.tstart))\n",
    "        \n",
    "set_seed_everywhere(args.seed, args.cuda)        \n",
    "test_ouv = 'The particular layout of the complex is unique to this site'\n",
    "k=3\n",
    "with Timer():\n",
    "    predictions = predict_rating(test_ouv,classifier,vectorizer,classes,k=k)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Interpretability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "def infer_tokens_importance(vocab, classifier, vectorizer, classes, k=50):\n",
    "    \"\"\"Predict the rating of a review\n",
    "    \n",
    "    Args:\n",
    "        vocab (list of str): the whole vocabulary\n",
    "        classifier (ReviewClassifier): the trained model\n",
    "        vectorizer (ReviewVectorizer): the corresponding vectorizer\n",
    "        classes (list of str): The name of the ouv classes\n",
    "        k (int): show the largest k prediction, default to 1\n",
    "    \"\"\"\n",
    "    classifier.eval()\n",
    "    X = sparse_to_tensor(vectorizer.vectorizer.transform(list(vocab.keys())))\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        result = classifier(X, apply_softmax=True)\n",
    "    \n",
    "    vocab_id = result[1:].topk(k, dim=0)[1]\n",
    "    vocab_weight = result[1:].topk(k, dim=0)[0]\n",
    "    return vocab_id, vocab_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19081"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = vectorizer.vectorizer.vocabulary_\n",
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_k = infer_tokens_importance(vocab, classifier, vectorizer, classes, k=50)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([50, 11])"
      ]
     },
     "execution_count": 167,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_k.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "id_vocab = {vocab[token]:token for token in vocab.keys()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_top_k_DataFrame(vocab, classifier, vectorizer, classes, k=10):\n",
    "    \n",
    "    vocab_id = infer_tokens_importance(vocab, classifier, vectorizer, classes, k)[0]\n",
    "    df = pd.DataFrame(columns = classes)\n",
    "    for i in range(len(classes)):\n",
    "        \n",
    "        indices = vocab_id[:,i].tolist()\n",
    "        words = pd.Series([id_vocab[j] for j in indices])\n",
    "        df[classes[i]] = words\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Criteria i</th>\n",
       "      <th>Criteria ii</th>\n",
       "      <th>Criteria iii</th>\n",
       "      <th>Criteria iv</th>\n",
       "      <th>Criteria v</th>\n",
       "      <th>Criteria vi</th>\n",
       "      <th>Criteria vii</th>\n",
       "      <th>Criteria viii</th>\n",
       "      <th>Criteria ix</th>\n",
       "      <th>Criteria x</th>\n",
       "      <th>Others</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>builders</td>\n",
       "      <td>at uxmal</td>\n",
       "      <td>area all</td>\n",
       "      <td>and characteristics</td>\n",
       "      <td>abroad</td>\n",
       "      <td>display of</td>\n",
       "      <td>and patronage</td>\n",
       "      <td>in</td>\n",
       "      <td>arabia and</td>\n",
       "      <td>biblical</td>\n",
       "      <td>jaguars of</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>building and</td>\n",
       "      <td>beki</td>\n",
       "      <td>area an</td>\n",
       "      <td>abraham</td>\n",
       "      <td>be protected</td>\n",
       "      <td>european baroque</td>\n",
       "      <td>of threatened</td>\n",
       "      <td>succession</td>\n",
       "      <td>cross</td>\n",
       "      <td>affected</td>\n",
       "      <td>who</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bab</td>\n",
       "      <td>at turaif</td>\n",
       "      <td>forested mountains</td>\n",
       "      <td>conditions in</td>\n",
       "      <td>abraham darby</td>\n",
       "      <td>displacement and</td>\n",
       "      <td>meeting place</td>\n",
       "      <td>artistic works</td>\n",
       "      <td>australian tectonic</td>\n",
       "      <td>largest concentration</td>\n",
       "      <td>kindibo</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>buffer against</td>\n",
       "      <td>and society</td>\n",
       "      <td>harmonious interaction</td>\n",
       "      <td>giving rise</td>\n",
       "      <td>coal mining</td>\n",
       "      <td>atoll</td>\n",
       "      <td>is notable</td>\n",
       "      <td>society of</td>\n",
       "      <td>in an</td>\n",
       "      <td>affected by</td>\n",
       "      <td>some are</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>connections with</td>\n",
       "      <td>and st</td>\n",
       "      <td>accentuated</td>\n",
       "      <td>above the</td>\n",
       "      <td>gandharan</td>\n",
       "      <td>displacement</td>\n",
       "      <td>habitat</td>\n",
       "      <td>and historical</td>\n",
       "      <td>arc</td>\n",
       "      <td>considered as</td>\n",
       "      <td>initial</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>baalbek</td>\n",
       "      <td>admirably</td>\n",
       "      <td>accompanied</td>\n",
       "      <td>marae ahu</td>\n",
       "      <td>successive cultures</td>\n",
       "      <td>display</td>\n",
       "      <td>and pelagic</td>\n",
       "      <td>century by</td>\n",
       "      <td>history as</td>\n",
       "      <td>all reflected</td>\n",
       "      <td>known from</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>continuous use</td>\n",
       "      <td>nasrid</td>\n",
       "      <td>is typical</td>\n",
       "      <td>cool temperate</td>\n",
       "      <td>listed</td>\n",
       "      <td>and diffusion</td>\n",
       "      <td>and particularly</td>\n",
       "      <td>improving</td>\n",
       "      <td>the greatest</td>\n",
       "      <td>halls</td>\n",
       "      <td>terraced</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>to sustain</td>\n",
       "      <td>at various</td>\n",
       "      <td>civilization in</td>\n",
       "      <td>active</td>\n",
       "      <td>medieval church</td>\n",
       "      <td>and design</td>\n",
       "      <td>and ideas</td>\n",
       "      <td>mogao</td>\n",
       "      <td>monte san</td>\n",
       "      <td>history that</td>\n",
       "      <td>san</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>buddhist art</td>\n",
       "      <td>application</td>\n",
       "      <td>appreciation of</td>\n",
       "      <td>gjirokastra</td>\n",
       "      <td>gandharan school</td>\n",
       "      <td>and decoration</td>\n",
       "      <td>and many</td>\n",
       "      <td>but are</td>\n",
       "      <td>appropriate</td>\n",
       "      <td>conservation significance</td>\n",
       "      <td>year period</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>nabataean</td>\n",
       "      <td>appears to</td>\n",
       "      <td>shoreline</td>\n",
       "      <td>and chinese</td>\n",
       "      <td>abbey is</td>\n",
       "      <td>below sea</td>\n",
       "      <td>st george</td>\n",
       "      <td>of alpine</td>\n",
       "      <td>outstanding manner</td>\n",
       "      <td>and mostly</td>\n",
       "      <td>the hidden</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>building corals</td>\n",
       "      <td>exposed and</td>\n",
       "      <td>its architectural</td>\n",
       "      <td>private buildings</td>\n",
       "      <td>such</td>\n",
       "      <td>european capitals</td>\n",
       "      <td>both of</td>\n",
       "      <td>of all</td>\n",
       "      <td>australian</td>\n",
       "      <td>and most</td>\n",
       "      <td>known for</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>temperate forest</td>\n",
       "      <td>and speciation</td>\n",
       "      <td>of lao</td>\n",
       "      <td>rococo</td>\n",
       "      <td>out as</td>\n",
       "      <td>particular had</td>\n",
       "      <td>and ottoman</td>\n",
       "      <td>most</td>\n",
       "      <td>fauna with</td>\n",
       "      <td>associated fauna</td>\n",
       "      <td>the orkhon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>and was</td>\n",
       "      <td>and squares</td>\n",
       "      <td>alexander</td>\n",
       "      <td>of ten</td>\n",
       "      <td>winds</td>\n",
       "      <td>and density</td>\n",
       "      <td>old rauma</td>\n",
       "      <td>mammals include</td>\n",
       "      <td>foreign</td>\n",
       "      <td>proof of</td>\n",
       "      <td>of mangroves</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>buddhist monastic</td>\n",
       "      <td>and social</td>\n",
       "      <td>human impacts</td>\n",
       "      <td>ensembles illustrating</td>\n",
       "      <td>its monumental</td>\n",
       "      <td>in tropical</td>\n",
       "      <td>ferrous</td>\n",
       "      <td>mer</td>\n",
       "      <td>migration and</td>\n",
       "      <td>aesthetic qualities</td>\n",
       "      <td>the giant</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>jesuit missions</td>\n",
       "      <td>grove</td>\n",
       "      <td>anatolian</td>\n",
       "      <td>site the</td>\n",
       "      <td>the one</td>\n",
       "      <td>ambohimanga is</td>\n",
       "      <td>contributions of</td>\n",
       "      <td>respect</td>\n",
       "      <td>democratic republic</td>\n",
       "      <td>aphrodisias</td>\n",
       "      <td>this species</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>foot</td>\n",
       "      <td>ad num</td>\n",
       "      <td>education</td>\n",
       "      <td>across wide</td>\n",
       "      <td>in syracuse</td>\n",
       "      <td>and developed</td>\n",
       "      <td>resting on</td>\n",
       "      <td>medieval and</td>\n",
       "      <td>lowland gorilla</td>\n",
       "      <td>its wall</td>\n",
       "      <td>biosphere</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>indonesia and</td>\n",
       "      <td>commemorative</td>\n",
       "      <td>components of</td>\n",
       "      <td>power of</td>\n",
       "      <td>scope</td>\n",
       "      <td>also has</td>\n",
       "      <td>rome the</td>\n",
       "      <td>into new</td>\n",
       "      <td>defence structures</td>\n",
       "      <td>bears significant</td>\n",
       "      <td>correspond to</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>designs</td>\n",
       "      <td>feed</td>\n",
       "      <td>of planned</td>\n",
       "      <td>act</td>\n",
       "      <td>its military</td>\n",
       "      <td>workshop of</td>\n",
       "      <td>primary</td>\n",
       "      <td>classic</td>\n",
       "      <td>contain some</td>\n",
       "      <td>cape</td>\n",
       "      <td>today in</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>assumption monastery</td>\n",
       "      <td>and stable</td>\n",
       "      <td>of sana</td>\n",
       "      <td>ensembles of</td>\n",
       "      <td>ganges</td>\n",
       "      <td>discoveries</td>\n",
       "      <td>fusion</td>\n",
       "      <td>and metal</td>\n",
       "      <td>di noto</td>\n",
       "      <td>apex of</td>\n",
       "      <td>bell tower</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>has represented</td>\n",
       "      <td>caliphate</td>\n",
       "      <td>organized</td>\n",
       "      <td>above sea</td>\n",
       "      <td>and technical</td>\n",
       "      <td>relatively</td>\n",
       "      <td>ongoing biological</td>\n",
       "      <td>reconquest</td>\n",
       "      <td>series of</td>\n",
       "      <td>as exemplified</td>\n",
       "      <td>temporary</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Criteria i     Criteria ii            Criteria iii  \\\n",
       "0               builders        at uxmal                area all   \n",
       "1           building and            beki                 area an   \n",
       "2                    bab       at turaif      forested mountains   \n",
       "3         buffer against     and society  harmonious interaction   \n",
       "4       connections with          and st             accentuated   \n",
       "5                baalbek       admirably             accompanied   \n",
       "6         continuous use          nasrid              is typical   \n",
       "7             to sustain      at various         civilization in   \n",
       "8           buddhist art     application         appreciation of   \n",
       "9              nabataean      appears to               shoreline   \n",
       "10       building corals     exposed and       its architectural   \n",
       "11      temperate forest  and speciation                  of lao   \n",
       "12               and was     and squares               alexander   \n",
       "13     buddhist monastic      and social           human impacts   \n",
       "14       jesuit missions           grove               anatolian   \n",
       "15                  foot          ad num               education   \n",
       "16         indonesia and   commemorative           components of   \n",
       "17               designs            feed              of planned   \n",
       "18  assumption monastery      and stable                 of sana   \n",
       "19       has represented       caliphate               organized   \n",
       "\n",
       "               Criteria iv           Criteria v        Criteria vi  \\\n",
       "0      and characteristics               abroad         display of   \n",
       "1                  abraham         be protected   european baroque   \n",
       "2            conditions in        abraham darby   displacement and   \n",
       "3              giving rise          coal mining              atoll   \n",
       "4                above the            gandharan       displacement   \n",
       "5                marae ahu  successive cultures            display   \n",
       "6           cool temperate               listed      and diffusion   \n",
       "7                   active      medieval church         and design   \n",
       "8              gjirokastra     gandharan school     and decoration   \n",
       "9              and chinese             abbey is          below sea   \n",
       "10       private buildings                 such  european capitals   \n",
       "11                  rococo               out as     particular had   \n",
       "12                  of ten                winds        and density   \n",
       "13  ensembles illustrating       its monumental        in tropical   \n",
       "14                site the              the one     ambohimanga is   \n",
       "15             across wide          in syracuse      and developed   \n",
       "16                power of                scope           also has   \n",
       "17                     act         its military        workshop of   \n",
       "18            ensembles of               ganges        discoveries   \n",
       "19               above sea        and technical         relatively   \n",
       "\n",
       "          Criteria vii    Criteria viii          Criteria ix  \\\n",
       "0        and patronage               in           arabia and   \n",
       "1        of threatened       succession                cross   \n",
       "2        meeting place   artistic works  australian tectonic   \n",
       "3           is notable       society of                in an   \n",
       "4              habitat   and historical                  arc   \n",
       "5          and pelagic       century by           history as   \n",
       "6     and particularly        improving         the greatest   \n",
       "7            and ideas            mogao            monte san   \n",
       "8             and many          but are          appropriate   \n",
       "9            st george        of alpine   outstanding manner   \n",
       "10             both of           of all           australian   \n",
       "11         and ottoman             most           fauna with   \n",
       "12           old rauma  mammals include              foreign   \n",
       "13             ferrous              mer        migration and   \n",
       "14    contributions of          respect  democratic republic   \n",
       "15          resting on     medieval and      lowland gorilla   \n",
       "16            rome the         into new   defence structures   \n",
       "17             primary          classic         contain some   \n",
       "18              fusion        and metal              di noto   \n",
       "19  ongoing biological       reconquest            series of   \n",
       "\n",
       "                   Criteria x         Others  \n",
       "0                    biblical     jaguars of  \n",
       "1                    affected            who  \n",
       "2       largest concentration        kindibo  \n",
       "3                 affected by       some are  \n",
       "4               considered as        initial  \n",
       "5               all reflected     known from  \n",
       "6                       halls       terraced  \n",
       "7                history that            san  \n",
       "8   conservation significance    year period  \n",
       "9                  and mostly     the hidden  \n",
       "10                   and most      known for  \n",
       "11           associated fauna     the orkhon  \n",
       "12                   proof of   of mangroves  \n",
       "13        aesthetic qualities      the giant  \n",
       "14                aphrodisias   this species  \n",
       "15                   its wall      biosphere  \n",
       "16          bears significant  correspond to  \n",
       "17                       cape       today in  \n",
       "18                    apex of     bell tower  \n",
       "19             as exemplified      temporary  "
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_top_k_DataFrame(vocab, classifier, vectorizer, classes, k=20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "make_top_k_DataFrame(vocab, classifier, vectorizer, classes, k=50).to_csv(args.save_dir+'top_words.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 393,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.set_split('test')\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "conf_mat_test = np.zeros((len(classes)-1,len(classes)-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 394,
   "metadata": {},
   "outputs": [],
   "source": [
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    \n",
    "    # get the data compute fuzzy labels\n",
    "    X = batch_dict['x_data']\n",
    "\n",
    "    y_target = batch_dict['y_target']\n",
    "    y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "    # compute the output\n",
    "    y_pred = classifier(X)\n",
    "    \n",
    "    conf_mat_test = np.add(conf_mat_test,confusion_matrix(y_target.argmax(axis=1), y_pred.argmax(axis=1),\n",
    "                                                          labels=range(len(classes)-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[16.,  6.,  4.,  7.,  0.,  1.,  2.,  0.,  0.,  0.],\n",
       "       [ 0., 44.,  9., 18.,  2.,  0.,  1.,  0.,  0.,  0.],\n",
       "       [ 1.,  5., 38., 14.,  4.,  0.,  0.,  0.,  0.,  1.],\n",
       "       [ 1., 13., 14., 52.,  1.,  0.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  1.,  9., 10.,  7.,  0.,  3.,  1.,  1.,  0.],\n",
       "       [ 1.,  3., 11.,  9.,  0., 17.,  0.,  0.,  0.,  2.],\n",
       "       [ 0.,  1.,  0.,  2.,  0.,  0., 34.,  1.,  1.,  3.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  1.,  8., 15.,  4.,  2.],\n",
       "       [ 0.,  0.,  0.,  1.,  0.,  1.,  0.,  3., 28., 14.],\n",
       "       [ 0.,  0.,  0.,  2.,  0.,  0.,  1.,  0.,  7., 54.]])"
      ]
     },
     "execution_count": 395,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_mat_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 396,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.set_split('val')\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "conf_mat_val = np.zeros((len(classes)-1,len(classes)-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 397,
   "metadata": {},
   "outputs": [],
   "source": [
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    \n",
    "    # get the data compute fuzzy labels\n",
    "    X = batch_dict['x_data']\n",
    "\n",
    "    y_target = batch_dict['y_target']\n",
    "    y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "    # compute the output\n",
    "    y_pred = classifier(X)\n",
    "    \n",
    "    conf_mat_val = np.add(conf_mat_val,confusion_matrix(y_target.argmax(axis=1), y_pred.argmax(axis=1),labels=range(len(classes)-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[10.,  6.,  7.,  8.,  0.,  3.,  1.,  0.,  0.,  0.],\n",
       "       [ 0., 50.,  5.,  9.,  0.,  1.,  1.,  0.,  0.,  0.],\n",
       "       [ 3.,  4., 57., 11.,  2.,  2.,  0.,  0.,  0.,  0.],\n",
       "       [ 1., 11.,  9., 58.,  0.,  2.,  1.,  0.,  0.,  0.],\n",
       "       [ 0.,  2.,  4.,  8.,  9.,  0.,  1.,  0.,  1.,  0.],\n",
       "       [ 1.,  7., 10.,  5.,  0., 17.,  0.,  0.,  0.,  0.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0., 34.,  0.,  3.,  2.],\n",
       "       [ 0.,  1.,  0.,  3.,  0.,  0.,  7., 22.,  3.,  4.],\n",
       "       [ 0.,  0.,  1.,  0.,  0.,  0.,  9.,  2., 21.,  7.],\n",
       "       [ 0.,  0.,  0.,  0.,  0.,  0.,  2.,  1.,  5., 58.]])"
      ]
     },
     "execution_count": 398,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_mat_val"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.set_split('train')\n",
    "set_seed_everywhere(args.seed, args.cuda)\n",
    "batch_generator = generate_batches(dataset, \n",
    "                                   batch_size=args.batch_size, \n",
    "                                   device=args.device)\n",
    "conf_mat_train = np.zeros((len(classes)-1,len(classes)-1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 400,
   "metadata": {},
   "outputs": [],
   "source": [
    "for batch_index, batch_dict in enumerate(batch_generator):\n",
    "    \n",
    "    # get the data compute fuzzy labels\n",
    "    X = batch_dict['x_data']\n",
    "\n",
    "    y_target = batch_dict['y_target']\n",
    "    y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "    Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                            how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "    # compute the output\n",
    "    y_pred = classifier(X)\n",
    "    \n",
    "    conf_mat_train = np.add(conf_mat_train,confusion_matrix(y_target.argmax(axis=1), y_pred.argmax(axis=1),labels=range(len(classes)-1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[306.,   3.,   5.,  15.,   0.,   0.,   1.,   0.,   0.,   0.],\n",
       "       [  1., 608.,   5.,  13.,   0.,   1.,   0.,   0.,   0.,   0.],\n",
       "       [  1.,   2., 634.,   7.,   0.,   1.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   4.,   3., 763.,   0.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   3.,  12.,  30., 162.,   0.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   6.,   4.,   1.,   0., 312.,   0.,   0.,   0.,   0.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0., 372.,   0.,   3.,   7.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   4., 254.,   1.,   1.],\n",
       "       [  0.,   0.,   1.,   1.,   0.,   0.,   1.,   1., 351.,  14.],\n",
       "       [  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   3., 563.]])"
      ]
     },
     "execution_count": 401,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conf_mat_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(conf_mat_test),pd.DataFrame(conf_mat_val),pd.DataFrame(conf_mat_train)],axis=1).to_csv(args.save_dir+'confusion_matrix.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.concat([pd.DataFrame(conf_mat_test),pd.DataFrame(conf_mat_val),pd.DataFrame(conf_mat_train)],axis=1).to_csv(args.save_dir+'baseline_confusion_matrix.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 366,
   "metadata": {},
   "outputs": [],
   "source": [
    "def per_class_metrics(confusion_matrix, classes):\n",
    "    '''\n",
    "    Compute the per class precision, recall, and F1 for all the classes\n",
    "    \n",
    "    Args:\n",
    "    confusion_matrix (np.ndarry) with shape of (n_classes,n_classes): a confusion matrix of interest\n",
    "    classes (list of str) with shape (n_classes,): The names of classes\n",
    "    \n",
    "    Returns:\n",
    "    metrics_dict (dictionary): a dictionary that records the per class metrics\n",
    "    '''\n",
    "    num_class = confusion_matrix.shape[0]\n",
    "    metrics_dict = {}\n",
    "    for i in range(num_class):\n",
    "        key = classes[i]\n",
    "        temp_dict = {}\n",
    "        row = confusion_matrix[i,:]\n",
    "        col = confusion_matrix[:,i]\n",
    "        val = confusion_matrix[i,i]\n",
    "        precision = val/row.sum()\n",
    "        recall = val/col.sum()\n",
    "        F1 = 2*(precision*recall)/(precision+recall)\n",
    "        temp_dict['precision'] = precision\n",
    "        temp_dict['recall'] = recall\n",
    "        temp_dict['F1'] = F1\n",
    "        metrics_dict[key] = temp_dict\n",
    "    \n",
    "    return metrics_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 367,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_dict = {}\n",
    "metrics_dict['test'] = per_class_metrics(conf_mat_test, classes[:-1])\n",
    "metrics_dict['val'] = per_class_metrics(conf_mat_val, classes[:-1])\n",
    "metrics_dict['train'] = per_class_metrics(conf_mat_train, classes[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 368,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df = pd.DataFrame.from_dict({(i,j): metrics_dict[i][j] \n",
    "                           for i in metrics_dict.keys() \n",
    "                           for j in metrics_dict[i].keys()},\n",
    "                       orient='index')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'per_class_metrics.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 369,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_df.to_csv(args.save_dir+'baseline_per_class_metrics.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try on totally Unseen Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "metadata": {},
   "outputs": [],
   "source": [
    "#ouv_csv='Data/ouv_with_splits_full.csv',\n",
    "new_ouv_csv='Data/sd_full.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_jac_k_accuracy(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    y_pred_indices = y_pred.topk(k, dim=1)[1]\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "        \n",
    "    n_correct = torch.tensor([torch.tensor([y_pred_indices[j][i] in y_target_indices[j] for i in range(k)]).sum()>0 \n",
    "                              for j in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100\n",
    "\n",
    "def compute_jac_1_accuracy(y_pred, y_target, k=3, multilabel=False):\n",
    "    \n",
    "    y_pred_indices = y_pred.topk(1, dim=1)[1]\n",
    "    y_target_indices = y_target.topk(k, dim=1)[1]\n",
    "        \n",
    "    n_correct = torch.tensor([torch.tensor([y_pred_indices[j] in y_target_indices[j] for i in range(k)]).sum()>0 \n",
    "                              for j in range(len(y_pred))]).sum().item()\n",
    "    return n_correct / len(y_pred_indices) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 410,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Elapsed: 6.545941114425659\n"
     ]
    }
   ],
   "source": [
    "with Timer():\n",
    "    loss_func = cross_entropy\n",
    "    set_seed_everywhere(args.seed, args.cuda)\n",
    "    train_state = make_train_state(args)\n",
    "    dataset = OuvDataset.load_dataset_and_load_vectorizer(new_ouv_csv, args.vectorizer_file, \n",
    "                                                          ngrams=args.ngrams, vectorizer=vectorizer.vectorizer)\n",
    "\n",
    "    dataset.set_split('val')\n",
    "    verbose=False\n",
    "    try:\n",
    "      # Iterate over training dataset\n",
    "\n",
    "        # setup: batch generator, set loss and acc to 0, set train mode on\n",
    "        dataset.set_split('val')\n",
    "        batch_generator = generate_batches(dataset, \n",
    "                                           batch_size=args.batch_size, \n",
    "                                           device=args.device)\n",
    "        running_loss = 0.0\n",
    "        running_1_acc = 0.0\n",
    "        running_k_acc = 0.0\n",
    "        running_k_jac = 0.0\n",
    "        classifier.eval()\n",
    "\n",
    "        for batch_index, batch_dict in enumerate(batch_generator):\n",
    "\n",
    "            # step 2. get the data compute fuzzy labels\n",
    "            X = batch_dict['x_data']\n",
    "\n",
    "            y_target = batch_dict['y_target']\n",
    "            y_fuzzy = batch_dict['y_fuzzy']\n",
    "\n",
    "            Y = compute_fuzzy_label(y_target, y_fuzzy, fuzzy= args.fuzzy, \n",
    "                                    how=args.fuzzy_how, lbd = args.fuzzy_lambda)\n",
    "\n",
    "            # step 3. compute the output\n",
    "            with torch.no_grad():\n",
    "                y_pred = classifier(X)\n",
    "\n",
    "            # step 4. compute the loss\n",
    "            loss = loss_func(y_pred, Y)\n",
    "            loss_t = loss.item()\n",
    "            running_loss += (loss_t - running_loss) / (batch_index + 1)\n",
    "\n",
    "            # -----------------------------------------\n",
    "            # compute the accuracies\n",
    "            acc_1_t = compute_jac_1_accuracy(y_pred, y_target)\n",
    "            acc_k_t = compute_jac_k_accuracy(y_pred, y_target, args.k)\n",
    "            jac_k_t = compute_jaccard_index(y_pred, y_target, len(classes))\n",
    "\n",
    "            running_1_acc += (acc_1_t - running_1_acc) / (batch_index + 1)\n",
    "            running_k_acc += (acc_k_t - running_k_acc) / (batch_index + 1)\n",
    "            running_k_jac += (jac_k_t - running_k_jac) / (batch_index + 1)\n",
    "\n",
    "            # update bar\n",
    "            if verbose:\n",
    "                val_bar.set_postfix(loss=running_loss, \n",
    "                                acc_1=running_1_acc,\n",
    "                                acc_k=running_k_acc,\n",
    "                                jac_k=running_k_jac,\n",
    "                                epoch=epoch_index)\n",
    "                val_bar.update()\n",
    "\n",
    "        train_state['val_loss'].append(running_loss)\n",
    "        train_state['val_1_acc'].append(running_1_acc)\n",
    "        train_state['val_k_acc'].append(running_k_acc)\n",
    "        train_state['val_k_jac'].append(running_k_jac)\n",
    "\n",
    "    except KeyboardInterrupt:\n",
    "        print(\"Exiting loop\")\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 411,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': False,\n",
       " 'early_stopping_step': 0,\n",
       " 'early_stopping_best_k_acc_val': 0,\n",
       " 'learning_rate': 0.0002,\n",
       " 'epoch_index': 0,\n",
       " 'train_loss': [],\n",
       " 'train_1_acc': [],\n",
       " 'train_k_acc': [],\n",
       " 'train_k_jac': [],\n",
       " 'val_loss': [1.9158676049580743],\n",
       " 'val_1_acc': [71.11979166666666],\n",
       " 'val_k_acc': [95.28645833333333],\n",
       " 'val_k_jac': [0.3634536236524582],\n",
       " 'test_loss': -1,\n",
       " 'test_1_acc': -1,\n",
       " 'test_k_acc': -1,\n",
       " 'test_k_jac': -1,\n",
       " 'model_filename': 'model_storage/ngram/model.pth'}"
      ]
     },
     "execution_count": 411,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# LS Model\n",
    "train_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 408,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'stop_early': False,\n",
       " 'early_stopping_step': 0,\n",
       " 'early_stopping_best_k_acc_val': 0,\n",
       " 'learning_rate': 0.0002,\n",
       " 'epoch_index': 0,\n",
       " 'train_loss': [],\n",
       " 'train_1_acc': [],\n",
       " 'train_k_acc': [],\n",
       " 'train_k_jac': [],\n",
       " 'val_loss': [2.050477572616485],\n",
       " 'val_1_acc': [70.49479166666666],\n",
       " 'val_k_acc': [95.13020833333334],\n",
       " 'val_k_jac': [0.35870815912882487],\n",
       " 'test_loss': -1,\n",
       " 'test_1_acc': -1,\n",
       " 'test_k_acc': -1,\n",
       " 'test_k_jac': -1,\n",
       " 'model_filename': 'model_storage/ngram/model.pth'}"
      ]
     },
     "execution_count": 408,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Baseline\n",
    "train_state"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## END"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}