NER.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocess Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(data_lines):\n",
    "    for sent in data_lines:\n",
    "        for word_data in sent:\n",
    "            data = word_data.split()\n",
    "            word = data[0]\n",
    "            pos = data[1]\n",
    "            ner_tag = data[-1]\n",
    "            yield word, pos, ner_tag\n",
    "\n",
    "with open('../datasets/NER/ner_data/train') as f:\n",
    "    lines = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = ''.join(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = lines.lower()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = lines.split('-docstart- -x- o o')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "947"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(lines)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = [line.strip() for line in lines]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "dlines = [line.split('\\n') for line in lines] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "flattened = []\n",
    "for para in dlines:\n",
    "    for line in para:\n",
    "        if line:\n",
    "            flattened.append(line.split())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(flattened)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('../datasets/NER/prep_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>eu</td>\n",
       "      <td>nnp</td>\n",
       "      <td>i-np</td>\n",
       "      <td>i-org</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>rejects</td>\n",
       "      <td>vbz</td>\n",
       "      <td>i-vp</td>\n",
       "      <td>o</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>german</td>\n",
       "      <td>jj</td>\n",
       "      <td>i-np</td>\n",
       "      <td>i-misc</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>call</td>\n",
       "      <td>nn</td>\n",
       "      <td>i-np</td>\n",
       "      <td>o</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>to</td>\n",
       "      <td>to</td>\n",
       "      <td>i-vp</td>\n",
       "      <td>o</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         0    1     2       3\n",
       "0       eu  nnp  i-np   i-org\n",
       "1  rejects  vbz  i-vp       o\n",
       "2   german   jj  i-np  i-misc\n",
       "3     call   nn  i-np       o\n",
       "4       to   to  i-vp       o"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "flattened = []\n",
    "for para in dlines:\n",
    "    words_in_line  = []\n",
    "    for line in para:\n",
    "        if line:\n",
    "            word = line.split()[0]\n",
    "            words_in_line.append(word)\n",
    "    flattened.append(words_in_line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:gensim.models.word2vec:consider setting layer size to a multiple of 4 for greater performance\n"
     ]
    }
   ],
   "source": [
    "model = Word2Vec(flattened,min_count=1, size=50, workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('../models/ner_word_to_vec50')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('belgium', 0.9993789196014404),\n",
       " ('15-12', 0.9985662698745728),\n",
       " ('men', 0.9984050989151001),\n",
       " ('pounds', 0.9981710910797119),\n",
       " ('toyota', 0.998096227645874),\n",
       " ('metres', 0.9976305365562439),\n",
       " ('hill', 0.9975293874740601),\n",
       " ('rabobank', 0.9970297813415527),\n",
       " ('anders', 0.996955931186676),\n",
       " ('williams', 0.9969029426574707)]"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.most_similar('britain')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "words_unique = df[0].unique()\n",
    "pos1_unique = df[1].unique()\n",
    "pos2_unique = df[2].unique()\n",
    "ner_tag_unique = df[3].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21009, 45, 17, 8)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(words_unique), len(pos1_unique), len(pos2_unique), len(ner_tag_unique)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.19610199,  0.36634412,  0.37448475,  0.28899369,  0.50808477,\n",
       "        0.12252712,  0.23630257,  0.84013253,  0.0398668 ,  0.95358223,\n",
       "        0.22630259,  0.73313618,  0.52588075,  0.1085002 ,  0.81756812,\n",
       "        0.69893545,  0.50664914,  0.41141805,  0.82464719,  0.29948178,\n",
       "        0.33172587,  0.17498596,  0.13708961,  0.14883095,  0.02761589,\n",
       "        0.91030473,  0.38769689,  0.08415267,  0.51086742,  0.31247234,\n",
       "        0.07131792,  0.39750078,  0.13482596,  0.99242491,  0.84296721,\n",
       "        0.41715375,  0.22690408,  0.08056165,  0.33271489,  0.55464542,\n",
       "        0.13982174,  0.38572928,  0.91170478,  0.10298305,  0.74147236,\n",
       "        0.17546977,  0.97546738,  0.97363341,  0.12150187,  0.9990142 ], dtype=float32)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def word_to_tensor(word):\n",
    "    if word in model.wv:\n",
    "        return model.wv[word]\n",
    "    else:\n",
    "#         print('not in model')\n",
    "        return np.random.rand(model.vector_size).astype(np.float32)\n",
    "word_to_tensor('hello')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos1_to_ix = {}\n",
    "ix_to_pos1 = {}\n",
    "pos2_to_ix = {}\n",
    "ix_to_pos2 = {}\n",
    "\n",
    "for i in range(len(pos1_unique)):\n",
    "    pos1_to_ix[pos1_unique[i]]  = i\n",
    "    ix_to_pos1[i] = pos1_unique[i]\n",
    "    \n",
    "for i in range(len(pos2_unique)):\n",
    "    pos2_to_ix[pos2_unique[i]]  = i\n",
    "    ix_to_pos2[i] = pos2_unique[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "ix_to_ner = {}\n",
    "ner_to_ix = {}\n",
    "\n",
    "for i in range(len(ner_tag_unique)):\n",
    "    ner_to_ix[ner_tag_unique[i]] = i\n",
    "    ix_to_ner[i] = ner_tag_unique[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'b-adjp': 16,\n",
       " 'b-advp': 14,\n",
       " 'b-np': 4,\n",
       " 'b-pp': 10,\n",
       " 'b-sbar': 15,\n",
       " 'b-vp': 9,\n",
       " 'i-adjp': 6,\n",
       " 'i-advp': 7,\n",
       " 'i-conjp': 11,\n",
       " 'i-intj': 12,\n",
       " 'i-lst': 13,\n",
       " 'i-np': 0,\n",
       " 'i-pp': 3,\n",
       " 'i-prt': 8,\n",
       " 'i-sbar': 5,\n",
       " 'i-vp': 1,\n",
       " 'o': 2}"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos2_to_ix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_lines = [line.split('\\n\\n') for line in lines]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare Vectors and data segregation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[('eu', 'nnp', 'i-np'), ('rejects', 'vbz', 'i-vp'), ('german', 'jj', 'i-np'), ('call', 'nn', 'i-np'), ('to', 'to', 'i-vp'), ('boycott', 'vb', 'i-vp'), ('british', 'jj', 'i-np'), ('lamb', 'nn', 'i-np'), ('.', '.', 'o')], [('peter', 'nnp', 'i-np'), ('blackburn', 'nnp', 'i-np')], [('brussels', 'nnp', 'i-np'), ('1996-08-22', 'cd', 'i-np')], [('the', 'dt', 'i-np'), ('european', 'nnp', 'i-np'), ('commission', 'nnp', 'i-np'), ('said', 'vbd', 'i-vp'), ('on', 'in', 'i-pp'), ('thursday', 'nnp', 'i-np'), ('it', 'prp', 'b-np'), ('disagreed', 'vbd', 'i-vp'), ('with', 'in', 'i-pp'), ('german', 'jj', 'i-np'), ('advice', 'nn', 'i-np'), ('to', 'to', 'i-pp'), ('consumers', 'nns', 'i-np'), ('to', 'to', 'i-vp'), ('shun', 'vb', 'i-vp'), ('british', 'jj', 'i-np'), ('lamb', 'nn', 'i-np'), ('until', 'in', 'i-sbar'), ('scientists', 'nns', 'i-np'), ('determine', 'vbp', 'i-vp'), ('whether', 'in', 'i-sbar'), ('mad', 'jj', 'i-np'), ('cow', 'nn', 'i-np'), ('disease', 'nn', 'i-np'), ('can', 'md', 'i-vp'), ('be', 'vb', 'i-vp'), ('transmitted', 'vbn', 'i-vp'), ('to', 'to', 'i-pp'), ('sheep', 'nn', 'i-np'), ('.', '.', 'o')], [('germany', 'nnp', 'i-np'), (\"'s\", 'pos', 'b-np'), ('representative', 'nn', 'i-np'), ('to', 'to', 'i-pp'), ('the', 'dt', 'i-np'), ('european', 'nnp', 'i-np'), ('union', 'nnp', 'i-np'), (\"'s\", 'pos', 'b-np'), ('veterinary', 'jj', 'i-np'), ('committee', 'nn', 'i-np'), ('werner', 'nnp', 'i-np'), ('zwingmann', 'nnp', 'i-np'), ('said', 'vbd', 'i-vp'), ('on', 'in', 'i-pp'), ('wednesday', 'nnp', 'i-np'), ('consumers', 'nns', 'i-np'), ('should', 'md', 'i-vp'), ('buy', 'vb', 'i-vp'), ('sheepmeat', 'nn', 'i-np'), ('from', 'in', 'i-pp'), ('countries', 'nns', 'i-np'), ('other', 'jj', 'i-adjp'), ('than', 'in', 'i-pp'), ('britain', 'nnp', 'i-np'), ('until', 'in', 'i-sbar'), ('the', 'dt', 'i-np'), ('scientific', 'jj', 'i-np'), ('advice', 'nn', 'i-np'), ('was', 'vbd', 'i-vp'), ('clearer', 'jjr', 'i-adjp'), ('.', '.', 'o')]]\n",
      "[['i-org', 'o', 'i-misc', 'o', 'o', 'o', 'i-misc', 'o', 'o'], ['i-per', 'i-per'], ['i-loc', 'o'], ['o', 'i-org', 'i-org', 'o', 'o', 'o', 'o', 'o', 'o', 'i-misc', 'o', 'o', 'o', 'o', 'o', 'i-misc', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o'], ['i-loc', 'o', 'o', 'o', 'o', 'i-org', 'i-org', 'o', 'o', 'o', 'i-per', 'i-per', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'o', 'i-loc', 'o', 'o', 'o', 'o', 'o', 'o', 'o']]\n"
     ]
    }
   ],
   "source": [
    "flat_para = []\n",
    "X_words = []\n",
    "X_pos1 = []\n",
    "X_pos2 = []\n",
    "y_ner = []\n",
    "for ess in split_lines:\n",
    "\n",
    "    for para in ess:\n",
    "        \n",
    "        chunks = para.split('\\n')\n",
    "#         print(chunks)\n",
    "        words = []\n",
    "        pos1 = []\n",
    "        pos2 = []\n",
    "        ner = []\n",
    "        for part in chunks:\n",
    "            if part:\n",
    "                word, pos1_tag, pos2_tag, ner_tag = part.split()\n",
    "                words.append((word,pos1_tag,pos2_tag))\n",
    "                pos1.append(pos1_tag)\n",
    "                pos2.append(pos2_tag)\n",
    "                ner.append(ner_tag)\n",
    "\n",
    "        X_words.append(words)\n",
    "\n",
    "        y_ner.append(ner)\n",
    "        \n",
    "print(X_words[:5])\n",
    "print(y_ner[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### prepare vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Prepare X vectors\"\"\"\n",
    "def convert_to_vec(X_words):\n",
    "    X = []\n",
    "    for sent in X_words:\n",
    "        X_val = []\n",
    "        for word in sent:\n",
    "            wv = model.wv[word[0]]\n",
    "            wv_val = wv.tolist()\n",
    "            wv_val.append(pos1_to_ix[word[1]])\n",
    "            wv_val.append(pos2_to_ix[word[2]])\n",
    "            X_val.append(wv_val)\n",
    "        \n",
    "#         print(X_val)\n",
    "        X.append(X_val)\n",
    "    return X\n",
    "        \n",
    "    \n",
    "X = convert_to_vec(X_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 0, 0, 0, 0, 0, 0]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''' One hot vectors for some other framework not required in pytorch'''\n",
    "def convert_ner_vec(ner):\n",
    "    vec = [0]*8\n",
    "    vec[ner] = 1\n",
    "    return vec\n",
    "convert_ner_vec(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Prepare y vectors '''\n",
    "def convert_y_vec(y_vec):\n",
    "    y = []\n",
    "    for val in y_vec:\n",
    "        y_val = []\n",
    "        for v in val:\n",
    "            y_val.append(ner_to_ix[v])\n",
    "        y.append(y_val)\n",
    "    return y\n",
    "y = convert_y_vec(y_ner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(14042,)\n",
      "(14042,)\n"
     ]
    }
   ],
   "source": [
    "X_np = np.array(X)\n",
    "y_np = np.array(y)\n",
    "print(X_np.shape)\n",
    "print(y_np.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2, 1, 1, 1, 2, 1, 1]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' Train and validation set '''\n",
    "def train_test_split(X,y, train_ratio=0.9):\n",
    "    split_index = int(len(X)*train_ratio) \n",
    "    train_X = X[:split_index]\n",
    "    train_y = y[:split_index]\n",
    "    test_X = X[split_index:]\n",
    "    test_y = y[split_index:]\n",
    "    return train_X, test_X, train_y, test_y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Some dimensionality checks for self  assurance XD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9, 52)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(X[0]).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9,)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(y[0]).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12637 1405 12637 1405\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_np, y_np)\n",
    "print(len(X_train), len(X_test), len(y_train), len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(9,)"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(y_train[0]).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Time for Torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RNN(nn.Module):\n",
    "    def __init__(self,input_size, hidden_size,output_size):\n",
    "        super(RNN, self).__init__()\n",
    "        self.hidden_size = hidden_size\n",
    "        self.i2h = nn.Linear(input_size + hidden_size, hidden_size)\n",
    "        self.i2o = nn.Linear(input_size + hidden_size, 100)\n",
    "        self.l1 = nn.Linear(100,50)\n",
    "        self.l2 = nn.Linear(50,10)\n",
    "        self.l3 =nn.Linear(10,output_size)\n",
    "        \n",
    "    def initHidden(self):\n",
    "        return Variable(torch.zeros(1,self.hidden_size))\n",
    "    \n",
    "    def forward(self, input, hidden):\n",
    "        combined = torch.cat((input,hidden), 1)\n",
    "        hidden = self.i2h(combined)\n",
    "        output = self.i2o(combined)\n",
    "        output = F.relu(self.l1(output))\n",
    "        output = F.relu(self.l2(output))\n",
    "        output = F.softmax(self.l3(output))\n",
    "        return output, hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "INPUT_SIZE = 52\n",
    "HIDDEN_SIZE = 256\n",
    "OUTPUT_SIZE = 8\n",
    "learning_rate = 1e-3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 52])\n",
      "torch.Size([1, 256])\n",
      "torch.Size([1, 8])\n",
      "torch.Size([1, 256])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 2.0861\n",
       "[torch.FloatTensor of size 1]"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# can ignore this cell meant for validation after all torch supports DCG\n",
    "\n",
    "# trial_val = Variable(torch.Tensor(X[0][1]).unsqueeze(0))\n",
    "# print(trial_val.size())\n",
    "# trial_hidden = rnn.initHidden()\n",
    "# print(trial_hidden.size())\n",
    "# output, hidden = rnn(trial_val, trial_hidden)\n",
    "# print(output.size())\n",
    "# print(hidden.size())\n",
    "# criterion(output,Variable( torch.Tensor(y[0][1]).long()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  0 Avg loss 2.0894403616\n",
      "Epoch  1 Avg loss 2.04145355984\n",
      "Epoch  2 Avg loss 1.90388846487\n",
      "Epoch  3 Avg loss 1.80077001527\n",
      "Epoch  4 Avg loss 1.73451424841\n",
      "Epoch  5 Avg loss 1.68860841941\n",
      "Epoch  6 Avg loss 1.65501919779\n",
      "Epoch  7 Avg loss 1.6294387786\n",
      "Epoch  8 Avg loss 1.60934497502\n",
      "Epoch  9 Avg loss 1.59316476928\n",
      "CPU times: user 5h 27min 44s, sys: 1min 41s, total: 5h 29min 26s\n",
      "Wall time: 2h 16min 52s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "INPUT_SIZE = 52\n",
    "HIDDEN_SIZE = 128\n",
    "OUTPUT_SIZE = 8\n",
    "learning_rate = 5e-3\n",
    "\n",
    "rnn = RNN(INPUT_SIZE, HIDDEN_SIZE, OUTPUT_SIZE)\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.Adam(rnn.parameters(),lr=learning_rate)\n",
    "num_epochs = 10\n",
    "all_losses = []\n",
    "\n",
    "losses = []\n",
    "iter_losses = []\n",
    "for i in range(num_epochs):\n",
    "    iter_loss = []\n",
    "    optimizer.zero_grad()\n",
    "    for j in range(len(X_train)):\n",
    "        loss_per_sample = []\n",
    "        \n",
    "        hidden = rnn.initHidden()\n",
    "        for k in range(len(X_train[j])):\n",
    "            inputs = Variable(torch.Tensor(X_train[j][k]).unsqueeze(0))\n",
    "#             print(y_train[i][j])\n",
    "            targets = Variable(torch.Tensor([y_train[j][k]]).long())            \n",
    "            output, hidden = rnn(inputs,hidden)\n",
    "#             print(output.size())\n",
    "            loss = criterion(output, targets)\n",
    "            loss_per_sample.append(loss.data[0])\n",
    "            loss.backward(retain_graph=True)\n",
    "        \n",
    "        all_losses.extend(loss_per_sample)\n",
    "        iter_loss.append(np.average(loss_per_sample))\n",
    "    optimizer.step()\n",
    "    losses.append(np.average(all_losses))\n",
    "    iter_losses.append(iter_loss)\n",
    "    print('Epoch ', i, 'Avg loss', losses[i])\n",
    "            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### When u know what to plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAH3pJREFUeJzt3Xd41tXdx/H3N5uQhJCQBAgJAVkJ\nW8JQREG0Yh1oq0+hrba2Fmlx1fH0qba1VTttrXVUiqOOWrTWvRVEEURDGAIhbELCyiCQQBhZ5/kj\nkYIFEuEOv3t8XtfFdZHkcP8+133Jxx/nd+5zzDmHiIgElzCvA4iIiO+p3EVEgpDKXUQkCKncRUSC\nkMpdRCQIqdxFRIKQyl1EJAip3EVEgpDKXUQkCEV4deFOnTq5rKwsry4vIhKQFi1aVOGcS2lpnGfl\nnpWVRX5+vleXFxEJSGa2qTXjWpyWMbMMM5tjZivNrMDMbjjCmH5mtsDMDpjZLccTWEREfKc1d+71\nwM3OucVmFg8sMrP3nHMrDxlTCVwPXNIWIUVE5Mtp8c7dObfNObe4+fe7gUIg/QtjypxzC4G6Nkkp\nIiJfypdaLWNmWcBQ4NO2CCMiIr7R6nI3szjgBeBG51z18VzMzKaYWb6Z5ZeXlx/PS4iISCu0qtzN\nLJKmYn/GOffi8V7MOTfDOZfrnMtNSWlxJY+IiByn1qyWMeAxoNA5d2/bRxIRkRPVmtUyo4ErgOVm\ntrT5e7cBmQDOuelm1hnIBxKARjO7Ecg53umbY6nYc4DpH6xn2rhedGwf5euXFxEJCi2Wu3NuHmAt\njNkOdPNVqGOZv66Cx+dv5F/5JVw/vjdXnpZFVIR2URAROVTAteLEIem8feOZDMnsyN1vFPKVP3/I\nOwXb0UHfIiL/EXDlDtAnLZ6nvjeCJ64aTmR4GNc8vYjJj3zCii1VXkcTEfELAVnunxvbN5W3bhjD\nXRP7s6Z0Dxc9OI9bn/+Msur9XkcTEfFUQJc7QER4GFeclsWcW8bygzE9eXnpFsb+8QPun72WfbUN\nXscTEfFEwJf75zq0i+S2r2Yz66azOKtPCve+t4az//QBLy/ZQmOj5uNFJLQETbl/rntyex7+9jCe\nmzKKTnHR3PjcUi59+GPyiyq9jiYictIEXbl/bmTPZF6ZNpo/XT6Y7VX7uGz6Aqb9czEllXu9jiYi\n0uaCttwBwsKMrw/rxpxbxnLD+N7MLixl/L0f8ru3VrF7vzawFJHgFdTl/rnYqAh+fG4fPrhlHBcO\n6sL0D9cz9p4PeObTTdQ3NHodT0TE50Ki3D/XuUMM9/7PEF69djSnpMRx+0sruOD+eXy0VjtUikhw\nCaly/9ygbok8d80oHv7Wqeyra+CKx/K46u95rCvb7XU0ERGfCMlyBzAzzh/YhfduOpOfnt+P/KKd\nnHffR9zxygoqa2q9jicickJCttw/Fx0RzjVnncIHt45l8ogMnv5kE2PvmcOjH22gtl7z8SISmEK+\n3D+XHBfN3ZcM/K9Nyd5eoU3JRCTwqNy/4Iubkk39hzYlE5HAo3I/iiNtSnbL859Rqk3JRCQAqNyP\n4Yubkr2ydAvjtCmZiAQAlXsrHG1TsjeWbfM6mojIEancv4RDNyVLjovi2pmLWVK80+tYIiL/ReV+\nHEb2TObZKaeRFh/D7S+t0BYGIuJ3VO7HKS46gjsuymHltmqeXLDJ6zgiIodRuZ+ACQM6M65vCve+\nu5ptVfu8jiMicpDK/QSYGXdOHEB9o+PO11Z6HUdE5CCV+wnKSIrl+vG9eWvFdt5fVep1HBERQOXu\nEz8Y05NeqXH84pUCrX8XEb+gcveBqIgwfn3JADbv3McD76/1Oo6IiMrdV0b2TOayYd2YMXcDa0q1\nL7yIeEvl7kM/Pb8fcTER/OylFdpJUkQ8pXL3oeS4aH56fj/yiir596LNXscRkRCmcvexy4dlkNu9\nI795s5CdOtFJRDyicvexsDDj7ksHsHt/Pb97a5XXcUQkRLVY7maWYWZzzGylmRWY2Q1HGGNmdr+Z\nrTOzZWZ2atvEDQz9Oifw/TE9eC6/hIVFlV7HEZEQ1Jo793rgZudcDjAKmGZmOV8Ycz7Qu/nXFOBh\nn6YMQDeM7016Yjtuf2k5ddpYTEROshbL3Tm3zTm3uPn3u4FCIP0LwyYCT7kmnwCJZtbF52kDSGxU\nBL+6uOkUp8fmbfQ6joiEmC81525mWcBQ4NMv/CgdKDnk68389/8AQs45OWl8JSeN+2atoaRyr9dx\nRCSEtLrczSwOeAG40TlXfTwXM7MpZpZvZvnl5eXH8xIB546L+xNmxi9fLdDadxE5aVpV7mYWSVOx\nP+Oce/EIQ7YAGYd83a35e4dxzs1wzuU653JTUlKOJ2/ASU9sx4/P6cPsVWW8u1Ibi4nIydGa1TIG\nPAYUOufuPcqwV4Erm1fNjAKqnHM6YLTZd0dn0a9zPL98tYCaA/VexxGRENCaO/fRwBXA2Wa2tPnX\nV81sqplNbR7zJrABWAc8AvyobeIGpsjwMH596UC2Ve3nvllrvI4jIiEgoqUBzrl5gLUwxgHTfBUq\nGA3r3pHJIzJ5fH4Rlw7tRk7XBK8jiUgQ0ydUT6KfTOhLYrtIbn95OY2NergqIm1H5X4SJcZGcfsF\n2Swp3sWzC0ta/gMiIsdJ5X6SXTo0nVE9k/jdW4VU7DngdRwRCVIq95PMzLj7koHsq2vgN28Ueh1H\nRIKUyt0DvVLjmHrWKby4ZAsfr6/wOo6IBCGVu0emjetFZlIsP3t5BQfqdai2iPiWyt0jMZHh3Dmx\nPxvKa5jx4Qav44hIkFG5e2hs31QuGNSFB+aso6iixus4IhJEVO4e+8WFOUSFh/HzV3Sotoj4jsrd\nY2kJMdzylT58tLaCN5ZrOx4R8Q2Vux+44rQsBqZ34M7XVlK9v87rOCISBFTufiA8zPj1pQMo33OA\ne9/VxmIicuJU7n5iULdErhzVnacWFLFs8y6v44hIgFO5+5Gbz+tLclw0t7+0ggZtLCYiJ0Dl7kcS\nYiL5xYU5LN9SxT8+2eR1HBEJYCp3P3PhoC6M6d2Je95ZTWn1fq/jiEiAUrn7GTPjrokDqG1o5K7X\nV3odR0QClMrdD2V1as+143rx+rJtfLim3Os4IhKAVO5+6pqzetKzU3t+8coK9tdpYzER+XJU7n4q\nOiKcuy8ZwKYde/nrnHVexxGRAKNy92On9+rEpUPTefjD9awr2+N1HBEJICp3P3fbV7NpFxnOz1/W\nxmIi0noqdz+XEh/NT87vx4INO3h56Rav44hIgFC5B4DJwzMZmpnI3a8XUrVXG4uJSMtU7gEgLMz4\n9SUD2bWvjt+/s8rrOCISAFTuASKnawJXnZ7FPz8tZtGmnV7HERE/p3IPIDee24cuHWK4/aXl1Dc0\neh1HRPyYyj2AxEVHcMdF/Vm1fTdPfFzkdRwR8WMq9wBzXv80xvdL5d731rB11z6v44iIn1K5Bxgz\n45cX96fROX71WoHXcUTET6ncA1BGUiw3jO/DOwWlzFpZ6nUcEfFDLZa7mT1uZmVmtuIoP+9oZi+Z\n2TIzyzOzAb6PKV909Zge9EmL445XC9hbW+91HBHxM625c38CmHCMn98GLHXODQKuBP7ig1zSgsjw\nMO6+ZCBbdu3jvllrvY4jIn6mxXJ3zs0FKo8xJAd4v3nsKiDLzNJ8E0+OZUSPJCaPyGTG3A28vWK7\n13FExI/4Ys79M+BrAGY2AugOdPPB60or3HFRDkMyErnpX0sp3FbtdRwR8RO+KPffAYlmthS4DlgC\nHPF0CTObYmb5ZpZfXq4ThnwhJjKcGVcMIyEmkqufzKdizwGvI4mIHzjhcnfOVTvnrnLODaFpzj0F\n2HCUsTOcc7nOudyUlJQTvbQ0S02IYcaVw6jYc4Af/WMxtfX69KpIqDvhcjezRDOLav7yamCuc07z\nAyfZoG6J/PHyweQVVfKLV7T3u0ioi2hpgJnNBMYCncxsM3AHEAngnJsOZANPmpkDCoDvt1laOaaL\nBndl9fbdPDhnHf06x/Pd0T28jiQiHmmx3J1zk1v4+QKgj88SyQm56dw+rC7dzV1vFNIrNZ4zenfy\nOpKIeECfUA0yYWHGn78xhF4pcfzomUVsrKjxOpKIeEDlHoTioiN49Du5RISHcfWTC6ner9ObREKN\nyj1IZSTF8tdvncqmHXu57p9LaGjUA1aRUKJyD2KjeiZz58QBfLimnN+/reP5REJJiw9UJbB9c2Qm\nq7ZXM2PuBvqkxXPZMH14WCQU6M49BPz8whxOPyWZ215crvNXRUKEyj0ERIaH8ddvnUqXxBiueXqR\nTnASCQEq9xCRGBvFo1fmsr+ugSlP57Ov9ojb/4hIkFC5h5DeafHcP3kIBVurueXfn2mLApEgpnIP\nMWf3S+MnE/rxxrJtPPj+Oq/jiEgb0WqZEHTNmT1Zs303f3pvDb3T4pkwoLPXkUTEx3TnHoLMjN98\nbSCDdciHSNBSuYeomMhwHrliGPExETrkQyQIqdxDWGpCDI9cmatDPkSCkMo9xA3qlsgfLhtEXlEl\nd7yqQz5EgoUeqAoTh6SzpnQ3D81ZT980HfIhEgx05y4A3HxuX87NSeOuNwqZt7bC6zgicoJU7gIc\nfsjHtH8u1iEfIgFO5S4HfX7IR5ihQz5EApzKXQ6TkRTLw98exqYde7l+pg75EAlUKnf5L6N6JvOr\nif35YLUO+RAJVFotI0f0rZHdWbVttw75EAlQunOXo/rFRTmc1rPpkI/FxTrkQySQqNzlqA495GPK\nU4vYVqVDPkQChcpdjqlj+ygeaT7k4wdP6ZAPkUChcpcW9UmL5y+Tmg75uFWHfIgEBJW7tMr47DT+\n97x+vK5DPkQCglbLSKtNPasna0p1yIdIINCdu7SamfFbHfIhEhBU7vKlxESGM+OQQz526JAPEb+k\ncpcvLS0hhhlXNB3y8UMd8iHil1osdzN73MzKzGzFUX7ewcxeM7PPzKzAzK7yfUzxN4MzdMiHiD9r\nzZ37E8CEY/x8GrDSOTcYGAv8ycyiTjya+LuJQ9L50dhTmJlXwlMLNnkdR0QO0WK5O+fmApXHGgLE\nm5kBcc1j630TT/zdLV/pyznZqdz5+krmrCrzOo6INPPFnPuDQDawFVgO3OCc0yRsiAgLM+6bNJQ+\nafF8/8mF/O3D9ZqiEfEDvij384ClQFdgCPCgmSUcaaCZTTGzfDPLLy8v98GlxR/ERUfw/NTTmDCg\nM799axU//MdiduugDxFP+aLcrwJedE3WARuBfkca6Jyb4ZzLdc7lpqSk+ODS4i/ioiN46Jun8rML\nsnmvsJSJD81nbelur2OJhCxflHsxMB7AzNKAvsAGH7yuBBgz4+oxPXnm6pFU76tj4kPzee2zrV7H\nEglJrVkKORNYAPQ1s81m9n0zm2pmU5uH3AWcbmbLgdnAT5xzFW0XWfzdqJ7JvH7dGLK7JHDdzCXc\n+dpK6hr0GEbkZDKvHn7l5ua6/Px8T64tJ0dtfSO/ebOQJz4uYnhWRx765qmkJsR4HUskoJnZIudc\nbkvj9AlVaTNREWH88uL+/GXSEFZsqeaCB+axsOhYq2pFxFdU7tLmJg5J56Vpp9M+KpzJMz7h8Xkb\ntVxSpI2p3OWk6Nc5gVevO4Nx/Zo+8HTdzCXUHNBn3UTaispdTpqEmEj+9u1h3HpeX95cvo1LHprP\n+vI9XscSCUoqdzmpwsKMaeN68dT3RrKjppaJD87n7RXbvY4lEnRU7uKJM3p34rXrzuCUlPZM/cci\nfvfWKuq1XFLEZ1Tu4pn0xHb8a+ppfGtkJtM/XM8Vj+VRocM/RHxC5S6eio4I59eXDuSeywaxuHgn\nF94/j8XFO72OJRLwVO7iFy7PzeDFH51OZITxjb8t4OlPNmm5pMgJULmL3+jftQOvXzuGM3p14ucv\nr+Dm5z9jX22D17FEApLKXfxKh9hIHvvOcH58Th9eWrKFS/86n007aryOJRJwVO7id8LCjBvO6c3j\n3x3Otqr9XPjAPGYXlnodSySgqNzFb43rm8rr151BZlIs338yn3vfXU1Do+bhRVpD5S5+LSMplhd+\neDqXD+vG/e+v46onFrKzptbrWCJ+T+Uufi8mMpw/XDaI335tIJ+s38GFD8xj+eYqr2OJ+DWVuwQE\nM2PyiEyen3oazjm+Pv1jnltY7HUsEb+lcpeAMjgjkdevH8PIHkn85IXl/N8Ly9hfp+WSIl+kcpeA\nk9Q+iieuGsG143rx7MISLp++gJLKvV7HEvErKncJSOFhxi3n9eWRK3Mpqqjhogfn8eGacq9jifgN\nlbsEtHNz0njtujPonBDDd/+ex/2z19Ko5ZIiKncJfFmd2vPSj0ZzyZB07n1vDRMfms/8dRVexxLx\nlMpdgkK7qHDu/Z/B3PeNIVTW1PKtRz/lysfzKNiqJZMSmlTuEjTMjEuGpjP75rP42QXZLNu8iwvu\nn8eNzy7RA1cJOebVtqq5ubkuPz/fk2tLaKjaV8f0D9fz+LyNOAffHtWda8/uRVL7KK+jiRw3M1vk\nnMttcZzKXYLd9qr93DdrDf/KL6F9VARTx57C90b3oF1UuNfRRL40lbvIF6wt3c0f3lnNeytLSY2P\n5sfn9uHyYd2ICNfspASO1pa7/quWkNE7LZ5Hrszl+amnkZEUy09fXM55983lnYLtOvVJgo7KXULO\n8Kwk/j31NGZcMQyAa55exGXTF7CwqNLjZCK+o3KXkGRmfKV/Z9658Ux+97WBbN65l8unL+DqJ/NZ\nW7rb63giJ0xz7iLAvtoGHp+/kekfrKemtp7Lh2Vw47m96dKhndfRRA6jB6oix6GyppaH5qzj6QWb\nMIOrRvfgh2NPoUO7SK+jiQA+fKBqZo+bWZmZrTjKz281s6XNv1aYWYOZJR1PaBGvJbWP4ucX5jD7\n5rP46sAu/G3ues78wxwembtBWwtLQGnxzt3MzgT2AE855wa0MPYi4MfOubNburDu3CUQFGyt4vdv\nr2bumnLSE9tx07l9uGRoOuFh5nU0CVE+u3N3zs0FWruMYDIws5VjRfxe/64deOp7I3jm6pEktY/i\n5uc/44L7P2LO6jItnxS/5rPVMmYWC0wAXjjGmClmlm9m+eXl2ntbAsfoXp14ZdpoHpg8lL21DVz1\n94VMfuQTPivZ5XU0kSPy5VLIi4D5zrmj3uU752Y453Kdc7kpKSk+vLRI2wsLMy4a3JVZN53Fry7u\nz9rSPUx8aD7TnlnMxooar+OJHCbCh681CU3JSAiIigjjO6dn8fVh3ZgxdwOPfrSBdwq2M3lEJteP\n701KfLTXEUV8c+duZh2As4BXfPF6IoEgLjqCm87twwe3jmXyiExm5hVz1j1z+PN7a9hzoN7reBLi\nWrNaZiYwFugElAJ3AJEAzrnpzWO+C0xwzk1q7YW1WkaCzcaKGv74zmreWL6N5PZRTBqRwaThmWQk\nxXodTYKIPsQk4pGlJbt4YPZa5qwuo9HBmN6dmDQ8k3Nz0oiK0I4fcmJU7iIe27prH8/nb+a5hcVs\nrdpPcvsoLhvWjW8Mz6BnSpzX8SRAqdxF/ERDo2Pu2nKezStmVmEZDY2OkT2S+ObITM7r35mYSB0a\nIq2nchfxQ2XV+3l+0WaeW1hCceVeEmMjuXRoOpNHZNInLd7reBIAVO4ifqyx0fHx+h3MXFjMuwXb\nqWtwDOvekUnDM7hwUFcdAShHpXIXCRA79hzgxcVbmJlXzIaKGuKjI7hkaDqTRmTQv2sHr+OJn1G5\niwQY5xx5Gyt5dmEJbyzfRm19I4O6dWDyiEwuGtyVuGhffuZQApXKXSSA7dpby0tLtvBsXgmrS3cT\nGxXOxYO7MnlEJoO6dcBMu1KGKpW7SBBwzrGkZBfP5hXz2mfb2FfXQHaXBCaPyGDikHQdIhKCVO4i\nQWb3/jpeWbqVZxcWs2JLNTGRYVwwsCuTR2QwrHtH3c2HCJW7SBBbvrmKmQuLeXXpVvYcqKd3ahyT\nRmTytaHpdGwf5XU8aUMqd5EQUHOgnjeWbeOfecUsLdlFVHgYEwZ0ZtKIDE7rmay7+SCkchcJMau2\nV/NsXgkvLt5M9f56spJjuTw3gwkDOnOKtjsIGip3kRC1v66BN5dv49m8EvKKms7O6dGpPeP7pTI+\nO43hWR2JCNcGZoFK5S4ibN65l/dXlTGrsIxP1u+gtqGRhJgIxvZNZXx2KmP7pmrFTYBRuYvIYfYc\nqGfe2nJmFZYxZ1UZO2pqiQgzhmclMT47lXOy08jq1N7rmNIClbuIHFVDo2NpyS5mF5Yyu7CM1aW7\nATglpT3nZKcxPjuNUzMTNX3jh1TuItJqJZV7mdVc9J9u3EFdgyMxNpJxzdM3Z/ZJISFG0zf+QOUu\nIsdl9/465q6pYHZhKXNWl7Fzbx2R4cbIHskHp290dKB3VO4icsIaGh2Li3cya2UpswpLWV9eA0Cf\ntDjGZ6dxTnYqQzI6Eh6m9fQni8pdRHyuqKLm4PRNXlElDY2O5PZRjOuXyjnZqYzpnUJ77V7ZplTu\nItKmqvbV8eGa8qbpm1VlVO+vJyo8jFGnJHNOdtOa+vTEdl7HDDoqdxE5aeoaGskv2tm0+mZVGRsr\nmqZv+nWO55zsNMb1S2FgeiJREVp9c6JU7iLimfXle5hdWMqswjLyiyppdBAdEcaQjERG9EgiNyuJ\nUzMTidcKnC9N5S4ifmFnTS2fbtxB3sad5G+qpGBrNQ2NjjCDnK4J5HZPai78jqTGx3gd1++p3EXE\nL+05UM+S4p0sLNrJwo2VLCnZyf66RgCykmMZnpXU9KtHElnJsdrZ8gtaW+56rC0iJ1VcdARjeqcw\npncKALX1jRRsrWJhUSULi3Yyq7CU5xdtBqBTXDTDszoeLPzsLvH61Gwr6c5dRPxKY6NjQ8Ue8jbu\nbC78Sjbv3AdA+6hwTu3+n7IfkpFIu6hwjxOfXJqWEZGgsa1qH3kbK8kvair81aW7cQ4iw40B6R0O\nln1u945BfxKVyl1EglbV3joWFVc2PaQtqmTZ5ipqG5rm7XunxjG8R9LB6ZxuHYNrqwSVu4iEjP11\nDSzb3DRvn7exksWbdrL7QD0AXTvEkNv8gHZoRiK90+KIjgjcqRyVu4iErIZGx6rt1eQX7SSvqJKF\nGysp230AaJrK6ZUaT/+uCc2/OpDdJT5g1tz7rNzN7HHgQqDMOTfgKGPGAvcBkUCFc+6sli6scheR\nk8U5R3HlXpZtrmLltmoKtlazcmsVFXtqD47pnhx7sOxzuibQv0sCqQn+t+7el+V+JrAHeOpI5W5m\nicDHwATnXLGZpTrnylq6sMpdRLzknKNs9wEKtlaxcmtT4Rdsraa4cu/BMZ3iog/e4ec0F3/3pFjC\nPNwF02fr3J1zc80s6xhDvgm86Jwrbh7fYrGLiHjNzEhLiCEtIYaz+6Ud/H71/joKDyn7gq1VzF9X\nQX1j041wXHQE2V3im+7wuzSVfp+0eL/bN8cXH2LqA0Sa2QdAPPAX59xTRxpoZlOAKQCZmZk+uLSI\niG8lxEQysmcyI3smH/zegfoG1pbuoWBr1cHS/1d+CXtrG4CmefzezfP4OX4yj9+qB6rNd+6vH2Va\n5kEgFxgPtAMWABc459Yc6zU1LSMigayx0VG0o+Zg2a/c9t/z+FnJsQfLPqd5eudE9885mdsPbAZ2\nOOdqgBozmwsMBo5Z7iIigSwszOiZEkfPlDguGtwVOHwev2BLU+Gv2FLNm8u3H/xzKfHRTBnTkx+c\n2bNN8/mi3F8BHjSzCCAKGAn82QevKyISUI41j79ya/XBB7epCdFtnqXFcjezmcBYoJOZbQbuoGnJ\nI8656c65QjN7G1gGNAKPOudWtF1kEZHAkhATyaieyYw6ZB6/rbVmtczkVoy5B7jHJ4lEROSE+dfa\nHRER8QmVu4hIEFK5i4gEIZW7iEgQUrmLiAQhlbuISBBSuYuIBCHPDusws3Jg03H+8U5AhQ/jBDq9\nH4fT+/Efei8OFwzvR3fnXEpLgzwr9xNhZvmt2TgnVOj9OJzej//Qe3G4UHo/NC0jIhKEVO4iIkEo\nUMt9htcB/Izej8Pp/fgPvReHC5n3IyDn3EVE5NgC9c5dRESOIeDK3cwmmNlqM1tnZv/ndR4vmVmG\nmc0xs5VmVmBmN3idyWtmFm5mS8zsda+zeM3MEs3s32a2yswKzew0rzN5xcx+3Px3ZIWZzTSzEzvr\nLgAEVLmbWTjwEHA+kANMNrMcb1N5qh642TmXA4wCpoX4+wFwA1DodQg/8RfgbedcP5qOvgzJ98XM\n0oHrgdzmc6DDgUnepmp7AVXuwAhgnXNug3OuFngWmOhxJs8457Y55xY3/343TX95071N5R0z6wZc\nADzqdRavmVkH4EzgMQDnXK1zbpe3qTwVAbRrPg40FtjqcZ42F2jlng6UHPL1ZkK4zA5lZlnAUOBT\nb5N46j7gf2k67jHU9QDKgb83T1M9ambtvQ7lBefcFuCPQDGwDahyzr3rbaq2F2jlLkdgZnHAC8CN\nzrlqr/N4wcwuBMqcc4u8zuInIoBTgYedc0OBGiAkn1GZWUea/oXfA+gKtDezb3ubqu0FWrlvATIO\n+bpb8/dClplF0lTszzjnXvQ6j4dGAxebWRFN03Vnm9k/vI3kqc3AZufc5/+S+zdNZR+KzgE2OufK\nnXN1wIvA6R5nanOBVu4Lgd5m1sPMomh6KPKqx5k8Y2ZG05xqoXPuXq/zeMk591PnXDfnXBZN/128\n75wL+ruzo3HObQdKzKxv87fGAys9jOSlYmCUmcU2/50ZTwg8XI7wOsCX4ZyrN7NrgXdoeuL9uHOu\nwONYXhoNXAEsN7Olzd+7zTn3poeZxH9cBzzTfCO0AbjK4zyecM59amb/BhbTtMJsCSHwSVV9QlVE\nJAgF2rSMiIi0gspdRCQIqdxFRIKQyl1EJAip3EVEgpDKXUQkCKncRUSCkMpdRCQI/T83BOfElDuf\nuQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7ff78fd57160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(losses)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(rnn.state_dict(), '../models/ner.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model,X_test,y_test):\n",
    "    model.eval()\n",
    "    correct = 0.0\n",
    "    count = 0.0\n",
    "    print(np.array(X_test).shape)\n",
    "    for i in range(len(X_test)):\n",
    "\n",
    "        hidden = model.initHidden()\n",
    "        for j in range(len(X_test[i])):\n",
    "            count += 1\n",
    "            inputs = Variable(torch.Tensor(X_test[i][j]).unsqueeze(0))\n",
    "            output, hidden = model(inputs, hidden)\n",
    "#             print(output.topk(1)[1].data[0][0])\n",
    "            if output.topk(1)[1].data[0][0] == y_test[i][j]:\n",
    "                correct += 1\n",
    "            \n",
    "    print(correct)\n",
    "    print(count)\n",
    "    print('Test Acc.', correct/ count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1405,)\n",
      "21685.0\n",
      "24803.0\n",
      "Test Acc. 0.8742894004757489\n"
     ]
    }
   ],
   "source": [
    "# for i in range(len(X_test[:1])):\n",
    "#     for j in range(len(X_test[i])):\n",
    "#         print(X_test[i][j])\n",
    "\n",
    "\n",
    "evaluate(rnn,X_test,y_test)\n",
    "# print(y_test[2:3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Test Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('belgium', 0.9981899261474609),\n",
       " ('toyota', 0.9979879856109619),\n",
       " ('stated', 0.997677743434906),\n",
       " ('russia', 0.9975883364677429),\n",
       " ('unless', 0.9973466992378235),\n",
       " ('philippoussis', 0.9972662329673767),\n",
       " ('u.s.', 0.9969862699508667),\n",
       " ('metres', 0.9967542886734009),\n",
       " ('yamaha', 0.9964815974235535),\n",
       " ('4-0', 0.9964807033538818)]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv_model = Word2Vec.load('../models/ner_word_to_vec50')\n",
    "wv_model.most_similar('britain')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare testing data  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "''' some redundant code will cleanup later its 3:41 AM XD  '''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "with open('../datasets/NER/ner_data/test') as f:\n",
    "    lines = f.readlines()\n",
    "\n",
    "lines = ''.join(lines)\n",
    "lines = lines.lower()\n",
    "lines = ''.join(lines)\n",
    "lines = re.sub('-docstart- -x- -x- o', '', lines)\n",
    "# print(lines[:1])\n",
    "split_lines = lines.split('\\n\\n')\n",
    "split_lines = [line for line in split_lines if line]\n",
    "sent_lines = []\n",
    "for splits in split_lines:\n",
    "    sent_lines.append(splits.split('\\n'))\n",
    "\n",
    "# print(len(split_lines))\n",
    "# print(split_lines[:10])\n",
    "\n",
    "# print(len(sent_lines))\n",
    "# print(sent_lines[:10])\n",
    "flat_para = []\n",
    "X_words = []\n",
    "X_pos1 = []\n",
    "X_pos2 = []\n",
    "y_ner = []\n",
    "\n",
    "for para in sent_lines:\n",
    "#     print(para)\n",
    "#     chunks = para.split('\\n')\n",
    "#     print(chunks)\n",
    "    words = []\n",
    "    pos1 = []\n",
    "    pos2 = []\n",
    "    ner = []\n",
    "\n",
    "    for part in para:\n",
    "\n",
    "        if part:\n",
    "            word, pos1_tag, pos2_tag, ner_tag = part.split()\n",
    "            words.append((word,pos1_tag,pos2_tag))\n",
    "            pos1.append(pos1_tag)\n",
    "            pos2.append(pos2_tag)\n",
    "            ner.append(ner_tag)\n",
    "    if words:\n",
    "        X_words.append(words)\n",
    "\n",
    "        y_ner.append(ner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3453 3453\n"
     ]
    }
   ],
   "source": [
    "print(len(X_words),len(y_ner))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([('soccer', 'nn', 'i-np'),\n",
       "  ('-', ':', 'o'),\n",
       "  ('japan', 'nnp', 'i-np'),\n",
       "  ('get', 'vb', 'i-vp'),\n",
       "  ('lucky', 'nnp', 'i-np'),\n",
       "  ('win', 'nnp', 'i-np'),\n",
       "  (',', ',', 'o'),\n",
       "  ('china', 'nnp', 'i-np'),\n",
       "  ('in', 'in', 'i-pp'),\n",
       "  ('surprise', 'dt', 'i-np'),\n",
       "  ('defeat', 'nn', 'i-np'),\n",
       "  ('.', '.', 'o')],\n",
       " ['o', 'o', 'i-loc', 'o', 'o', 'o', 'o', 'i-per', 'o', 'o', 'o', 'o'],\n",
       " 12)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_words[0],  y_ner[0], len(y_ner[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12, 52)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"Prepare test tensor\"\"\"\n",
    "def val_to_tensor(value):\n",
    "    tensor = []\n",
    "    for val in value:\n",
    "        \n",
    "        wv = word_to_tensor(val[0]).tolist()\n",
    "        wv.append(pos1_to_ix[val[1]])\n",
    "        wv.append(pos2_to_ix[val[2]])\n",
    "        tensor.append(wv)\n",
    "    return tensor\n",
    "np.array(val_to_tensor(X_words[0])).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_set = [val_to_tensor(X) for X in X_words]\n",
    "y_test_set = convert_y_vec(y_ner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3453\n"
     ]
    }
   ],
   "source": [
    "print(len(X_test_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3453,)\n"
     ]
    }
   ],
   "source": [
    "print(np.array(y_test_set).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "INPUT_SIZE = 52\n",
    "HIDDEN_SIZE = 128\n",
    "OUTPUT_SIZE = 8\n",
    "learning_rate = 5e-3\n",
    "model = RNN(INPUT_SIZE, HIDDEN_SIZE, OUTPUT_SIZE)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_state_dict(torch.load('../models/ner.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3453,)\n",
      "38250.0\n",
      "46435.0\n",
      "Test Acc. 0.8237320986324971\n"
     ]
    }
   ],
   "source": [
    "evaluate(model, X_test_set, y_test_set)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}