Hindi-Language-Modeling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hindi2Vec Language Modeling\n",
    "\n",
    "The goal of this notebook is to train Hindi word embeddings using the [fast.ai](http://www.fast.ai/) version of [AWD LSTM Language Model](https://arxiv.org/abs/1708.02182)--basically LSTM with dropouts--with data from [Wikipedia](https://dumps.wikimedia.org/hiwiki/latest/hiwiki-latest-pages-articles.xml.bz2) (pulled on March 2, 2018). \n",
    "\n",
    "We achieved perplexity of 46.81, compared to [state-of-the-art on November 17, 2017](https://github.com/RedditSota/state-of-the-art-result-for-machine-learning-problems) at 40.68 for English language. To the best of our knowledge, there is no comparable research in Hindi language at the point of writing (March 2, 2018)."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import dill as pickle\n",
    "import json\n",
    "from IPython.display import Image\n",
    "from IPython.core.display import HTML\n",
    "\n",
    "from spacy.lang.hi import *\n",
    "\n",
    "import re\n",
    "import torchtext\n",
    "from torchtext import vocab, data\n",
    "from torchtext.datasets import language_modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from fastai.learner import *\n",
    "from fastai.rnn_reg import *\n",
    "from fastai.rnn_train import *\n",
    "from fastai.nlp import *\n",
    "from fastai.lm_rnn import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-25T03:30:25.407566Z",
     "start_time": "2018-01-25T03:30:21.597641Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "PATH='C:\\\\Users\\\\nirantk\\\\Desktop\\\\hindi-wiki\\\\'\n",
    "# path to the xml wikipedia dump\n",
    "\n",
    "EXT_PATH = 'extract\\\\'\n",
    "TRN_PATH = 'train\\\\'\n",
    "VAL_PATH = 'valid\\\\'\n",
    "SAMPLE_PATH = 'sample\\\\'\n",
    "\n",
    "EXT = f'{PATH}{EXT_PATH}'\n",
    "TRN = f'{PATH}{TRN_PATH}'\n",
    "VAL = f'{PATH}{VAL_PATH}'\n",
    "SAMPLE = f'{PATH}{SAMPLE_PATH}'\n",
    "\n",
    "ext_files = !ls {EXT}\n",
    "sample_files = !ls {SAMPLE}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-16T10:16:59.532902Z",
     "start_time": "2018-01-16T10:16:59.460836Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Exact command line statement used to extract content from xml file\n",
    "# !python wikiextractor/WikiExtractor.py hiwiki-latest-pages-articles.xml -o extract -b 10M --ignored_tags abbr,b,big --discard_elements gallery,timeline,noinclude"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save as Plain Text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "%%prun\n",
    "\n",
    "\n",
    "def clean_files(extracted_filelist, TRN):    \n",
    "    cleaned_all = []\n",
    "    for ext_file in extracted_filelist:\n",
    "        input_file = f'{EXT}{ext_file}'\n",
    "        with open(input_file,'r', encoding='utf-8') as f:\n",
    "            raw_txt = f.readlines()\n",
    "            cleaned_doc = []\n",
    "            for line in raw_txt:\n",
    "                new_line = re.sub('<[^<]+?>', '', line)\n",
    "                new_line = re.sub('__[^<]+?__', '', new_line) \n",
    "                new_line = new_line.strip()\n",
    "                if new_line != '':\n",
    "                    cleaned_doc.append(new_line)\n",
    "\n",
    "            new_doc = \"\\n\".join(cleaned_doc)\n",
    "            cleaned_all.append(new_doc)\n",
    "            with open(f\"{TRN}{ext_file}.txt\", \"w\", encoding='utf-8') as text_file:\n",
    "                text_file.write(new_doc)\n",
    "    return cleaned_all\n",
    "\n",
    "cleaned_all = clean_files(ext_files, TRN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(f'Preview:\\n{cleaned_all[0][:500]}\\n\\nLength of list (should be equal to number of documents): {len(cleaned_all)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Validation Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-17T15:19:28.662393Z",
     "start_time": "2018-01-17T15:19:28.603161Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "random.shuffle(trn_files)\n",
    "len_valid = int(0.2 * len(trn_files)) \n",
    "val_files = trn_files[:len_valid]\n",
    "trn_files = trn_files[len_valid:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-17T15:19:34.451766Z",
     "start_time": "2018-01-17T15:19:34.392211Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# # WARNING: Run exactly ONCE\n",
    "\n",
    "# import shutil, os\n",
    "# for root, dirs, files in os.walk(TRN):\n",
    "#     for file in files:\n",
    "#         if file.endswith(\".txt\") & (file in val_files):\n",
    "#             shutil.move(os.path.join(root, file),VAL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-17T15:20:30.038345Z",
     "start_time": "2018-01-17T15:20:29.934723Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['wiki_00.txt', 'wiki_01.txt', 'wiki_04.txt', 'wiki_05.txt', 'wiki_06.txt', 'wiki_07.txt', 'wiki_08.txt', 'wiki_11.txt', 'wiki_12.txt', 'wiki_13.txt', 'wiki_14.txt', 'wiki_15.txt', 'wiki_16.txt', 'wiki_17.txt', 'wiki_19.txt', 'wiki_20.txt', 'wiki_21.txt', 'wiki_22.txt', 'wiki_24.txt', 'wiki_25.txt', 'wiki_26.txt', 'wiki_28.txt', 'wiki_29.txt', 'wiki_30.txt', 'wiki_31.txt', 'wiki_32.txt', 'wiki_33.txt', 'wiki_34.txt', 'wiki_35.txt', 'wiki_36.txt']\n",
      "['wiki_02.txt', 'wiki_03.txt', 'wiki_10.txt', 'wiki_23.txt']\n",
      "30\n",
      "4\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(None, None, None, None)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trn_files = !ls {TRN}\n",
    "val_files = !ls {VAL}\n",
    "print(trn_files), print(val_files), print(len(trn_files)), print(len(val_files))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Processing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def word_tokenize(document):\n",
    "    nlp = Hindi() # from spacy.lang.hi\n",
    "    return [token.text for token in nlp(document)]\n",
    "    \n",
    "def docs_tokenize(documents_as_lists):   \n",
    "    for document in documents_as_lists:\n",
    "        tokens = word_tokenize(document)\n",
    "        tokens_list.extend(tokens)\n",
    "    \n",
    "    return tokens_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading from tokens_list.txt\n",
      "Found 30598557 tokens\n",
      "Wall time: 16 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tokens_filename = \"tokens_list.txt\"\n",
    "tokens_list = []\n",
    "\n",
    "#TODO refactor from try except blocks to if else with if statement checking if file exists using Pathlib\n",
    "\n",
    "try:\n",
    "    print(f'Reading from {tokens_filename}')\n",
    "    with open(tokens_filename, \"r\") as f:\n",
    "         tokens_list = json.load(f)\n",
    "    \n",
    "except FileNotFoundError:\n",
    "    print(f'FileNotFound. Trying to tokenize from cleaned_all')\n",
    "    tokens_list = docs_tokenize(cleaned_all)\n",
    "    \n",
    "    with open('tokens_list.txt', 'w') as outfile:\n",
    "        json.dump(tokens_list, outfile)\n",
    "\n",
    "print(f'Found {len(tokens_list)} tokens') # 30598557 tokens expected"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Min Count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T15:51:18.836570Z",
     "start_time": "2018-01-24T15:51:18.535781Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# TEXT = pickle.load(open(f'{PATH}models/TEXT.pkl','rb'))\n",
    "# freqs = pd.DataFrame.from_dict(TEXT.vocab.freqs,orient='index')\n",
    "# freqs.sort_values(0,ascending=False).head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T15:51:57.284742Z",
     "start_time": "2018-01-24T15:51:57.143426Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# cnt = []\n",
    "# for i in range(49):\n",
    "#     row_cnt = freqs[freqs[0]>=i+1].shape[0]\n",
    "#     cnt.append(row_cnt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T15:51:58.695888Z",
     "start_time": "2018-01-24T15:51:58.523805Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# plt.plot(cnt)\n",
    "# plt.axvline(x=10,color='red', linestyle='--')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T15:52:42.170567Z",
     "start_time": "2018-01-24T15:52:42.118686Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# #number of legitimate thai words at freqs\n",
    "# freqs[freqs[0]<=10].sort_values(ascending=False,by=0).head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data Loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "assert torch.cuda.is_available()\n",
    "assert torch.backends.cudnn.enabled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T16:04:14.088043Z",
     "start_time": "2018-01-24T16:04:14.060994Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TEXT = data.Field(lower=True, tokenize=word_tokenize)\n",
    "#batch size\n",
    "bs=16\n",
    "#backprop through time\n",
    "bptt=70"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T16:41:27.289827Z",
     "start_time": "2018-01-24T16:04:14.892151Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#FILES = dict(train=f'{SAMPLE_PATH}', validation=f'{SAMPLE_PATH}', test=f'{SAMPLE_PATH}')\n",
    "FILES = dict(train=f'{TRN_PATH}', validation=f'{VAL_PATH}', test=f'{SAMPLE_PATH}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 21min 20s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "md = LanguageModelData.from_text_files(PATH, TEXT, **FILES, bs=bs, bptt=bptt, min_freq=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 751 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "pickle.dump(TEXT, open(f'{PATH}//models//TEXT_min_freq50.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21791, 21821, 1, 24407648)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(md.trn_dl), md.nt, len(md.trn_ds), len(md.trn_ds[0].text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<unk>', '<pad>', 'के', '।', ',', 'में', 'है', 'की', 'और', 'से', 'का', 'को']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab.itos[:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "123"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "TEXT.vocab.stoi['सभी']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['हम',\n",
       " 'होंगे',\n",
       " 'कामयाब',\n",
       " 'दैनिक',\n",
       " 'पूजा',\n",
       " 'दैनिक',\n",
       " 'पूजा',\n",
       " 'विधि',\n",
       " 'हिन्दू',\n",
       " 'धर्म',\n",
       " 'की',\n",
       " 'कई']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "md.trn_ds[0].text[:12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(Variable containing:\n",
       "    430      3   2017  ...     997    607    745\n",
       "   1862    990   2414  ...       2      4      2\n",
       "   4871      0    343  ...    2576    540    163\n",
       "         ...            ⋱           ...         \n",
       "      0      5     23  ...       8    540      4\n",
       "      4      0      3  ...      81      4    415\n",
       "    648      0   7395  ...    1472     88    281\n",
       " [torch.cuda.LongTensor of size 69x16 (GPU 0)], Variable containing:\n",
       "   1862\n",
       "    990\n",
       "   2414\n",
       "   ⋮   \n",
       "   6439\n",
       "     90\n",
       "     12\n",
       " [torch.cuda.LongTensor of size 1104 (GPU 0)])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(iter(md.trn_dl))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T16:41:27.379231Z",
     "start_time": "2018-01-24T16:41:27.336150Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       "  430\n",
       " 1862\n",
       " 4871\n",
       " 1559\n",
       " 1007\n",
       " 1559\n",
       " 1007\n",
       "  565\n",
       "  589\n",
       "  202\n",
       "[torch.cuda.LongTensor of size 10x1 (GPU 0)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#trn_ds is list; one for each txt file\n",
    "txt = md.trn_ds[0].text[:10]\n",
    "TEXT.numericalize([txt])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Language Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T16:41:27.406405Z",
     "start_time": "2018-01-24T16:41:27.380815Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "em_sz = 300  # size of each embedding vector\n",
    "nh = 500     # number of hidden activations per layer\n",
    "nl = 3       # number of layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T16:41:27.429580Z",
     "start_time": "2018-01-24T16:41:27.407646Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "opt_fn = partial(optim.Adam, betas=(0.7, 0.99))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T16:41:27.669131Z",
     "start_time": "2018-01-24T16:41:27.431195Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner = md.get_model(opt_fn, em_sz, nh, nl,\n",
    "               dropouti=0.05, dropout=0.05, wdrop=0.1, dropoute=0.02, dropouth=0.05)\n",
    "learner.reg_fn = partial(seq2seq_reg, alpha=2, beta=1)\n",
    "learner.clip=0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T16:42:11.341458Z",
     "start_time": "2018-01-24T16:41:27.701583Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a87d17d411104ab1958cafb18609fea7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 76%|█████████████████████████████████████████████████▌               | 16635/21791 [20:59<06:30, 13.21it/s, loss=18.8]"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEOCAYAAACEiBAqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VNX5x/HPk0kIS8KihF1EBETFPa64oKLiUq1at2rdi9rWfSlUf65VqUuttnWhitpW0Sq4KyAi4soqCCigAiLIvpN1MnN+f9ybkMQEEsidO5P5vl+vNHed8zAd58k5555zzDmHiIikr4ywAxARkXApEYiIpDklAhGRNKdEICKS5pQIRETSnBKBiEiaUyIQEUlzSgQiImlOiUBEJM0pEYiIpLnMsAOoi7Zt27pu3bqFHYaISEqZOnXqKudc3tauS4lE0K1bN6ZMmRJ2GCIiKcXMfqjLdWoaEhFJc4ElAjMbZmYrzGxWpWP7mtkXZjbdzKaY2UFBlS8iInUTZI3gOWBAtWMPAHc55/YFbvf3RUQkRIElAufcBGBN9cNAS3+7FfBTUOWLiEjdJLqz+DpgtJk9hJeEDktw+SIiUk2iO4uvAq53zu0EXA88U9uFZjbQ70eYsnLlyoQFKCKSbhKdCC4CRvrbrwC1dhY754Y65/Kdc/l5eVt9DFZEpFFZV1jKmNnLWLWpJPCyEp0IfgKO8rePAb5NcPkiIinh+5UFDPzPVGYtWR94WYH1EZjZcKAf0NbMFgN3AL8FHjWzTKAYGBhU+SIiqcw5B0AkwwIvK7BE4Jw7r5ZTBwRVpohIYxGLe4kgw4JPBBpZLCKShGJOiUBEJK35eSAhTUNKBCIiSWhz01DwZSkRiIgkofJEoBqBiEiaisbiAGRFgv+aViIQEUlCZX6NIDOiGoGISFoqrxFkZqhGICKSlspiXo0gSzUCEZH0VBb3awTqIxARSU/R8hqBnhoSEUlPZTHVCERE0pqeGhIRSXObm4ZUIxARSUubm4ZUIxARSUvR8qYhdRaLiKSnsliczAzDNA21iEh6Kou7hDQLgRKBiEhSisbiCekohgCXqhQRkW337KcLE1aWagQiImlONQIRkSTUvmU2e3VunZCyAqsRmNkwM1thZrOqHb/azOaa2WwzeyCo8kVEUlnEjDbNsxJSVpBNQ88BAyofMLOjgdOAvZ1zewIPBVi+iEjKKo3FaZKZmNb7wEpxzk0A1lQ7fBUwxDlX4l+zIqjyRURSWUm0ESSCWvQCjjCziWb2kZkdmODyRURSQkkCawSJ7izOBNoAhwAHAv8zs+7OOVf9QjMbCAwE6Nq1a0KDFBEJk3OO0rI42QmYghoSXyNYDIx0nklAHGhb04XOuaHOuXznXH5eXl5CgxQRCVP5zKONtWnodeAYADPrBTQBViU4BhGRpFbqzzya8k1DZjYc6Ae0NbPFwB3AMGCY/0hpKXBRTc1CIiLpbH1RFICWTRPz+GhgicA5d14tpy4IqkwRkcZgfaGXCFo3gnEEIiKyDaIJbhpSIhARSTIVfQSRSELKUyIQEUkypWVeIsjSegQiIukp0U8NKRGIiCSZzTUCJQIRkbRU3lmcrRqBiEh6Uo1ARCTN6fFREZE0pxqBiEiaKylTjUBEJK1VzD6qGoGISHoqVY1ARCS9RWNxIhlGJEMji0VE0lJpLJ6w6SVAiUBEJOmUlsUT1j8ASgQiIkmnNIEL14MSgYhI0omqRiAikt5KY3GyVCMQEUlf6iMQEUlzxdEY2VlKBCIiaaugJEZOdmbCygssEZjZMDNbYWazajh3k5k5M2sbVPkiIqlqU0lZ40gEwHPAgOoHzWwn4DhgUYBli4ikrEaTCJxzE4A1NZx6BLgFcEGVLSKSygpKymjRGBJBTczsVGCJc25GIssVEUklGxNcI0hYSWbWHLgVOL6O1w8EBgJ07do1wMhERJJHaVmc0rJ442gaqsGuwC7ADDNbCHQBpplZh5ouds4Ndc7lO+fy8/LyEhimiEh4iqIxAJo1iSSszISlHOfcTKBd+b6fDPKdc6sSFYOISLIrKfMSQXZW4hJBkI+PDgc+B3Yzs8VmdllQZYmINBYlUW9RmuwETjERWI3AOXfeVs53C6psEZFUVb5ecSITgUYWi4gkkdKKRNAImoZERKT+KvoIVCMQEUlPahoSEUlzFU1Dmn1URCQ9ldcImkTURyAikpY2jyNQjUBEJC2Vqo9ARCS9lejxURGR9FbizzXURDUCEZH0pMdHRUTSXLE/11DTxjDpnIiI1F9RNEaTSAaRDEtYmUoEIiJJpDgaS+ijo6BEICKSVErKYjRLYLMQKBGIiCSVotJYQvsHQIlARCSpFEfjqhGIiKSzNQWlOFxCy0zYmsUiIrJ1kxauSXiZSgQiIkmkaVYGu7XPTWiZahoSEUkiO7bIpkc7JQIRkbRVUFpGTnYj6Sw2s2FmtsLMZlU69qCZzTGzr8zsNTNrHVT5IiKpqKCkjObZiW21D7JG8BwwoNqx94E+zrm9gXnA4ADLFxFJKdFYnGjM0byxPD7qnJsArKl2bIxzrszf/QLoElT5IiKppsifgrpZk0aSCOrgUuC9EMsXEUkqxemUCMzsVqAMeGEL1ww0sylmNmXlypWJC05EJCTFpf4U1AlcnQxCSARmdhFwCnC+c67W4XPOuaHOuXznXH5eXl7iAhQRCUl501Ci5xpKaNe0mQ0A/ggc5ZwrTGTZIiLJbnMfQSOZhtrMhgOfA7uZ2WIzuwz4B5ALvG9m083syaDKFxFJNeV9BIluGqpTjcDMrgWeBTYCTwP7AYOcc2Nqu8c5d14Nh5/ZliBFRNJBeSLITtLHRy91zm0AjgfygEuAIYFFJSKShjavV5ycTUPli2eeBDzrnJtR6ZiIiDSAkrJwOovrmgimmtkYvEQw2sxygXhwYYmIpJ/iJH9q6DJgX2C+c67QzHbAax4SEZEGUtE0lJmcTUOHAnOdc+vM7ALgNmB9cGGJiKSfsGoEdU0ETwCFZrYPcAvwA/DvwKISEUlDmzuLkzMRlPmjgE8DHnXOPYo3HkBERBpIcVmMrIgRyUjsszh17SPYaGaDgd8AR5hZBMgKLiwRkfRTHI0lfDAZ1L1GcA5QgjeeYBnQGXgwsKhERNJQcTSe8MFkUMdE4H/5vwC0MrNTgGLnnPoIREQaUEk0lvDBZFDHRGBmZwOTgLOAs4GJZvarIAMTEUk3xWWxhHcUQ937CG4FDnTOrQAwszxgLPBqUIGJiKSb4mg8eWsEQEZ5EvCtrse9IiJSB8XRGM2SuEYwysxGA8P9/XOAd4MJSUQkPRVFY+RkJ3SZGKCOicA5d7OZnQn0xZtsbqhz7rVAIxMRSTNFpTHycrITXm6dU49zbgQwIsBYRETSWnE0lvCF62EricDMNgI1rStsgHPOtQwkKhGRNFSUjH0EzjlNIyEikiBFpeE8Pqonf0REkkRxNB5K05ASgYhIEiiLxSmNxUNpGlIiEBFJAsVl3hTUjSoRmNkwM1thZrMqHdvBzN43s2/9322CKl9EJJUUlfqL0jSypqHngAHVjg0CPnDO9QQ+8PdFRNJe+epkjapG4JybAKypdvg04Hl/+3ngl0GVLyKSSopCTASJHsvc3jm3FMA5t9TM2gVZ2KhZy5i2aG3Fvvn/Y94WZt4x84+Zbb7OzNvPMNt8TQ3HIhkZZEWMrEiG/1P37SaZGWRmGFmZGTTxzyV6ZSIRSQ6bSsoAaJ5sA8rCZGYDgYEAXbt23abXmLRgDS9O+gHDcDicqzQ6zlHlmLcSZ/n2doe/zczwkkQkg0w/YeRkZ9IuN5tOrZvRqXVTOrZqRuc2zdh/pza0aq6F4kQag3WFpQC0adEk4WUnOhEsN7OOfm2gI7Citgudc0OBoQD5+fnb9NV8+y/24PZf7LFNgTq3OUnEK7b9384/BsRijmg8TlnMEfUf/4rGvP3SWJxoWZyof03Ftn9NbdulsUqvVxZnY0kZKzYUM2nBGpZtKCYW994OM+jTqRXXHtuTY3dvh5lqEyKpak1BFIAdmjf+RPAmcBEwxP/9RoLLr7PyZiCACMnzBRuLO1ZsLGbhqkImL1zD69OXcPm/p3DSXh3469n7hjIqUUS239oCr0bQukXia/mBJQIzGw70A9qa2WLgDrwE8D8zuwxYhLfimdRDJMPo2KoZHVs149Bdd+SqfrsydMJ8Hhw9lw1FU3jm4nyyQ1j8WkS2z9rCUjIzjNxknYZ6Wzjnzqvl1LFBlZmOsiIZ/P7oHrTLzebmV7/inx9+zw3H9Qo7LBGpp7WFpbRu3iSUJl6NLG4kzsrfiV/u24knxn/HsvXFYYcjIvW0tiBKm5Ae/lAiaESu7d+LaMwxYtrisEMRkXpaW1gayhNDoETQqOzStgX7d23N218tDTsUEamndYWqEUgDOWmvjnyzdAPzV24KOxQRqYc1haW0CeHRUVAiaHRO3KsjAKNmLws5EhGpK+cc6/zO4jAoETQynVs3Y+8urRg9e3nYoYhIHRWUxojGnJqGpOGcsGcHZvy4jqXri8IORUTqoHwwmZqGpMEM6NMBgDen/xRyJCJSF+sKveklWqtGIA1l17wcenfI5f735jB80qKwwxGRrVhdUALAjjmqEUgDeuy8/Wibk80db8xm1Cx1HIsksxUbvETQLrdpKOUrETRSvdrn8t61R9CmRRZX/ncqZzz+KSs2asSxSLIpjsa4ZcRXALRrmR1KDEoEjVhebjbjbzqasw7owrRF6zjo3g/44Bs9TSSSTCYt8BZy/NUBXUKbMFKJoJFr1iTCg2ftwz9/vT8AV/xnKgX+SkgiEr53Z3ozAfzppN1Di0GJIE2cvHdHHj5rH8rijmc+WRB2OCIC/LC6gJcm/wjADiHNMwRKBGnlzAO60H/39vz1/Xl0G/QOY79WM5FImEZMTY4JIpUI0szgk3rTq30OAJf/ewqjNRWFSGgKS2MAjL7uyFDjUCJIM7vm5fDuNUfw/vVH0iwrwhX/mcrdb31NcTQWdmgiaee1L5fQu0Muu3XIDTUOJYI0lBnJoGf7XD7549Ec2n1Hhn26gN+9MI1Y3IUdmkjaGD93BasLSpmzbGPYoSgRpLMdc7IZPvAQbj9lD8bNWcGv//UF6/2h7iISrOUbvHE91xzTI+RIlAgEuPTwXQCYuGAN+9w9hqk/rA05IpHGb7U/0dxV/ZQIJEksuP8k/vzLPgD87oWprNpUEnJEIo3XO18t5YFRc8kwb6xP2EJJBGZ2vZnNNrNZZjbczMKZYEMqmBkXHLIzQ87Yi+UbSjjo3rFKBiIB+f2L0wBIlm65hCcCM+sMXAPkO+f6ABHg3ETHITU796CuXHFUd+IOBvztY75boSUvRRpSUenmJ/RO3rtjiJFsFlbTUCbQzMwygeaAJs5PIoNP3J0rjuzOqk0lnPXkZ3q0VKQBLVhVULFdPvVL2BKeCJxzS4CHgEXAUmC9c25MouOQLRt80u48dNY+rC2M0vv/RnHlf6Yyf6VqByLba7k/C/CrVx4aciSbhdE01AY4DdgF6AS0MLMLarhuoJlNMbMpK1euTHSYgjcb4uATewMwavYyzn7qc+LJ0qgpkqJW+I+NdmiVPF2jYTQN9QcWOOdWOueiwEjgsOoXOeeGOufynXP5eXl5CQ9SPFcctSujrjuC7nktWLWplOGTteKZyPZY7i9Ck5cbztoDNQkjESwCDjGz5mZmwLHANyHEIXXUu0NLxlznTUlx62uz+GL+6rBDEklZyzYUs0OLJqGtPVCTMPoIJgKvAtOAmX4MQxMdh9RPZiSDN/7QF4DfPj9FHcgi2+jLRetol0S1AQjpqSHn3B3Oud7OuT7Oud845/TAegro1T6Xu07dk40lZfzptZlhhyOSch774Fu+Wboh6f6Q0shiqZfzD+5K97YtGDltCT+tKwo7HJGUUVIW46/vzwPgHn8Uf7JQIpB6yYxk8M/zvWefDxsyruIJCBHZsrFfrwCg6w7NOaJncj0Ao0Qg9bZ7x5acuX8XAM4d+gUlZclVzRVJNtFYvGJaiRcuPzjkaH5OiUC2ycNn78N1/Xsyf1UBu902itMf/5SvFq8LOyyRpBCPO2b/tL7ij6TZP22oONelTbOwwqpVZtgBSOq6qt+uLF5bxKhZy/hy0TrOf3oik2/tT9Os5HksTiQML03+seKBis6tm7HE708b/ttD8J6aTy7mXPKPFM3Pz3dTpkwJOwzZghFTF3PjKzMAmHXXCeRk628MST+xuOOMxz9lxuL1NZ7/7t4TyYwkriHGzKY65/K3dp2ahqRBnL5fZ07s0wGAPneM5t2ZS0OOSCTxxsxeVpEEju3djv67twfgjd/3ZeGQkxOaBOpDNQJpUI+O/ZZHxnqPyD15wf4M6JMc0+yKBC0ai9Pz1vcA2L9ra0b+rm/IEalGICG5tn9PPht0DABX/ncaT4z/ng3FWgdZGr/yWnBu08ykSAL1oUQgDa5T62ZM/NOxAPxl1Bz2vnMMfYeMY9KCNSFHJhKMH1YXcO1L0wH46OajQ46m/pQIJBDtWzZl8q39K9ZBXrKuiLOf+pw3pi9hTUEpFw2bxLg5y5lZS6eaSKpYtLqQox4cD8Afju7BDi2ahBvQNlAfgSTEl4vWcvrjn9V47ppjevDL/TrTPS8nwVGJbJ+1BaXsd8/7FfsLh5wcYjQ/pz4CSSr7dW3Du9ccwQE7twFgz04t6dO5JQCPjfuOYx7+iInzV1MWi2uksqSMJyd8X7H90c39wgtkO+lhb0mYPTq1ZMRVVdcg+nDuCm57bRZL1hVxztAvqpybdOuxtMttytxlGzl36Of8+uCuXH1MT35cU0iXNs1p1kQD1yRcH87x5g+af99JZGQk30CxulLTkCSFhasKOObh8VRfCbPyqMzqzsnfiXtP7/OzZ7NLy+JsLI4SizvmLt9Ir/a55GRn0kKD3KSBlJbF6XWb96joCXu256nfbLX1JRR1bRrSfxmSFLq1bcG3955ExP+rasK8lVw4bFJFEvhs0DE8OvZbXp7yY8U9L0/5scr+1oy78Sj1Q8h2cc6xrjDK55VW6bv9F3uGGFHDUI1AktbqTSXMXbaRg7vvWJEgKp8bPHImY75evsXXyN+5DYvWFLJi4+a1jz4ddAydWyffxF+S3CoPGAOvtvrRzf2SdrQwqEYgjcCOOdkc1qPmJf12zMlm6IX5lP8hM23RWnZq05xpi9by/coCfntEd5pkbv4P9KN5K7lo2CQA+g4ZV3H890fvys0n9A7wXyGNxeF/GVdl/9cHd03qJFAfqhFI2nDO8cDouTwx/vsqx8/cvwsPn71PSFFJMiuOxnj0g29ZVxhl+KRFAHx8y9EM+3QBg0/cvcofG8morjUCJQJJO2/N+IkPvlnOfWfsxR63jwag/+7tOLxHWy48tFtKP/0hDadyh3C5r+8+geZNUqchRYlApA5WbSrhnKc+5/uVBRXHzGDabceR0zSTwtIYrZplhRihhOWON2bx/Oc/VOxffFg37jw1tTqGkzoRmFlr4GmgD+CAS51zn9d2vRKBBG3e8o3c9+43jJ+78mfnsiLGG78/nD06tQwhMkmk4miMzAxj5JdLuOXVr7j88F247ZQ9wg5rmyV7Inge+Ng597SZNQGaO+dqXedQiUASoSwWZ/ikRbw0+ccqSwuCt07zu9ccDpCUK0zJ9vtxTSFHPPBhlWOJXkimoSVtIjCzlsAMoLurY+FKBJJokxeuoVf7XJo3iTBy2mL+OMJbdjDD4N1rj6B3B9UOGotNJWXc8PL0nz2KPPaGI+nRLjekqBpGMieCfYGhwNfAPsBU4FrnXEFt9ygRSJiisTh97hhNSVn8Z+fOzu/CXaf20XQXKap6h/Bj5+3Hqft0CjGihpXMiSAf+ALo65ybaGaPAhucc/9X7bqBwECArl27HvDDDz/8/MVEEqQsFmf6j+uY8O0qHvvg2xqv2a9ra544/wA6tGqa4OikvpxzRGOuShJ4++rD6dO5VYhRNbxkTgQdgC+cc938/SOAQc65WudvVY1Akkk87nB4HYuH3PcBG0vKKs4N2LMDT/7mgPCCk62KxR173TmaspijNObV8hbcf1Kj7PtJ2pHFzrllZvajme3mnJsLHIvXTCSSEsrHGbTIzuTL24+joCTGFwtW89G8lbw4cRELVhWwS9sWvDLlR5pkZnDavp0r7v39C9N4Z+ZSjuyVx78vPSisf0Jacc5RUhYnK5LBA6Pn8Nb0nygs9aY6b908iym39m+USaA+whoZcTXwgv/E0HzgkpDiENkumZEMWjXP4IQ9O7B3l1a8OHERRz80vso1WZEMBuzZgX4PjWfRmkLAm1RvxNTFnHlAlxCiTg9rC0q58ZUZjPOniq5upx2a8fSFB6b0U0ENJZRE4JybDiTnvK0i26hjq2ac2KcD781aVuX4m9N/Yu6yjRVJ4H9XHMrZT33Oja/MYKcdmnNgtzYc9aCXJF68/GD26NSS1s1Tb7nDZHPdy9P5aN7Px4UAvH/9kfRsn9pPBDUkjSwWaUDRWJyP5q6kWZMIPdvlMGTUHEbNWkbTrAhrCkor2qJHzVrKlf+dVuvrDNizA09csH/aN1nUh3OOfe4aw4bisirHR/7uMDYURXl4zDyGXngAHVulz8yzWqpSJARZkQz679Gevj3a0q5lUwbs2YHC0hhrCkp57Lz9Kr7YB/TpyJt/6Fvl3v26tq7YHjV7WZU578vNWrKe4qiW8qzJPz/87mdJYMpt/dm/axv67daOt64+PK2SQH2kzuxJIimob4+2Fdsn7Nm+yrm9u7Rm4ZCTKSwtI8OMplkR4nHHS5N/5M43Z/POV0s5bNe2DHlvDk9+VHXG1MfP35+T9uqYkH9DspuzbAMD/vZxxf4Llx/M7W/M4vyDd6ZtTs3TmEtVahoSCdhfRs2hWVaEa47tWed7bnh5OiO/XLLFa76990Sy0qijc31hlKmL1nDpc5u/C3bv2JLc7EwmLVwDwIu/PZjDdm1b20uknaQdR7AtlAgk3Xy3YiP9/zqhYn/8Tf3o0sZr1rjqhWm870+HkOqLptdH+aO3NbniqO5ceeSutGmhTvbK1EcgksJ6tMvlhcsP5oz9OjP2hiPp1rYFmZEMMiMZPHXB5gFrtT0V09h8tXhdRRJoEsng2UsO5LlLDqw4P/CI7koC20F9BCJJqm+PtlX6GMplZBhz7hnA3neO4aN5Kzm6d7sa74/FHcc98hHzVxbwrwvzOW6P9jVel0j3vfsNM35cx43H78ZBu+xQ5/ue+3Qh8PNpIMZcfyRtc7LZQUlgu6hGIJKCmmZFOKFPB577bCHXDP8S8JqTRvljGDaVlLHrn95lvr/gzh9HfFXl/hFTF/PntxM3oH/BqgK6DXqHoRPmM3HBGs5+6nPemL7lPpDKZi5ZzzG92/1sLqBe7XOVBBqAEoFIirrlhN0AeHPGTzzzyQL6/3UCV/53KnOXbWSo/5RRZoZx3kE7sam4jCJ/WoUfVhdw4yszePqTBUwIoGlpyboihk9aRCzuKCgp45H351UZbf3sxV6Tzu1vzCYe/3kf5epNJQz89xQWrPKSWElZjPmrCti9owaABUWJQCRF7bRDc/7PXz3rnkp/3Z/6j08YN3cF++zUmu/uO4kBfTpSGovzyNh5FJaWcdSD4yuuneI/bTN32UYuHDaJdYWlVcpYvLaQe97+mq09VOKc46HRc+k26B36DhnH4JEz6XPHaPa8YzSP+rO17tK2BQuHnMzRvdvx6Ln7sr4oyodzfz79w2tfLmHM18sZOmE+APNXFhCLO3ppJHBglAhEUthlh+9CXu7mZ+XPO2gnSsrizFqygeP9PoFDuntt8U9/PJ//+GvwXnjoznRu3YzFa4t4eMxcTvjbBCbMW8ntb8yueK0NxVEO/8uHPPPJAiYt8BLGqk0l7Hv3GLoNeocPvvGeXCooKWOXwe/yjw+/qxJbUaWBb2NvOIqxNxxVsV8+BuKy56fQbdA73PraTLoNeocFqwr48zvfAN74APCSFKDFgAKkRCCS4gaf2BuAB3+1N/edvhcd/fUQyhNBdmaEG4/rRdzB/e/NoXmTCHef1oc2LbIY+eUS/j5u8xf4mzN+osyfmnnM7M0rdl370nQAHnl/HusKo4D3Je6cY887Rm++/w99+e7eE5l11wl0b9uCB87cm4VDTqZHuxwilR5zzYpkVBns9cLERQBVmpC+XLSOxWsLuf2NWTSJZNA9r8X2v1lSIz01JJLiTt+vMx1aNeWQXXbEzPj0j8fww5pCdmm7+YvzxL068vD78wC44JCdAVi+oaTi/LG929EiO5M3Z/xEj1vf480/9OWmV2ZUnF+2oZh/f76QJeuKqpR9yXOTK7a/GHxsxaI8OZEMxt3Ub4txj7+5H2NmL2Pe8k2MnLaYFRs3x/PC5Qdz/tMTOfwv3hrCp+3bKa0GzyWaBpSJpIniaIzvVmyiR7scmmZF+Hb5Ro57ZALH9G7HsIsPZNWmEvL/PPZn9425/kiOf8Qb3NYkksGp+3bi9l/swd53jqm4ZtyNR9E9L2e74ltfFGXlxuKKdYK7DXqn4ty0/ztOTwdtAw0oE5EqmmZF6NO5FU2zvPWVe7bPZeGQkxnmP8XTNieb+fedxHX9e9KxVVPa5jRh7A1H0qt9Li8PPASA0licfbq0omXTLC44pCsAO+/YfLuTAECrZllVFosv71O4+YTdlAQCphqBiGyVc46Ln53M8g3FvHrVYeRke63Km0rKaJ4VSZtpLlJN0i5VKSKpx8x4voalNcsTgqQ2NQ2JiKQ5JQIRkTSnRCAikuaUCERE0lxoicDMImb2pZm9HVYMIiISbo3gWuCbEMsXERFCSgRm1gU4GXg6jPJFRGSzsGoEfwNuAeIhlS8iIr6EjwYxs1OAFc65qWbWbwvXDQQG+rubzGwu0ApY7x/b2nb577bAqnqGWfn16nO++vFExbu1mLcl3q3FWflYot7jrR1Ltc9ETbE1VLzbGrM+E8HGW9Px+rzH9Y135zpc4w0dT+QPcD+wGFgILAMKgf/W8d6hdd2u9HvKNsQ4dFvOVz+eqHi3FvO2xLu1OMN4j7d2LNU+E3V4XwP7TNT1PdZnIrzPRC1xbtdnorbYNwIwAAAI30lEQVSfhDcNOecGO+e6OOe6AecC45xzF9Tx9rfqsV35WH1t7d7azlc/nqh4t3b/tsRbfb+mOBP9Hm/tWKp9JirvJ/ozUdt5fSbqLujPROXthvpM1CjUSef8pqGbnHOnBFjGFFeHSZeSRarFC6kXs+INXqrFnO7xhjpjlHNuPDA+4GKGBvz6DS3V4oXUi1nxBi/VYk7reFNiGmoREQmOppgQEUlzSgQiImlOiUBEJM2ldSIwsyPM7Ekze9rMPgs7nq0xswwzu9fM/m5mF4Udz9aYWT8z+9h/j/uFHU9dmVkLM5vqD35Mama2u//+vmpmV4UdT12Y2S/N7F9m9oaZHR92PFtjZt3N7BkzezXsWGrjf2af99/X8+t7f8omAjMbZmYrzGxWteMDzGyumX1nZoO29BrOuY+dc1cCbwPPJ3u8wGlAZyCKNygvMA0UrwM2AU0JOF4/toaIGeCPwP+CibJKXA3xGf7G/wyfDQT++GMDxfy6c+63wMXAOQGG21DxznfOXRZknDWpZ+xnAK/67+up9S6sIUenJfIHOBLYH5hV6VgE+B7oDjQBZgB7AHvhfdlX/mlX6b7/AS2TPV5gEHCFf++rKRBvhn9fe+CFVPhMAP3xBjpeDJyS7PH695wKfAb8OhXe40r3PQzsn0LxBvrf3HbGPhjY17/mxfqWlbIrTzvnJphZt2qHDwK+c87NBzCzl4DTnHP3AzVW882sK7DeObchwHAbJF4zWwyU+rux4KJtuPfXtxbIDiLOyhroPT4aaIH3H1eRmb3rnAtkcsSGeo+dc28Cb5rZO8CLQcRaqayGeI8NGAK855ybluzxhqU+sePVuLsA09mGlp6UTQS16Az8WGl/MXDwVu65DHg2sIi2rL7xjgT+bmZHABOCDKwW9YrXzM4ATgBaA/8INrRa1Stm59ytAGZ2MbAqqCSwBfV9j/vhNQtkA+8GGlnt6vs5vhqv5tXKzHo4554MMrga1Pc93hG4F9jPzAb7CSMstcX+GPAPMzuZbZiGorElAqvh2BZHzDnn7ggolrqoV7zOuUK8xBWW+sY7Ei95hanenwkA59xzDR9KndT3PR5P8KPzt6a+MT+G98UVlvrGuxq4Mrhw6qXG2J1zBcAl2/qiKdtZXIvFwE6V9rsAP4UUS10o3uClWsypFi+kXsypFm9lgcTe2BLBZKCnme1iZk3wOv3eDDmmLVG8wUu1mFMtXki9mFMt3sqCiT2RveAN3KM+HFjK5kcpL/OPnwTMw+tZvzXsOBWvYm4s8aZizKkWb1ixa9I5EZE019iahkREpJ6UCERE0pwSgYhImlMiEBFJc0oEIiJpTolARCTNKRFIgzOzTQko49Q6TindkGX2M7PDtuG+/czsaX/7YjMLa96lKsysW/Upjmu4Js/MRiUqJgmHEoEkLTOL1HbOOfemc25IAGVuaf6tfkC9EwHwJ+Dv2xRQyJxzK4GlZtY37FgkOEoEEigzu9nMJpvZV2Z2V6Xjr5u3CthsMxtY6fgmM7vbzCYCh5rZQjO7y8ymmdlMM+vtX1fxl7WZPWdmj5nZZ2Y238x+5R/PMLPH/TLeNrN3y89Vi3G8md1nZh8B15rZL8xsopl9aWZjzay9Px3wlcD1ZjbdvNXt8sxshP/vm1zTl6WZ5QJ7O+dm1HBuZzP7wH9vPvCnRMfMdjWzL/zXvLumGpZ5K1K9Y2YzzGyWmZ3jHz/Qfx9mmNkkM8v1//L/2H8Pp9VUqzGziJk9WOn/qysqnX4dqPeqV5JCwh5GrZ/G9wNs8n8fDwzFmzExA2+hjyP9czv4v5sBs4Ad/X0HnF3ptRYCV/vbvwOe9rcvBv7hbz8HvOKXsQfefO0Av8KbmjkD6IC3LsKvaoh3PPB4pf02UDHq/nLgYX/7TuCmSte9CBzub3cFvqnhtY8GRlTarxz3W8BF/valwOv+9tvAef72leXvZ7XXPRP4V6X9VngLlcwHDvSPtcSbYbg50NQ/1hOY4m93w1/0BBgI3OZvZwNTgF38/c7AzLA/V/oJ7qexTUMtyeV4/+dLfz8H74toAnCNmZ3uH9/JP74ab8GdEdVep3wq66l4c+/X5HXnrR3wtZm1948dDrziH19mZh9uIdaXK213AV42s454X64LarmnP7CHWcXMwC3NLNc5t7HSNR2BlbXcf2ilf89/gAcqHf+lv/0i8FAN984EHjKzvwBvO+c+NrO9gKXOuckAzl9sycxa4M1Vvy/e+9urhtc7Hti7Uo2pFd7/JwuAFUCnWv4N0ggoEUiQDLjfOfdUlYPeYir9gUOdc4VmNh5vXWOAYudc9dXXSvzfMWr/zJZU2rZqv+uioNL234G/Oufe9GO9s5Z7MvD+DUVbeN0iNv/btqbOE3855+aZ2QF4E5Ddb2Zj8JpwanqN64HlwD5+zMU1XGN4Na/RNZxrivfvkEZKfQQSpNHApWaWA2Bmnc2sHd5fm2v9JNAbOCSg8j8BzvT7CtrjdfbWRStgib99UaXjG4HcSvtjgD+U7/h/cVf3DdCjlnI+w5tGGLw2+E/87S/wmn6odL4KM+sEFDrn/otXY9gfmAN0MrMD/Wty/c7vVng1hTjwG7x1b6sbDVxlZln+vb38mgR4NYgtPl0kqU2JQALjnBuD17TxuZnNBF7F+yIdBWSa2VfAPXhffEEYgTd97yzgKWAisL4O990JvGJmHwOrKh1/Czi9vLMYuAbI9ztXv6aGVaycc3PwlmTMrX7Ov/8S/334DXCtf/w64AYzm4TXtFRTzHsBk8xsOnAr8GfnXClwDt5ypjOA9/H+mn8cuMjMvsD7Ui+o4fWeBr4GpvmPlD7F5trX0cA7NdwjjYSmoZZGzcxynHObzFt3dhLQ1zm3LMExXA9sdM49XcfrmwNFzjlnZufidRyfFmiQW45nAt7i7mvDikGCpT4CaezeNrPWeJ2+9yQ6CfieAM6qx/UH4HXuGrAO74miUJhZHl5/iZJAI6YagYhImlMfgYhImlMiEBFJc0oEIiJpTolARCTNKRGIiKQ5JQIRkTT3/59J3NTKKVq2AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x199b3d569e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 76%|█████████████████████████████████████████████████▌               | 16635/21791 [21:10<06:33, 13.10it/s, loss=18.8]"
     ]
    }
   ],
   "source": [
    "#find suitable learning rates\n",
    "learner.lr_find(1e-07,1e2)\n",
    "learner.sched.plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T16:57:14.166255Z",
     "start_time": "2018-01-24T16:56:54.144823Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a71a1376eb5b45a3a7d96dac8b58ae40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=15), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  0%|                                                                    | 12/21791 [00:00<29:47, 12.19it/s, loss=9.99]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception in thread Thread-12:\n",
      "Traceback (most recent call last):\n",
      "  File \"C:\\Users\\nirantk\\Anaconda3\\envs\\fastai\\lib\\threading.py\", line 916, in _bootstrap_inner\n",
      "    self.run()\n",
      "  File \"C:\\Users\\nirantk\\Anaconda3\\envs\\fastai\\lib\\site-packages\\tqdm\\_tqdm.py\", line 144, in run\n",
      "    for instance in self.tqdm_cls._instances:\n",
      "  File \"C:\\Users\\nirantk\\Anaconda3\\envs\\fastai\\lib\\_weakrefset.py\", line 60, in __iter__\n",
      "    for itemref in self.data:\n",
      "RuntimeError: Set changed size during iteration\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      5.371777   5.243932  \n",
      "    1      4.339965   4.741244                                                                                         \n",
      "    2      4.437619   4.622295                                                                                         \n",
      "    3      4.05479    4.533704                                                                                         \n",
      "    4      4.007942   4.438206                                                                                         \n",
      "    5      3.997428   4.379419                                                                                         \n",
      "    6      4.074864   4.318187                                                                                         \n",
      "    7      3.923672   4.335486                                                                                         \n",
      "    8      3.855617   4.309592                                                                                         \n",
      "    9      3.814313   4.264634                                                                                         \n",
      "    10     3.822053   4.253757                                                                                         \n",
      "    11     3.822665   4.225037                                                                                         \n",
      " 94%|█████████████████████████████████████████████████████████████▎   | 20565/21791 [26:28<01:34, 12.94it/s, loss=4.18]"
     ]
    }
   ],
   "source": [
    "learner.fit(1e-4, 4, wds=1e-6, cycle_len=1, cycle_mult=2)\n",
    "learner.save_encoder('adam1_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.save_encoder('adam1_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-12-26T04:31:17.340615Z",
     "start_time": "2017-12-26T04:31:04.722284Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#loss 3.84171; perplexity 46.61\n",
    "learner.fit(1e-3, 1, wds=1e-6, cycle_len=20, cycle_save_name='adam3_20')\n",
    "#learner.save_encoder('adam3_enc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.save_encoder('adam3_hindi_minfreq10_enc_v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d91c024647d4e5b8ad6dc8b174ae133",
       "version_major": 2,
       "version_minor": 0
      },
      "text/html": [
       "<p>Failed to display Jupyter Widget of type <code>HBox</code>.</p>\n",
       "<p>\n",
       "  If you're reading this message in the Jupyter Notebook or JupyterLab Notebook, it may mean\n",
       "  that the widgets JavaScript is still loading. If this message persists, it\n",
       "  likely means that the widgets JavaScript library is either not installed or\n",
       "  not enabled. See the <a href=\"https://ipywidgets.readthedocs.io/en/stable/user_install.html\">Jupyter\n",
       "  Widgets Documentation</a> for setup instructions.\n",
       "</p>\n",
       "<p>\n",
       "  If you're reading this message in another frontend (for example, a static\n",
       "  rendering on GitHub or <a href=\"https://nbviewer.jupyter.org/\">NBViewer</a>),\n",
       "  it may mean that your frontend doesn't currently support widgets.\n",
       "</p>\n"
      ],
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=5), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                                                                                         \n",
      "    0      3.50012    4.0503    \n",
      "    1      3.489653   4.020124                                                                                         \n",
      "    2      3.498954   3.959882                                                                                         \n",
      "    3      3.501738   3.891938                                                                                         \n",
      "    4      3.602018   3.846254                                                                                         \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[3.846254]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.fit(1e-3, 1, wds=1e-6, cycle_len=5, cycle_save_name='adam3_20')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.save_encoder('adam3_20')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'सीआईडी सोनी चैनल पर प्रसारित होने वाला हिन्दी भाषा का एक धारावाहिक है , जिसे भारत का सबसे लंबा चलने वाला धारावाहिक होने का श्रेय प्राप्त है । अपराध व जासूसी शैली पर आधारित इस धारावाहिक में शिवाजी साटम , दयानन्द शेट्टी और आदित्य श्रीवास्तव मुख्य किरदार निभा रहे हैं । इसके सर्जक , निर्देशक और लेखक बृजेन्द्र पाल सिंह हैं । इसका निर्माण फायरवर्क्स नामक कंपनी ने किया है जिसके संस्थापक बृजेन्द्र पाल सिंह और प्रदीप उपूर हैं । 21 जनवरी 1998 से शुरु होकर यह धारावाहिक अब तक लगातार चल रहा है । इसका प्रसारण प्रत्येक शनिवार और रविवार को रात 10 बजे होता है । इसका पुनः प्रसारण सोनी पल चैनल पर रात 9 बजे होता है जिसमें इसके पुराने प्रकरण दिखाये जाते हैं । इस धारावाहिक ने 21 जनवरी 2018 को अपने प्रसारण के 20 वर्ष पूर्ण किये और 21वें वर्ष में प्रवेश किया । इससे पहले , 27 सितम्बर 2013 को इस धारावाहिक ने अपनी 1000वीं कड़ी पूरी की ।'"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "m=learner.model\n",
    "ss=\"\"\"सीआईडी सोनी चैनल पर प्रसारित होने वाला हिन्दी भाषा का एक धारावाहिक है, जिसे भारत का सबसे लंबा चलने वाला धारावाहिक होने का श्रेय प्राप्त है। अपराध व जासूसी शैली पर आधारित इस धारावाहिक में शिवाजी साटम, दयानन्द शेट्टी और आदित्य श्रीवास्तव मुख्य किरदार निभा रहे हैं। इसके सर्जक, निर्देशक और लेखक बृजेन्द्र पाल सिंह हैं। इसका निर्माण फायरवर्क्स नामक कंपनी ने किया है जिसके संस्थापक बृजेन्द्र पाल सिंह और प्रदीप उपूर हैं। 21 जनवरी 1998 से शुरु होकर यह धारावाहिक अब तक लगातार चल रहा है। इसका प्रसारण प्रत्येक शनिवार और रविवार को रात 10 बजे होता है। इसका पुनः प्रसारण सोनी पल चैनल पर रात 9 बजे होता है जिसमें इसके पुराने प्रकरण दिखाये जाते हैं। इस धारावाहिक ने 21 जनवरी 2018 को अपने प्रसारण के 20 वर्ष पूर्ण किये और 21वें वर्ष में प्रवेश किया। इससे पहले, 27 सितम्बर 2013 को इस धारावाहिक ने अपनी 1000वीं कड़ी पूरी की। \"\"\"\n",
    "s = [spacy_tok(ss)]\n",
    "t=TEXT.numericalize(s)\n",
    "' '.join(s[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 10984\n",
       "  3460\n",
       "  1517\n",
       "    17\n",
       "  1687\n",
       "    75\n",
       "   224\n",
       "   195\n",
       "   118\n",
       "    10\n",
       "    12\n",
       "  1518\n",
       "     6\n",
       "     4\n",
       "   178\n",
       "    42\n",
       "    10\n",
       "    84\n",
       "  2496\n",
       "  1390\n",
       "   224\n",
       "  1518\n",
       "    75\n",
       "    10\n",
       "  2127\n",
       "   110\n",
       "     6\n",
       "     3\n",
       "  1493\n",
       "   128\n",
       "  7799\n",
       "   504\n",
       "    17\n",
       "   340\n",
       "    24\n",
       "  1518\n",
       "     5\n",
       "  4312\n",
       "     0\n",
       "     4\n",
       "  9587\n",
       "  9818\n",
       "     8\n",
       "  5518\n",
       "  8218\n",
       "   159\n",
       "  1436\n",
       "  5021\n",
       "   109\n",
       "    16\n",
       "     3\n",
       "    54\n",
       "     0\n",
       "     4\n",
       "  1340\n",
       "     8\n",
       "   625\n",
       "     0\n",
       "  3573\n",
       "   206\n",
       "    16\n",
       "     3\n",
       "   119\n",
       "   158\n",
       "     0\n",
       "   228\n",
       "   305\n",
       "    20\n",
       "    18\n",
       "     6\n",
       "   215\n",
       "  1681\n",
       "     0\n",
       "  3573\n",
       "   206\n",
       "     8\n",
       "  7810\n",
       "     0\n",
       "    16\n",
       "     3\n",
       "  1067\n",
       "   413\n",
       "  1293\n",
       "     9\n",
       "   536\n",
       "   363\n",
       "    22\n",
       "  1518\n",
       "   183\n",
       "    47\n",
       "   969\n",
       "   857\n",
       "   120\n",
       "     6\n",
       "     3\n",
       "   119\n",
       "  1313\n",
       "   274\n",
       "  5463\n",
       "     8\n",
       "  4219\n",
       "    11\n",
       "   966\n",
       "   389\n",
       "  1743\n",
       "    46\n",
       "     6\n",
       "     3\n",
       "   119\n",
       "   834\n",
       "  1313\n",
       "  3460\n",
       "  7139\n",
       "  1517\n",
       "    17\n",
       "   966\n",
       "   498\n",
       "  1743\n",
       "    46\n",
       "     6\n",
       "   135\n",
       "    54\n",
       "   390\n",
       "  3072\n",
       " 14479\n",
       "   138\n",
       "    16\n",
       "     3\n",
       "    24\n",
       "  1518\n",
       "    20\n",
       "  1067\n",
       "   413\n",
       "  5629\n",
       "    11\n",
       "    48\n",
       "  1313\n",
       "     2\n",
       "   528\n",
       "    55\n",
       "   520\n",
       "   265\n",
       "     8\n",
       "     0\n",
       "    55\n",
       "     5\n",
       "   521\n",
       "    18\n",
       "     3\n",
       "   319\n",
       "    89\n",
       "     4\n",
       "  1377\n",
       "   958\n",
       "  1036\n",
       "    11\n",
       "    24\n",
       "  1518\n",
       "    20\n",
       "    86\n",
       "     0\n",
       "  1894\n",
       "   362\n",
       "     7\n",
       "     3\n",
       "[torch.cuda.LongTensor of size 165x1 (GPU 0)]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Set batch size to 1\n",
    "m[0].bs=1\n",
    "# Turn off dropout\n",
    "m.eval()\n",
    "# Reset hidden state\n",
    "m.reset()\n",
    "# Get predictions from model\n",
    "res, *other_things = m(t)\n",
    "# Put the batch size back to what it was\n",
    "m[0].bs=bs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[Variable containing:\n",
       "  ( 0 ,.,.) = \n",
       "    5.6415e-03 -7.1320e-02 -2.2373e-02  ...  -4.2131e-01 -8.0832e-02  1.7624e-02\n",
       "  \n",
       "  ( 1 ,.,.) = \n",
       "    6.4953e-03  1.9529e-02  4.1037e-01  ...   6.1077e-02 -3.3719e-02  1.2098e-02\n",
       "  \n",
       "  ( 2 ,.,.) = \n",
       "    5.4429e-03  6.8038e-02 -4.6050e-01  ...  -6.6798e-04 -1.9427e-01 -6.1399e-03\n",
       "  ... \n",
       "  \n",
       "  (162,.,.) = \n",
       "   -1.1956e-01 -4.1483e-01 -2.1589e-01  ...  -3.5744e-01  7.1733e-01  7.3748e-02\n",
       "  \n",
       "  (163,.,.) = \n",
       "   -2.0688e-01  6.6090e-01 -6.5121e-03  ...  -7.7238e-01  1.1244e-02  5.3727e-03\n",
       "  \n",
       "  (164,.,.) = \n",
       "   -2.9987e-01 -3.7307e-02 -4.7536e-03  ...   1.2151e-01  1.4413e-02  2.9323e-02\n",
       "  [torch.cuda.FloatTensor of size 165x1x500 (GPU 0)], Variable containing:\n",
       "  ( 0 ,.,.) = \n",
       "   -1.1753e-02  3.5966e-02  5.9837e-03  ...   5.3034e-02  4.6100e-03 -3.9832e-02\n",
       "  \n",
       "  ( 1 ,.,.) = \n",
       "   -1.8573e-02  5.4061e-02 -1.3606e-02  ...  -1.8617e-01  3.7597e-03 -2.9424e-02\n",
       "  \n",
       "  ( 2 ,.,.) = \n",
       "   -1.9215e-02  1.1223e-01 -1.0460e-02  ...  -8.1387e-02  1.6686e-02  1.5788e-01\n",
       "  ... \n",
       "  \n",
       "  (162,.,.) = \n",
       "   -6.2167e-02  1.5144e-01 -2.0713e-03  ...  -5.1179e-02 -8.7179e-01  2.0836e-02\n",
       "  \n",
       "  (163,.,.) = \n",
       "   -2.7075e-02 -6.8117e-03 -5.5317e-03  ...   1.4562e-01 -9.8030e-01  2.0159e-02\n",
       "  \n",
       "  (164,.,.) = \n",
       "   -3.1385e-02 -1.9396e-03 -3.3755e-02  ...   4.2083e-02 -1.3868e-03 -7.3318e-02\n",
       "  [torch.cuda.FloatTensor of size 165x1x500 (GPU 0)], Variable containing:\n",
       "  ( 0 ,.,.) = \n",
       "   -7.6667e-02 -4.9106e-03  1.0548e-01  ...  -5.4446e-02  3.7817e-02 -8.6417e-02\n",
       "  \n",
       "  ( 1 ,.,.) = \n",
       "    6.2226e-01 -3.0509e-02  1.7749e-01  ...   5.0094e-01 -4.0406e-01 -5.8726e-01\n",
       "  \n",
       "  ( 2 ,.,.) = \n",
       "    7.1825e-01 -1.5146e-01  3.6100e-01  ...   4.5854e-01 -3.3733e-01 -4.3351e-01\n",
       "  ... \n",
       "  \n",
       "  (162,.,.) = \n",
       "   -1.6903e-01 -2.5537e-02 -4.0591e-01  ...   2.7953e-01  1.9832e-01 -1.6843e-01\n",
       "  \n",
       "  (163,.,.) = \n",
       "    3.3020e-02  4.3968e-02 -7.9912e-02  ...   1.4014e-01 -8.9190e-02 -6.4902e-02\n",
       "  \n",
       "  (164,.,.) = \n",
       "    3.8852e-01 -7.8448e-02 -5.4142e-02  ...   4.2303e-01 -2.7155e-01 -1.1643e-02\n",
       "  [torch.cuda.FloatTensor of size 165x1x300 (GPU 0)]], [Variable containing:\n",
       "  ( 0 ,.,.) = \n",
       "    5.6415e-03 -7.1320e-02 -2.2373e-02  ...  -4.2131e-01 -8.0832e-02  1.7624e-02\n",
       "  \n",
       "  ( 1 ,.,.) = \n",
       "    6.4953e-03  1.9529e-02  4.1037e-01  ...   6.1077e-02 -3.3719e-02  1.2098e-02\n",
       "  \n",
       "  ( 2 ,.,.) = \n",
       "    5.4429e-03  6.8038e-02 -4.6050e-01  ...  -6.6798e-04 -1.9427e-01 -6.1399e-03\n",
       "  ... \n",
       "  \n",
       "  (162,.,.) = \n",
       "   -1.1956e-01 -4.1483e-01 -2.1589e-01  ...  -3.5744e-01  7.1733e-01  7.3748e-02\n",
       "  \n",
       "  (163,.,.) = \n",
       "   -2.0688e-01  6.6090e-01 -6.5121e-03  ...  -7.7238e-01  1.1244e-02  5.3727e-03\n",
       "  \n",
       "  (164,.,.) = \n",
       "   -2.9987e-01 -3.7307e-02 -4.7536e-03  ...   1.2151e-01  1.4413e-02  2.9323e-02\n",
       "  [torch.cuda.FloatTensor of size 165x1x500 (GPU 0)], Variable containing:\n",
       "  ( 0 ,.,.) = \n",
       "   -1.1753e-02  3.5966e-02  5.9837e-03  ...   5.3034e-02  4.6100e-03 -3.9832e-02\n",
       "  \n",
       "  ( 1 ,.,.) = \n",
       "   -1.8573e-02  5.4061e-02 -1.3606e-02  ...  -1.8617e-01  3.7597e-03 -2.9424e-02\n",
       "  \n",
       "  ( 2 ,.,.) = \n",
       "   -1.9215e-02  1.1223e-01 -1.0460e-02  ...  -8.1387e-02  1.6686e-02  1.5788e-01\n",
       "  ... \n",
       "  \n",
       "  (162,.,.) = \n",
       "   -6.2167e-02  1.5144e-01 -2.0713e-03  ...  -5.1179e-02 -8.7179e-01  2.0836e-02\n",
       "  \n",
       "  (163,.,.) = \n",
       "   -2.7075e-02 -6.8117e-03 -5.5317e-03  ...   1.4562e-01 -9.8030e-01  2.0159e-02\n",
       "  \n",
       "  (164,.,.) = \n",
       "   -3.1385e-02 -1.9396e-03 -3.3755e-02  ...   4.2083e-02 -1.3868e-03 -7.3318e-02\n",
       "  [torch.cuda.FloatTensor of size 165x1x500 (GPU 0)], Variable containing:\n",
       "  ( 0 ,.,.) = \n",
       "   -7.6667e-02 -4.9106e-03  1.0548e-01  ...  -5.4446e-02  3.7817e-02 -8.6417e-02\n",
       "  \n",
       "  ( 1 ,.,.) = \n",
       "    6.2226e-01 -3.0509e-02  1.7749e-01  ...   5.0094e-01 -4.0406e-01 -5.8726e-01\n",
       "  \n",
       "  ( 2 ,.,.) = \n",
       "    7.1825e-01 -1.5146e-01  3.6100e-01  ...   4.5854e-01 -3.3733e-01 -4.3351e-01\n",
       "  ... \n",
       "  \n",
       "  (162,.,.) = \n",
       "   -1.6903e-01 -2.5537e-02 -4.0591e-01  ...   2.7953e-01  1.9832e-01 -1.6843e-01\n",
       "  \n",
       "  (163,.,.) = \n",
       "    3.3020e-02  4.3968e-02 -7.9912e-02  ...   1.4014e-01 -8.9190e-02 -6.4902e-02\n",
       "  \n",
       "  (164,.,.) = \n",
       "    3.8852e-01 -7.8448e-02 -5.4142e-02  ...   4.2303e-01 -2.7155e-01 -1.1643e-02\n",
       "  [torch.cuda.FloatTensor of size 165x1x300 (GPU 0)]]]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other_things"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['<unk>', 'यह', 'इस', 'इसके', 'इसका', 'इसमें', 'इसकी', 'इसे', 'फ़िल्म', 'बाद']"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nexts = torch.topk(res[-1], 10)[1]\n",
    "[TEXT.vocab.itos[o] for o in to_np(nexts)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "सीआईडी सोनी चैनल पर प्रसारित होने वाला हिन्दी भाषा का एक धारावाहिक है, जिसे भारत का सबसे लंबा चलने वाला धारावाहिक होने का श्रेय प्राप्त है। अपराध व जासूसी शैली पर आधारित इस धारावाहिक में शिवाजी साटम, दयानन्द शेट्टी और आदित्य श्रीवास्तव मुख्य किरदार निभा रहे हैं। इसके सर्जक, निर्देशक और लेखक बृजेन्द्र पाल सिंह हैं। इसका निर्माण फायरवर्क्स नामक कंपनी ने किया है जिसके संस्थापक बृजेन्द्र पाल सिंह और प्रदीप उपूर हैं। 21 जनवरी 1998 से शुरु होकर यह धारावाहिक अब तक लगातार चल रहा है। इसका प्रसारण प्रत्येक शनिवार और रविवार को रात 10 बजे होता है। इसका पुनः प्रसारण सोनी पल चैनल पर रात 9 बजे होता है जिसमें इसके पुराने प्रकरण दिखाये जाते हैं। इस धारावाहिक ने 21 जनवरी 2018 को अपने प्रसारण के 20 वर्ष पूर्ण किये और 21वें वर्ष में प्रवेश किया। इससे पहले, 27 सितम्बर 2013 को इस धारावाहिक ने अपनी 1000वीं कड़ी पूरी की।  \n",
      "\n",
      "यह एक ऐसा व्यक्ति है जो अपने आप को एक दूसरे से अलग करता है । यह एक ऐसा व्यक्ति है जो अपने आप को एक दूसरे से अलग करता है । यह एक ऐसा व्यक्ति है जो अपने आप को एक दूसरे से अलग करता है । यह एक ऐसा व्यक्ति है जो अपने आप को एक व्यक्ति के रूप में अपने जीवन में एक महत्वपूर्ण भूमिका निभाता है । यह एक ऐसा व्यक्ति है जो अपने आप को एक व्यक्ति के रूप में अपने जीवन में एक महत्वपूर्ण भूमिका निभाता है । यह एक ऐसा व्यक्ति है जो ...\n"
     ]
    }
   ],
   "source": [
    "print(ss,\"\\n\")\n",
    "for i in range(100):\n",
    "    n=res[-1].topk(2)[1]\n",
    "    n = n[1] if n.data[0]==0 else n[0]\n",
    "    print(TEXT.vocab.itos[n.data[0]], end=' ')\n",
    "    res,*_ = m(n[0].unsqueeze(0))\n",
    "print('...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T17:09:04.381055Z",
     "start_time": "2018-01-24T17:05:17.677117Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#m = learner.model\n",
    "#pickle.dump(m,open(f'{PATH}models/wiki_lang.pkl','wb'))\n",
    "TEXT = pickle.load(open(f'{PATH}models/TEXT.pkl','rb'))\n",
    "m = pickle.load(open(f'{PATH}models/wiki_lang.pkl','rb'))\n",
    "m[0].bs=1\n",
    "m.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T17:13:05.644224Z",
     "start_time": "2018-01-24T17:13:05.588131Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def gen_text(ss,topk):\n",
    "    s = [word_tokenize(ss)]\n",
    "    t = TEXT.numericalize(s)\n",
    "    m.reset()\n",
    "    pred,*_ = m(t)\n",
    "    pred_i = torch.topk(pred[-1], topk)[1]\n",
    "    return [TEXT.vocab.itos[o] for o in to_np(pred_i)]\n",
    "\n",
    "def gen_sentences(ss,nb_words):\n",
    "    result = []\n",
    "    s = [word_tokenize(ss)]\n",
    "    t = TEXT.numericalize(s)\n",
    "    m.reset()\n",
    "    pred,*_ = m(t)\n",
    "    for i in range(nb_words):\n",
    "        pred_i = pred[-1].topk(2)[1]\n",
    "        pred_i = pred_i[1] if pred_i.data[0] < 2 else pred_i[0]\n",
    "        result.append(TEXT.vocab.itos[pred_i.data[0]])\n",
    "        pred,*_ = m(pred_i[0].unsqueeze(0))\n",
    "    return(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T17:13:36.841186Z",
     "start_time": "2018-01-24T17:13:36.725232Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ss=\"\"\"สวัสดีครับพี่น้องเสื้อ\"\"\"\n",
    "gen_text(ss,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T17:13:38.363900Z",
     "start_time": "2018-01-24T17:13:37.986379Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "''.join(gen_sentences(ss,50))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "emb_weights = list(learner.model.named_parameters())[0][1]\n",
    "emb_np = to_np(emb_weights.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21821, 300)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "emb_np.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TEXT = pickle.load(open(f'{PATH}//models//TEXT_min_freq50.pkl','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-24T17:14:19.257675Z",
     "start_time": "2018-01-24T17:14:19.219043Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TEXT.vocab.set_vectors(vectors=emb_weights.data,dim=300,stoi=TEXT.vocab.stoi)\n",
    "# pickle.dump(TEXT, open(f'{PATH}models/TEXT_vec.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-25T03:54:53.365410Z",
     "start_time": "2018-01-25T03:54:52.039915Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "TEXT_vec = pickle.load(open(f'{PATH}models/TEXT_vec.pkl','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-25T03:55:01.593405Z",
     "start_time": "2018-01-25T03:55:01.533652Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>290</th>\n",
       "      <th>291</th>\n",
       "      <th>292</th>\n",
       "      <th>293</th>\n",
       "      <th>294</th>\n",
       "      <th>295</th>\n",
       "      <th>296</th>\n",
       "      <th>297</th>\n",
       "      <th>298</th>\n",
       "      <th>299</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>&lt;unk&gt;</th>\n",
       "      <td>-0.025666</td>\n",
       "      <td>0.190339</td>\n",
       "      <td>-0.278813</td>\n",
       "      <td>-0.319441</td>\n",
       "      <td>-0.043037</td>\n",
       "      <td>0.034123</td>\n",
       "      <td>-0.271900</td>\n",
       "      <td>-0.014982</td>\n",
       "      <td>0.241724</td>\n",
       "      <td>-1.362962</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.105750</td>\n",
       "      <td>0.000138</td>\n",
       "      <td>0.252063</td>\n",
       "      <td>0.322336</td>\n",
       "      <td>-0.234095</td>\n",
       "      <td>-0.076334</td>\n",
       "      <td>-0.357609</td>\n",
       "      <td>-0.376448</td>\n",
       "      <td>0.016019</td>\n",
       "      <td>-0.099436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>&lt;pad&gt;</th>\n",
       "      <td>-0.125531</td>\n",
       "      <td>-0.153369</td>\n",
       "      <td>-0.190939</td>\n",
       "      <td>-0.342988</td>\n",
       "      <td>0.054864</td>\n",
       "      <td>-0.053099</td>\n",
       "      <td>0.114129</td>\n",
       "      <td>0.146933</td>\n",
       "      <td>-0.150632</td>\n",
       "      <td>0.331193</td>\n",
       "      <td>...</td>\n",
       "      <td>0.010165</td>\n",
       "      <td>0.212148</td>\n",
       "      <td>-0.309134</td>\n",
       "      <td>-0.256550</td>\n",
       "      <td>0.050639</td>\n",
       "      <td>0.086903</td>\n",
       "      <td>0.069713</td>\n",
       "      <td>0.099145</td>\n",
       "      <td>-0.143334</td>\n",
       "      <td>0.060667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>के</th>\n",
       "      <td>-0.124854</td>\n",
       "      <td>-0.068364</td>\n",
       "      <td>-0.364002</td>\n",
       "      <td>-0.383810</td>\n",
       "      <td>0.172778</td>\n",
       "      <td>-0.003992</td>\n",
       "      <td>-0.168285</td>\n",
       "      <td>0.075885</td>\n",
       "      <td>-0.030859</td>\n",
       "      <td>-2.595019</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.032839</td>\n",
       "      <td>0.077952</td>\n",
       "      <td>0.063838</td>\n",
       "      <td>0.858946</td>\n",
       "      <td>-0.325743</td>\n",
       "      <td>0.632531</td>\n",
       "      <td>-0.839241</td>\n",
       "      <td>-0.341894</td>\n",
       "      <td>-0.058192</td>\n",
       "      <td>-0.158500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>।</th>\n",
       "      <td>-0.150303</td>\n",
       "      <td>-0.221962</td>\n",
       "      <td>-0.626972</td>\n",
       "      <td>-0.456973</td>\n",
       "      <td>0.470308</td>\n",
       "      <td>-0.213924</td>\n",
       "      <td>-0.112261</td>\n",
       "      <td>-0.437237</td>\n",
       "      <td>0.708551</td>\n",
       "      <td>3.330899</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.472635</td>\n",
       "      <td>0.085662</td>\n",
       "      <td>-0.071612</td>\n",
       "      <td>-1.019616</td>\n",
       "      <td>-0.371203</td>\n",
       "      <td>-0.139772</td>\n",
       "      <td>-1.050309</td>\n",
       "      <td>-0.382592</td>\n",
       "      <td>-0.067022</td>\n",
       "      <td>-0.025990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>,</th>\n",
       "      <td>-0.160962</td>\n",
       "      <td>-0.361875</td>\n",
       "      <td>-0.190295</td>\n",
       "      <td>-0.389795</td>\n",
       "      <td>0.216144</td>\n",
       "      <td>-0.050485</td>\n",
       "      <td>-0.192041</td>\n",
       "      <td>-2.112090</td>\n",
       "      <td>-0.685204</td>\n",
       "      <td>4.356941</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.240056</td>\n",
       "      <td>-0.039866</td>\n",
       "      <td>-0.316080</td>\n",
       "      <td>0.169702</td>\n",
       "      <td>-0.359465</td>\n",
       "      <td>-0.325764</td>\n",
       "      <td>-0.872892</td>\n",
       "      <td>-0.408702</td>\n",
       "      <td>-0.118085</td>\n",
       "      <td>-0.199788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>में</th>\n",
       "      <td>-0.209305</td>\n",
       "      <td>0.087706</td>\n",
       "      <td>-0.289885</td>\n",
       "      <td>-0.381661</td>\n",
       "      <td>0.208492</td>\n",
       "      <td>-0.069959</td>\n",
       "      <td>-0.083901</td>\n",
       "      <td>1.090752</td>\n",
       "      <td>-0.049023</td>\n",
       "      <td>2.092380</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.080802</td>\n",
       "      <td>0.119540</td>\n",
       "      <td>-0.651442</td>\n",
       "      <td>0.565120</td>\n",
       "      <td>-0.270207</td>\n",
       "      <td>0.440532</td>\n",
       "      <td>-0.828658</td>\n",
       "      <td>-0.318647</td>\n",
       "      <td>-0.055147</td>\n",
       "      <td>-0.139348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>है</th>\n",
       "      <td>-0.163855</td>\n",
       "      <td>0.305773</td>\n",
       "      <td>-0.496239</td>\n",
       "      <td>-0.385285</td>\n",
       "      <td>-0.023947</td>\n",
       "      <td>-0.196321</td>\n",
       "      <td>-0.206529</td>\n",
       "      <td>0.549503</td>\n",
       "      <td>0.392630</td>\n",
       "      <td>1.914814</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.138870</td>\n",
       "      <td>0.029760</td>\n",
       "      <td>-0.292541</td>\n",
       "      <td>-1.653055</td>\n",
       "      <td>-0.312935</td>\n",
       "      <td>0.663974</td>\n",
       "      <td>-0.684114</td>\n",
       "      <td>-0.543373</td>\n",
       "      <td>0.049822</td>\n",
       "      <td>-0.194706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>की</th>\n",
       "      <td>-0.179126</td>\n",
       "      <td>-0.189791</td>\n",
       "      <td>-0.428677</td>\n",
       "      <td>-0.418656</td>\n",
       "      <td>0.017380</td>\n",
       "      <td>-0.009977</td>\n",
       "      <td>-0.137235</td>\n",
       "      <td>0.626785</td>\n",
       "      <td>0.379204</td>\n",
       "      <td>-0.712076</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.199796</td>\n",
       "      <td>0.077377</td>\n",
       "      <td>-0.134444</td>\n",
       "      <td>0.530944</td>\n",
       "      <td>-0.384973</td>\n",
       "      <td>-0.683671</td>\n",
       "      <td>-0.876726</td>\n",
       "      <td>-0.259445</td>\n",
       "      <td>-0.011620</td>\n",
       "      <td>-0.162889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>और</th>\n",
       "      <td>-0.129317</td>\n",
       "      <td>-0.197093</td>\n",
       "      <td>-0.321741</td>\n",
       "      <td>-0.402978</td>\n",
       "      <td>0.194263</td>\n",
       "      <td>-0.088978</td>\n",
       "      <td>-0.178371</td>\n",
       "      <td>1.334814</td>\n",
       "      <td>-0.463591</td>\n",
       "      <td>-1.078326</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.347151</td>\n",
       "      <td>-0.018696</td>\n",
       "      <td>-0.109680</td>\n",
       "      <td>0.606728</td>\n",
       "      <td>-0.374177</td>\n",
       "      <td>-0.089182</td>\n",
       "      <td>-0.906496</td>\n",
       "      <td>-0.386876</td>\n",
       "      <td>-0.097840</td>\n",
       "      <td>-0.239605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>से</th>\n",
       "      <td>-0.204383</td>\n",
       "      <td>0.001643</td>\n",
       "      <td>-0.448961</td>\n",
       "      <td>-0.339362</td>\n",
       "      <td>-0.116989</td>\n",
       "      <td>-0.131715</td>\n",
       "      <td>-0.173486</td>\n",
       "      <td>-0.483954</td>\n",
       "      <td>-0.084200</td>\n",
       "      <td>-0.646569</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.130293</td>\n",
       "      <td>0.198819</td>\n",
       "      <td>-1.254919</td>\n",
       "      <td>0.746580</td>\n",
       "      <td>-0.378496</td>\n",
       "      <td>0.199947</td>\n",
       "      <td>0.131102</td>\n",
       "      <td>-0.393537</td>\n",
       "      <td>-0.015223</td>\n",
       "      <td>-0.101342</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 300 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            0         1         2         3         4         5         6    \\\n",
       "<unk> -0.025666  0.190339 -0.278813 -0.319441 -0.043037  0.034123 -0.271900   \n",
       "<pad> -0.125531 -0.153369 -0.190939 -0.342988  0.054864 -0.053099  0.114129   \n",
       "के    -0.124854 -0.068364 -0.364002 -0.383810  0.172778 -0.003992 -0.168285   \n",
       "।     -0.150303 -0.221962 -0.626972 -0.456973  0.470308 -0.213924 -0.112261   \n",
       ",     -0.160962 -0.361875 -0.190295 -0.389795  0.216144 -0.050485 -0.192041   \n",
       "में   -0.209305  0.087706 -0.289885 -0.381661  0.208492 -0.069959 -0.083901   \n",
       "है    -0.163855  0.305773 -0.496239 -0.385285 -0.023947 -0.196321 -0.206529   \n",
       "की    -0.179126 -0.189791 -0.428677 -0.418656  0.017380 -0.009977 -0.137235   \n",
       "और    -0.129317 -0.197093 -0.321741 -0.402978  0.194263 -0.088978 -0.178371   \n",
       "से    -0.204383  0.001643 -0.448961 -0.339362 -0.116989 -0.131715 -0.173486   \n",
       "\n",
       "            7         8         9      ...          290       291       292  \\\n",
       "<unk> -0.014982  0.241724 -1.362962    ...    -0.105750  0.000138  0.252063   \n",
       "<pad>  0.146933 -0.150632  0.331193    ...     0.010165  0.212148 -0.309134   \n",
       "के     0.075885 -0.030859 -2.595019    ...    -0.032839  0.077952  0.063838   \n",
       "।     -0.437237  0.708551  3.330899    ...    -0.472635  0.085662 -0.071612   \n",
       ",     -2.112090 -0.685204  4.356941    ...    -0.240056 -0.039866 -0.316080   \n",
       "में    1.090752 -0.049023  2.092380    ...    -0.080802  0.119540 -0.651442   \n",
       "है     0.549503  0.392630  1.914814    ...    -0.138870  0.029760 -0.292541   \n",
       "की     0.626785  0.379204 -0.712076    ...    -0.199796  0.077377 -0.134444   \n",
       "और     1.334814 -0.463591 -1.078326    ...    -0.347151 -0.018696 -0.109680   \n",
       "से    -0.483954 -0.084200 -0.646569    ...    -0.130293  0.198819 -1.254919   \n",
       "\n",
       "            293       294       295       296       297       298       299  \n",
       "<unk>  0.322336 -0.234095 -0.076334 -0.357609 -0.376448  0.016019 -0.099436  \n",
       "<pad> -0.256550  0.050639  0.086903  0.069713  0.099145 -0.143334  0.060667  \n",
       "के     0.858946 -0.325743  0.632531 -0.839241 -0.341894 -0.058192 -0.158500  \n",
       "।     -1.019616 -0.371203 -0.139772 -1.050309 -0.382592 -0.067022 -0.025990  \n",
       ",      0.169702 -0.359465 -0.325764 -0.872892 -0.408702 -0.118085 -0.199788  \n",
       "में    0.565120 -0.270207  0.440532 -0.828658 -0.318647 -0.055147 -0.139348  \n",
       "है    -1.653055 -0.312935  0.663974 -0.684114 -0.543373  0.049822 -0.194706  \n",
       "की     0.530944 -0.384973 -0.683671 -0.876726 -0.259445 -0.011620 -0.162889  \n",
       "और     0.606728 -0.374177 -0.089182 -0.906496 -0.386876 -0.097840 -0.239605  \n",
       "से     0.746580 -0.378496  0.199947  0.131102 -0.393537 -0.015223 -0.101342  \n",
       "\n",
       "[10 rows x 300 columns]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hindi2vec = pd.DataFrame(to_np(TEXT_vec.vocab.vectors))\n",
    "hindi2vec.index = TEXT_vec.vocab.itos\n",
    "hindi2vec.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-25T03:55:23.616554Z",
     "start_time": "2018-01-25T03:55:03.872457Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(21816, 300)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hindi2save = hindi2vec[~hindi2vec.index.str.contains(' ')]\n",
    "#remove lines with weird characters due to bad segmentation\n",
    "# thai2save = thai2save.iloc[3:-48,:]\n",
    "hindi2save.to_csv(f'{PATH}models//hindi2vec.vec',sep=' ',header=False, line_terminator='\\n')\n",
    "#add NB_ROWS NB_COLS as header\n",
    "hindi2save.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#TODO Convert hindivectors to gensim word2vec format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-01-25T03:59:08.601229Z",
     "start_time": "2018-01-25T03:59:01.839159Z"
    }
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-57-a18e20d11324>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mmodel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msave_word2vec_format\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mf'{PATH}models/hindi2vec.vec'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;34mf'{PATH}models/hindi2vec.vocab'\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "model.save_word2vec_format(f'{PATH}models/hindi2vec.vec',f'{PATH}models/hindi2vec.vocab',False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "fastAI",
   "language": "python",
   "name": "fastai"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}