Chapter14 - Intro to LSTMs - Part 2 - Learn to Write Like Shakespeare.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "class Tensor (object):\n",
    "    \n",
    "    def __init__(self,data,\n",
    "                 autograd=False,\n",
    "                 creators=None,\n",
    "                 creation_op=None,\n",
    "                 id=None):\n",
    "        \n",
    "        self.data = np.array(data)\n",
    "        self.autograd = autograd\n",
    "        self.grad = None\n",
    "        if(id is None):\n",
    "            self.id = np.random.randint(0,100000)\n",
    "        else:\n",
    "            self.id = id\n",
    "        \n",
    "        self.creators = creators\n",
    "        self.creation_op = creation_op\n",
    "        self.children = {}\n",
    "        \n",
    "        if(creators is not None):\n",
    "            for c in creators:\n",
    "                if(self.id not in c.children):\n",
    "                    c.children[self.id] = 1\n",
    "                else:\n",
    "                    c.children[self.id] += 1\n",
    "\n",
    "    def all_children_grads_accounted_for(self):\n",
    "        for id,cnt in self.children.items():\n",
    "            if(cnt != 0):\n",
    "                return False\n",
    "        return True \n",
    "        \n",
    "    def backward(self,grad=None, grad_origin=None):\n",
    "        if(self.autograd):\n",
    " \n",
    "            if(grad is None):\n",
    "                grad = Tensor(np.ones_like(self.data))\n",
    "\n",
    "            if(grad_origin is not None):\n",
    "                if(self.children[grad_origin.id] == 0):\n",
    "                    raise Exception(\"cannot backprop more than once\")\n",
    "                else:\n",
    "                    self.children[grad_origin.id] -= 1\n",
    "\n",
    "            if(self.grad is None):\n",
    "                self.grad = grad\n",
    "            else:\n",
    "                self.grad += grad\n",
    "            \n",
    "            # grads must not have grads of their own\n",
    "            assert grad.autograd == False\n",
    "            \n",
    "            # only continue backpropping if there's something to\n",
    "            # backprop into and if all gradients (from children)\n",
    "            # are accounted for override waiting for children if\n",
    "            # \"backprop\" was called on this variable directly\n",
    "            if(self.creators is not None and \n",
    "               (self.all_children_grads_accounted_for() or \n",
    "                grad_origin is None)):\n",
    "\n",
    "                if(self.creation_op == \"add\"):\n",
    "                    self.creators[0].backward(self.grad, self)\n",
    "                    self.creators[1].backward(self.grad, self)\n",
    "                    \n",
    "                if(self.creation_op == \"sub\"):\n",
    "                    self.creators[0].backward(Tensor(self.grad.data), self)\n",
    "                    self.creators[1].backward(Tensor(self.grad.__neg__().data), self)\n",
    "\n",
    "                if(self.creation_op == \"mul\"):\n",
    "                    new = self.grad * self.creators[1]\n",
    "                    self.creators[0].backward(new , self)\n",
    "                    new = self.grad * self.creators[0]\n",
    "                    self.creators[1].backward(new, self)                    \n",
    "                    \n",
    "                if(self.creation_op == \"mm\"):\n",
    "                    c0 = self.creators[0]\n",
    "                    c1 = self.creators[1]\n",
    "                    new = self.grad.mm(c1.transpose())\n",
    "                    c0.backward(new)\n",
    "                    new = self.grad.transpose().mm(c0).transpose()\n",
    "                    c1.backward(new)\n",
    "                    \n",
    "                if(self.creation_op == \"transpose\"):\n",
    "                    self.creators[0].backward(self.grad.transpose())\n",
    "\n",
    "                if(\"sum\" in self.creation_op):\n",
    "                    dim = int(self.creation_op.split(\"_\")[1])\n",
    "                    self.creators[0].backward(self.grad.expand(dim,\n",
    "                                                               self.creators[0].data.shape[dim]))\n",
    "\n",
    "                if(\"expand\" in self.creation_op):\n",
    "                    dim = int(self.creation_op.split(\"_\")[1])\n",
    "                    self.creators[0].backward(self.grad.sum(dim))\n",
    "                    \n",
    "                if(self.creation_op == \"neg\"):\n",
    "                    self.creators[0].backward(self.grad.__neg__())\n",
    "                    \n",
    "                if(self.creation_op == \"sigmoid\"):\n",
    "                    ones = Tensor(np.ones_like(self.grad.data))\n",
    "                    self.creators[0].backward(self.grad * (self * (ones - self)))\n",
    "                \n",
    "                if(self.creation_op == \"tanh\"):\n",
    "                    ones = Tensor(np.ones_like(self.grad.data))\n",
    "                    self.creators[0].backward(self.grad * (ones - (self * self)))\n",
    "                \n",
    "                if(self.creation_op == \"index_select\"):\n",
    "                    new_grad = np.zeros_like(self.creators[0].data)\n",
    "                    indices_ = self.index_select_indices.data.flatten()\n",
    "                    grad_ = grad.data.reshape(len(indices_), -1)\n",
    "                    for i in range(len(indices_)):\n",
    "                        new_grad[indices_[i]] += grad_[i]\n",
    "                    self.creators[0].backward(Tensor(new_grad))\n",
    "                    \n",
    "                if(self.creation_op == \"cross_entropy\"):\n",
    "                    dx = self.softmax_output - self.target_dist\n",
    "                    self.creators[0].backward(Tensor(dx))\n",
    "                    \n",
    "    def __add__(self, other):\n",
    "        if(self.autograd and other.autograd):\n",
    "            return Tensor(self.data + other.data,\n",
    "                          autograd=True,\n",
    "                          creators=[self,other],\n",
    "                          creation_op=\"add\")\n",
    "        return Tensor(self.data + other.data)\n",
    "\n",
    "    def __neg__(self):\n",
    "        if(self.autograd):\n",
    "            return Tensor(self.data * -1,\n",
    "                          autograd=True,\n",
    "                          creators=[self],\n",
    "                          creation_op=\"neg\")\n",
    "        return Tensor(self.data * -1)\n",
    "    \n",
    "    def __sub__(self, other):\n",
    "        if(self.autograd and other.autograd):\n",
    "            return Tensor(self.data - other.data,\n",
    "                          autograd=True,\n",
    "                          creators=[self,other],\n",
    "                          creation_op=\"sub\")\n",
    "        return Tensor(self.data - other.data)\n",
    "    \n",
    "    def __mul__(self, other):\n",
    "        if(self.autograd and other.autograd):\n",
    "            return Tensor(self.data * other.data,\n",
    "                          autograd=True,\n",
    "                          creators=[self,other],\n",
    "                          creation_op=\"mul\")\n",
    "        return Tensor(self.data * other.data)    \n",
    "\n",
    "    def sum(self, dim):\n",
    "        if(self.autograd):\n",
    "            return Tensor(self.data.sum(dim),\n",
    "                          autograd=True,\n",
    "                          creators=[self],\n",
    "                          creation_op=\"sum_\"+str(dim))\n",
    "        return Tensor(self.data.sum(dim))\n",
    "    \n",
    "    def expand(self, dim,copies):\n",
    "\n",
    "        trans_cmd = list(range(0,len(self.data.shape)))\n",
    "        trans_cmd.insert(dim,len(self.data.shape))\n",
    "        new_data = self.data.repeat(copies).reshape(list(self.data.shape) + [copies]).transpose(trans_cmd)\n",
    "        \n",
    "        if(self.autograd):\n",
    "            return Tensor(new_data,\n",
    "                          autograd=True,\n",
    "                          creators=[self],\n",
    "                          creation_op=\"expand_\"+str(dim))\n",
    "        return Tensor(new_data)\n",
    "    \n",
    "    def transpose(self):\n",
    "        if(self.autograd):\n",
    "            return Tensor(self.data.transpose(),\n",
    "                          autograd=True,\n",
    "                          creators=[self],\n",
    "                          creation_op=\"transpose\")\n",
    "        \n",
    "        return Tensor(self.data.transpose())\n",
    "    \n",
    "    def mm(self, x):\n",
    "        if(self.autograd):\n",
    "            return Tensor(self.data.dot(x.data),\n",
    "                          autograd=True,\n",
    "                          creators=[self,x],\n",
    "                          creation_op=\"mm\")\n",
    "        return Tensor(self.data.dot(x.data))\n",
    "    \n",
    "    def sigmoid(self):\n",
    "        if(self.autograd):\n",
    "            return Tensor(1 / (1 + np.exp(-self.data)),\n",
    "                          autograd=True,\n",
    "                          creators=[self],\n",
    "                          creation_op=\"sigmoid\")\n",
    "        return Tensor(1 / (1 + np.exp(-self.data)))\n",
    "\n",
    "    def tanh(self):\n",
    "        if(self.autograd):\n",
    "            return Tensor(np.tanh(self.data),\n",
    "                          autograd=True,\n",
    "                          creators=[self],\n",
    "                          creation_op=\"tanh\")\n",
    "        return Tensor(np.tanh(self.data))\n",
    "    \n",
    "    def index_select(self, indices):\n",
    "\n",
    "        if(self.autograd):\n",
    "            new = Tensor(self.data[indices.data],\n",
    "                         autograd=True,\n",
    "                         creators=[self],\n",
    "                         creation_op=\"index_select\")\n",
    "            new.index_select_indices = indices\n",
    "            return new\n",
    "        return Tensor(self.data[indices.data])\n",
    "\n",
    "    def softmax(self):\n",
    "        temp = np.exp(self.data)\n",
    "        softmax_output = temp / np.sum(temp,\n",
    "                                       axis=len(self.data.shape)-1,\n",
    "                                       keepdims=True)\n",
    "        return softmax_output    \n",
    "    \n",
    "    def cross_entropy(self, target_indices):\n",
    "\n",
    "        temp = np.exp(self.data)\n",
    "        softmax_output = temp / np.sum(temp,\n",
    "                                       axis=len(self.data.shape)-1,\n",
    "                                       keepdims=True)\n",
    "        \n",
    "        t = target_indices.data.flatten()\n",
    "        p = softmax_output.reshape(len(t),-1)\n",
    "        target_dist = np.eye(p.shape[1])[t]\n",
    "        loss = -(np.log(p) * (target_dist)).sum(1).mean()\n",
    "    \n",
    "        if(self.autograd):\n",
    "            out = Tensor(loss,\n",
    "                         autograd=True,\n",
    "                         creators=[self],\n",
    "                         creation_op=\"cross_entropy\")\n",
    "            out.softmax_output = softmax_output\n",
    "            out.target_dist = target_dist\n",
    "            return out\n",
    "\n",
    "        return Tensor(loss)\n",
    "        \n",
    "    \n",
    "    def __repr__(self):\n",
    "        return str(self.data.__repr__())\n",
    "    \n",
    "    def __str__(self):\n",
    "        return str(self.data.__str__())  \n",
    "    \n",
    "class Tanh(Layer):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "    def forward(self, input):\n",
    "        return input.tanh()\n",
    "    \n",
    "class Sigmoid(Layer):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "    def forward(self, input):\n",
    "        return input.sigmoid()\n",
    "\n",
    "class Layer(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        self.parameters = list()\n",
    "        \n",
    "    def get_parameters(self):\n",
    "        return self.parameters\n",
    "\n",
    "    \n",
    "class SGD(object):\n",
    "    \n",
    "    def __init__(self, parameters, alpha=0.1):\n",
    "        self.parameters = parameters\n",
    "        self.alpha = alpha\n",
    "    \n",
    "    def zero(self):\n",
    "        for p in self.parameters:\n",
    "            p.grad.data *= 0\n",
    "        \n",
    "    def step(self, zero=True):\n",
    "        \n",
    "        for p in self.parameters:\n",
    "            \n",
    "            p.data -= p.grad.data * self.alpha\n",
    "            \n",
    "            if(zero):\n",
    "                p.grad.data *= 0\n",
    "\n",
    "\n",
    "class Linear(Layer):\n",
    "\n",
    "    def __init__(self, n_inputs, n_outputs):\n",
    "        super().__init__()\n",
    "        W = np.random.randn(n_inputs, n_outputs) * np.sqrt(2.0/(n_inputs))\n",
    "        self.weight = Tensor(W, autograd=True)\n",
    "        self.bias = Tensor(np.zeros(n_outputs), autograd=True)\n",
    "        \n",
    "        self.parameters.append(self.weight)\n",
    "        self.parameters.append(self.bias)\n",
    "\n",
    "    def forward(self, input):\n",
    "        return input.mm(self.weight)+self.bias.expand(0,len(input.data))\n",
    "\n",
    "\n",
    "class Sequential(Layer):\n",
    "    \n",
    "    def __init__(self, layers=list()):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.layers = layers\n",
    "    \n",
    "    def add(self, layer):\n",
    "        self.layers.append(layer)\n",
    "        \n",
    "    def forward(self, input):\n",
    "        for layer in self.layers:\n",
    "            input = layer.forward(input)\n",
    "        return input\n",
    "    \n",
    "    def get_parameters(self):\n",
    "        params = list()\n",
    "        for l in self.layers:\n",
    "            params += l.get_parameters()\n",
    "        return params\n",
    "\n",
    "\n",
    "class Embedding(Layer):\n",
    "    \n",
    "    def __init__(self, vocab_size, dim):\n",
    "        super().__init__()\n",
    "        \n",
    "        self.vocab_size = vocab_size\n",
    "        self.dim = dim\n",
    "        \n",
    "        # this random initialiation style is just a convention from word2vec\n",
    "        self.weight = Tensor((np.random.rand(vocab_size, dim) - 0.5) / dim, autograd=True)\n",
    "        \n",
    "        self.parameters.append(self.weight)\n",
    "    \n",
    "    def forward(self, input):\n",
    "        return self.weight.index_select(input)\n",
    "\n",
    "\n",
    "class Tanh(Layer):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "    def forward(self, input):\n",
    "        return input.tanh()\n",
    "\n",
    "\n",
    "class Sigmoid(Layer):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "    def forward(self, input):\n",
    "        return input.sigmoid()\n",
    "    \n",
    "\n",
    "class CrossEntropyLoss(object):\n",
    "    \n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "    \n",
    "    def forward(self, input, target):\n",
    "        return input.cross_entropy(target)\n",
    "\n",
    "    \n",
    "class RNNCell(Layer):\n",
    "    \n",
    "    def __init__(self, n_inputs, n_hidden, n_output, activation='sigmoid'):\n",
    "        super().__init__()\n",
    "\n",
    "        self.n_inputs = n_inputs\n",
    "        self.n_hidden = n_hidden\n",
    "        self.n_output = n_output\n",
    "        \n",
    "        if(activation == 'sigmoid'):\n",
    "            self.activation = Sigmoid()\n",
    "        elif(activation == 'tanh'):\n",
    "            self.activation == Tanh()\n",
    "        else:\n",
    "            raise Exception(\"Non-linearity not found\")\n",
    "\n",
    "        self.w_ih = Linear(n_inputs, n_hidden)\n",
    "        self.w_hh = Linear(n_hidden, n_hidden)\n",
    "        self.w_ho = Linear(n_hidden, n_output)\n",
    "        \n",
    "        self.parameters += self.w_ih.get_parameters()\n",
    "        self.parameters += self.w_hh.get_parameters()\n",
    "        self.parameters += self.w_ho.get_parameters()        \n",
    "    \n",
    "    def forward(self, input, hidden):\n",
    "        from_prev_hidden = self.w_hh.forward(hidden)\n",
    "        combined = self.w_ih.forward(input) + from_prev_hidden\n",
    "        new_hidden = self.activation.forward(combined)\n",
    "        output = self.w_ho.forward(new_hidden)\n",
    "        return output, new_hidden\n",
    "    \n",
    "    def init_hidden(self, batch_size=1):\n",
    "        return Tensor(np.zeros((batch_size,self.n_hidden)), autograd=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 1: RNN Character Language Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys,random,math\n",
    "from collections import Counter\n",
    "import numpy as np\n",
    "import sys\n",
    "\n",
    "np.random.seed(0)\n",
    "# dataset from http://karpathy.github.io/2015/05/21/rnn-effectiveness/\n",
    "f = open('shakespear.txt','r')\n",
    "raw = f.read()\n",
    "f.close()\n",
    "\n",
    "vocab = list(set(raw))\n",
    "word2index = {}\n",
    "for i,word in enumerate(vocab):\n",
    "    word2index[word]=i\n",
    "indices = np.array(list(map(lambda x:word2index[x], raw)))\n",
    "\n",
    "embed = Embedding(vocab_size=len(vocab),dim=512)\n",
    "model = RNNCell(n_inputs=512, n_hidden=512, n_output=len(vocab))\n",
    "\n",
    "criterion = CrossEntropyLoss()\n",
    "optim = SGD(parameters=model.get_parameters() + embed.get_parameters(), alpha=0.05)\n",
    "\n",
    "batch_size = 32\n",
    "bptt = 16\n",
    "n_batches = int((indices.shape[0] / (batch_size)))\n",
    "\n",
    "trimmed_indices = indices[:n_batches*batch_size]\n",
    "batched_indices = trimmed_indices.reshape(batch_size, n_batches).transpose()\n",
    "\n",
    "input_batched_indices = batched_indices[0:-1]\n",
    "target_batched_indices = batched_indices[1:]\n",
    "\n",
    "n_bptt = int(((n_batches-1) / bptt))\n",
    "input_batches = input_batched_indices[:n_bptt*bptt].reshape(n_bptt,bptt,batch_size)\n",
    "target_batches = target_batched_indices[:n_bptt*bptt].reshape(n_bptt, bptt, batch_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'That,'"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([30,  4, 37, 42, 52])"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indices[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[30, 23,  7, 31, 50,  4, 30,  0,  0, 37, 37,  9, 50,  0, 52, 21,\n",
       "         0, 61,  4,  7,  9, 37,  0,  0,  0, 33,  0, 33, 33,  0,  0,  8],\n",
       "       [ 4, 21, 21, 31, 26,  9, 37, 53, 61,  8, 59,  9, 59, 42,  0, 21,\n",
       "        27, 50, 50, 21, 59,  1, 27, 57,  4,  8, 13, 20,  8, 20, 58,  0],\n",
       "       [37, 21, 14,  9,  9, 37, 59, 51, 50,  0, 59,  7, 57,  4, 16, 55,\n",
       "         9, 53, 59, 21, 57,  9, 35,  9,  9, 40, 50, 52,  0, 50, 50, 27],\n",
       "       [42, 17, 30, 35,  0, 35, 27,  0, 53,  8,  0,  0, 23,  9,  9, 43,\n",
       "         9, 52, 61, 39, 51,  0, 50,  9, 35, 59, 53, 21, 11, 35, 20, 50],\n",
       "       [52, 43, 33, 14, 16,  0, 50, 35,  0, 50, 16, 36, 21,  0,  0, 38,\n",
       "         8,  0,  0, 50, 23, 61, 16, 26, 52, 37, 59, 43, 53,  9,  9, 61]])"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batched_indices[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[30, 23,  7, 31, 50,  4, 30,  0,  0, 37, 37,  9, 50,  0, 52, 21,\n",
       "         0, 61,  4,  7,  9, 37,  0,  0,  0, 33,  0, 33, 33,  0,  0,  8],\n",
       "       [ 4, 21, 21, 31, 26,  9, 37, 53, 61,  8, 59,  9, 59, 42,  0, 21,\n",
       "        27, 50, 50, 21, 59,  1, 27, 57,  4,  8, 13, 20,  8, 20, 58,  0],\n",
       "       [37, 21, 14,  9,  9, 37, 59, 51, 50,  0, 59,  7, 57,  4, 16, 55,\n",
       "         9, 53, 59, 21, 57,  9, 35,  9,  9, 40, 50, 52,  0, 50, 50, 27],\n",
       "       [42, 17, 30, 35,  0, 35, 27,  0, 53,  8,  0,  0, 23,  9,  9, 43,\n",
       "         9, 52, 61, 39, 51,  0, 50,  9, 35, 59, 53, 21, 11, 35, 20, 50],\n",
       "       [52, 43, 33, 14, 16,  0, 50, 35,  0, 50, 16, 36, 21,  0,  0, 38,\n",
       "         8,  0,  0, 50, 23, 61, 16, 26, 52, 37, 59, 43, 53,  9,  9, 61]])"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_batches[0][0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 4, 21, 21, 31, 26,  9, 37, 53, 61,  8, 59,  9, 59, 42,  0, 21,\n",
       "        27, 50, 50, 21, 59,  1, 27, 57,  4,  8, 13, 20,  8, 20, 58,  0],\n",
       "       [37, 21, 14,  9,  9, 37, 59, 51, 50,  0, 59,  7, 57,  4, 16, 55,\n",
       "         9, 53, 59, 21, 57,  9, 35,  9,  9, 40, 50, 52,  0, 50, 50, 27],\n",
       "       [42, 17, 30, 35,  0, 35, 27,  0, 53,  8,  0,  0, 23,  9,  9, 43,\n",
       "         9, 52, 61, 39, 51,  0, 50,  9, 35, 59, 53, 21, 11, 35, 20, 50],\n",
       "       [52, 43, 33, 14, 16,  0, 50, 35,  0, 50, 16, 36, 21,  0,  0, 38,\n",
       "         8,  0,  0, 50, 23, 61, 16, 26, 52, 37, 59, 43, 53,  9,  9, 61],\n",
       "       [ 0, 48, 51, 57, 33,  9, 42,  9, 16,  0,  9,  4, 21, 57, 16,  3,\n",
       "         0, 20, 27, 51, 21, 50,  0,  0,  0, 16, 57,  8, 31,  0, 52, 51]])"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_batches[0][0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(iterations=400):\n",
    "    for iter in range(iterations):\n",
    "        total_loss = 0\n",
    "        n_loss = 0\n",
    "\n",
    "        hidden = model.init_hidden(batch_size=batch_size)\n",
    "        for batch_i in range(len(input_batches)):\n",
    "\n",
    "            hidden = Tensor(hidden.data, autograd=True)\n",
    "            loss = None\n",
    "            losses = list()\n",
    "            for t in range(bptt):\n",
    "                input = Tensor(input_batches[batch_i][t], autograd=True)\n",
    "                rnn_input = embed.forward(input=input)\n",
    "                output, hidden = model.forward(input=rnn_input, hidden=hidden)\n",
    "\n",
    "                target = Tensor(target_batches[batch_i][t], autograd=True)    \n",
    "                batch_loss = criterion.forward(output, target)\n",
    "                losses.append(batch_loss)\n",
    "                if(t == 0):\n",
    "                    loss = batch_loss\n",
    "                else:\n",
    "                    loss = loss + batch_loss\n",
    "            for loss in losses:\n",
    "                \"\"\n",
    "            loss.backward()\n",
    "            optim.step()\n",
    "            total_loss += loss.data\n",
    "            log = \"\\r Iter:\" + str(iter)\n",
    "            log += \" - Batch \"+str(batch_i+1)+\"/\"+str(len(input_batches))\n",
    "            log += \" - Loss:\" + str(np.exp(total_loss / (batch_i+1)))\n",
    "            if(batch_i == 0):\n",
    "                log += \" - \" + generate_sample(n=70, init_char='\\n').replace(\"\\n\",\" \")\n",
    "            if(batch_i % 10 == 0 or batch_i-1 == len(input_batches)):\n",
    "                sys.stdout.write(log)\n",
    "        optim.alpha *= 0.99\n",
    "        print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Iter:0 - Batch 191/195 - Loss:148.00388828554404                                                                       \n",
      " Iter:1 - Batch 191/195 - Loss:20.588816924127116 mhnethet tttttt t t t thett ttth thetttt thetth t tt t tttheth t t ttt\n",
      " Iter:2 - Batch 191/195 - Loss:15.282461756020384  h th the the the th the the thet the the the the the the the t the th\n",
      " Iter:3 - Batch 191/195 - Loss:13.048394405821716dh th the the the t the t the the the t the th the the the the the the\n",
      " Iter:4 - Batch 191/195 - Loss:11.774988723385185dh the the t th the t the the the the the the th th the t the th th th\n",
      " Iter:5 - Batch 191/195 - Loss:10.989391522842189dh the the the the the the t the the the the the the t the the the    \n",
      " Iter:6 - Batch 191/195 - Loss:10.400423045063138dh the the the t the t the the the the the the the the the the the the\n",
      " Iter:7 - Batch 191/195 - Loss:9.9208775761205764da th the the the the the t the the the the the the the the the the th\n",
      " Iter:8 - Batch 191/195 - Loss:9.511516020913637 da the the the the the the the the the the the the the the the the the\n",
      " Iter:9 - Batch 191/195 - Loss:9.148623407454805 th the the the the the the the  the the the the the t the the the the \n",
      " Iter:10 - Batch 191/195 - Loss:8.821849800958601th the the the the the the the the the the the the the the the th the \n",
      " Iter:11 - Batch 191/195 - Loss:8.523034059325719 th the the the the the the the the the the the the the the the the th \n",
      " Iter:12 - Batch 191/195 - Loss:8.253664350983296 the th the the t the the the the the the the the the the the the the t\n",
      " Iter:13 - Batch 191/195 - Loss:8.0069672337493755th the the the the the the the the the th the the the the the the the \n",
      " Iter:14 - Batch 191/195 - Loss:7.7739372955274885the the  to the the the the the the the the the the the the the the th\n",
      " Iter:15 - Batch 191/195 - Loss:7.5511986438587625 the th the the the the the the the the the the the the the the the the\n",
      " Iter:16 - Batch 191/195 - Loss:7.3390047318671625the the th the the the the the the the the the the the the the the  ha\n",
      " Iter:17 - Batch 191/195 - Loss:7.1373636919747545the the the the th the the the   he the the the the the t the the the \n",
      " Iter:18 - Batch 191/195 - Loss:6.9446057248731465the the the the the the the the  h the the the the the the the the  he\n",
      " Iter:19 - Batch 191/195 - Loss:6.759464262811306 the the the the the the  he the the the the the the  to the the the th\n",
      " Iter:20 - Batch 191/195 - Loss:6.5803850681516345the the the the the the the the the the the   hich the the the the the\n",
      " Iter:21 - Batch 191/195 - Loss:6.4063886200003965t wh  we a  to the the the w the the the the the  ha the   hace the th\n",
      " Iter:22 - Batch 191/195 - Loss:6.2365373926845556were the the  h the the    hace the the w the the the the the the th\n",
      " Iter:23 - Batch 191/195 - Loss:6.0703683905820465the with the the the the t the  ha the  ha the the the the the the    \n",
      " Iter:24 - Batch 191/195 - Loss:5.9101324074886685the with the the  to the the the the the   h the the the the    h the \n",
      " Iter:25 - Batch 191/195 - Loss:5.749468001942508- to the   have the the the    he t the with   himst the the the the the\n",
      " Iter:26 - Batch 191/195 - Loss:5.5937695794009965o the the w the the w the the the t the w t  ha s  h the   have the w\n",
      " Iter:27 - Batch 191/195 - Loss:5.4456949428981775to the w t t the the the the the   h the the the the  the   himst the \n",
      " Iter:28 - Batch 191/195 - Loss:5.2906332130566365to the we a face the the  t to the the the the the   hace the wast the\n",
      " Iter:29 - Batch 191/195 - Loss:5.142299805001576 to the the was wa say whe   ha   hace the  t  hath a s and the was  he\n",
      " Iter:30 - Batch 191/195 - Loss:4.9942134181372375to the the h the  to the the the wa sam the  to the the the the the   \n",
      " Iter:31 - Batch 191/195 - Loss:4.8489935397557655to the  t to the the the the the the the t the the the the the  t t th\n",
      " Iter:32 - Batch 191/195 - Loss:4.7071025984761385to the the the the the was the the  to the was the the the was the  th\n",
      " Iter:33 - Batch 191/195 - Loss:4.570672395786387 to the was the the the the the the the  have the  have the the the  t \n",
      " Iter:34 - Batch 191/195 - Loss:4.4383705299373585 to the was the the  t t the we a say    ha  we a say well the w  hath \n",
      " Iter:35 - Batch 191/195 - Loss:4.3076334964831915to the the t the  h the  t the the the    and the the the the t and th\n",
      " Iter:36 - Batch 191/195 - Loss:4.178814890600767 to the the the the  w woth a shall the the   and the word the  w  ha  \n",
      " Iter:37 - Batch 191/195 - Loss:4.0426158644820305to the was was was well the h the   well the the   and the have the   \n",
      " Iter:38 - Batch 191/195 - Loss:3.9240497331706576 to the the w woth the  t the the  we a s and the the the     and the h\n",
      " Iter:39 - Batch 191/195 - Loss:3.7904568222256065to the the the the the the himst the the the  wo the w wother  s and  \n",
      " Iter:40 - Batch 191/195 - Loss:3.6623246602453072 to the the the the the w  ha  w w the  to the the the the the the the \n",
      " Iter:41 - Batch 161/195 - Loss:3.5868070218013654to the  won  speake and   reping the the the      restains wone was   "
     ]
    }
   ],
   "source": [
    "train(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Iter:73 - Batch 133/195 - Loss:1.4278971135025023\n"
     ]
    }
   ],
   "source": [
    "train(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Iter:0 - Batch 191/195 - Loss:1.5069461425416464I of ke hadon, and kidains:  r pren  a suco most this thuse are, sir, \n",
      " Iter:1 - Batch 191/195 - Loss:1.4231989098544708 I  hot thi pomimm.  CHENER which thy  weick the man's and w rege, To t\n",
      " Iter:2 - Batch 191/195 - Loss:1.3957566042826986 I was aga now, I am the s eakn man's and s as h the seq man's and say \n",
      " Iter:3 - Batch 191/195 - Loss:1.3711573215374844 I wase, and sa day wear and sa day wear and saked  in the chas to the \n",
      " Iter:4 - Batch 191/195 - Loss:1.3063271397979614 I was ageing on the shak to the  eakn mine th every sted and say wend \n",
      " Iter:5 - Batch 191/195 - Loss:1.2782573171839362 I and eytif the stakn my hath somet mest and say we say, sich her, my \n",
      " Iter:6 - Batch 191/195 - Loss:1.2572971763852931 I was   eakn m of the strich  e ha bucthis sty which thy t upof thy to\n",
      " Iter:7 - Batch 191/195 - Loss:1.2334132737824353 I  hot this stod  age; And so not agabe sich her, my good lorget the t\n",
      " Iter:8 - Batch 191/195 - Loss:1.2115008935021373 I am ever a speakt?  ANG HENO P w  part doth ared face.  CHENERNE: I a\n",
      " Iter:9 - Batch 191/195 - Loss:1.2267139408557375I am in me himmand?  ANG HENO P  hath  will seem.  CHENE: I all the ch\n",
      " Iter:10 - Batch 191/195 - Loss:1.2364342251533027 I and ey B wi hort    offould sich her, my wear a speake that I  overy\n",
      " Iter:11 - Batch 191/195 - Loss:1.2156765978429473 I a pent me h as I seem.  MALV: I  hake of the strich  e  ouns  arbl s\n",
      " Iter:12 - Batch 191/195 - Loss:1.1859873698816483I was namids, and e to  eaknge, This, and I  ho make h to  eak to you \n",
      " Iter:13 - Batch 191/195 - Loss:1.1631807879803502 I am  andak: Hak: Ha not ded aids, agats unt and sa day wear and the m\n",
      " Iter:14 - Batch 191/195 - Loss:1.1926253572249608 I war and  h love, and say werr, be gent and semingmbee, and their hit\n",
      " Iter:15 - Batch 191/195 - Loss:1.2492829050499197 I was it good an the staknd eyes, an the st and emanot a day wear are \n",
      " Iter:16 - Batch 191/195 - Loss:1.1733306502007356 I  hot this seq me I have  e have stold  a  ag poor untruck wear andre\n",
      " Iter:17 - Batch 191/195 - Loss:1.2099463980632899I af  hold the sir, this st in thee, Whought the man pr would have tha\n",
      " Iter:18 - Batch 191/195 - Loss:1.2007235850050733 I out Gind ey lorget I no your  agh! Ghou are diar as   poto your j p \n",
      " Iter:19 - Batch 191/195 - Loss:1.1451445876574398 I am it wot untruck wear and I have t untruck w off cha cont would hav\n",
      " Iter:20 - Batch 191/195 - Loss:1.1292903594165675 I am  andak: Hak: f of my hath   rept thou m this, and such her flick \n",
      " Iter:21 - Batch 191/195 - Loss:1.1084157630708358 I a pent me himm.  CHENRE: I  seeavon  i go her, Yo make him sich thin\n",
      " Iter:22 - Batch 191/195 - Loss:1.1019740170386951 I af kis I at th to the tamn thy too, sir, sich thin the  rent the tam\n",
      " Iter:23 - Batch 191/195 - Loss:1.0904913027135066 I af light to the  rent that I will  rent the tamn the sea  he  rentle\n",
      " Iter:24 - Batch 191/195 - Loss:1.0860704762748256 I af  here it theight th to the mands; and thy too, sir, sich think in\n",
      " Iter:25 - Batch 191/195 - Loss:1.0777115519702094 I af and the man! Thich thy the tamnous s wear a good and thy too, sir\n",
      " Iter:26 - Batch 191/195 - Loss:1.0733095018722293  lace.  CHENRE: I all the chast  hath s as I some.  BENTIO: Why LOSPHi\n",
      " Iter:27 - Batch 191/195 - Loss:1.0665165094326634I af the mands; and thy too, s eath are, met' outshat. somet  res and \n",
      " Iter:28 - Batch 191/195 - Loss:1.0625091525035777 I af  here?  CHEOPIO: This a pent me him the contre commort who it wot\n",
      " Iter:29 - Batch 191/195 - Loss:1.0591559773670037 I a pent me maid wear as ear and I have my  hath in the chaids; and th\n",
      " Iter:30 - Batch 61/195 - Loss:1.0533843281265225 I af the mands your  spooolding  resentleming from this, and I have my"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-135-34a3a1792739>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0mlosses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbptt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m             \u001b[0minput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_batches\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mbatch_i\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mautograd\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m             \u001b[0mrnn_input\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0membed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m             \u001b[0moutput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mrnn_input\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhidden\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-100-3a4a21d7080c>\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, data, autograd, creators, creation_op, id)\u001b[0m\n\u001b[1;32m    264\u001b[0m                  id=None):\n\u001b[1;32m    265\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mautograd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "train(100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I war ded abdons would.\n",
      "\n",
      "CHENRO:\n",
      "Why, speed no virth to her,\n",
      "Plirt, goth Plish love,\n",
      "Befion\n",
      " hath if be fe woulds is feally your hir, the confectife to the nightion\n",
      "As  rent Ron my hath iom\n",
      "the worse, my goth Plish love,\n",
      "Befion\n",
      "Ass untrucerty of my fernight this we namn?\n",
      "\n",
      "ANG, makes:\n",
      "That's bond confect fe comes not commonour would be forch the conflill, the confectiffould b off your day, sind it sequns, be gent Rour jus confectife to the nightion\n",
      "As   poing from your jus  eep of m look o perves, the worse, my goth Plis rept ffough we name:\n",
      "Thould be good lorges ever word.\n",
      "\n",
      "DESS:\n",
      "Where exbinder: if not conflill, the confectife to the nightion\n",
      "As co move, sir, this we namn?\n",
      "\n",
      "ANG VINE PAET:\n",
      "There was courter hower how, my goth Plish lo res\n",
      "Toures\n",
      "ever wo formall, have abon, with a good lorges ever word.\n",
      "\n",
      "DESS:\n",
      "Where exbinder: if not conflill, the confectife to the nightion\n",
      "As co mo not?\n",
      "\n",
      "ANG:\n",
      "I horses ever with gent may. Thour hot never wear.\n",
      "\n",
      "PAGTI by him,\n",
      "And conflill, the confectif you le.\n",
      "\n",
      "CALV\n",
      "BENTI SIII:\n",
      "Sou looks no virther\n",
      "But kill be he'l of the worse, my goth Plish love,\n",
      "Befion\n",
      "Ass upof me threet, such her,\n",
      "I speak:\n",
      " hath if be fe woulds is feally your h offent, such her,\n",
      "I speak:\n",
      "Seco my same\n",
      "And kidon, with gent and the of Ser prent me n may. Thou my plest near:\n",
      "Sou looks no virther\n",
      "But kill be he'l of the worse, my goth Plish love,\n",
      "Befion\n",
      "Ass untrucerty of my fernight this we namn?\n",
      "\n",
      "ANG, makes:\n",
      "That's bond confect fe comes,\n",
      "Sour confectife to the nightion\n",
      "As  rent Ron my  list forch the confectife to the nightion\n",
      "As co mo not?\n",
      "\n",
      "ANG:\n",
      "I  wo face;\n",
      "The conflind is thee\n",
      "ter's same I have my lorges ever word.\n",
      "\n",
      "DESS:\n",
      "Where exbinder:\n",
      "Yot you le.\n",
      "\n",
      "CALV CHENRE:\n",
      "I will be her,\n",
      " oo by the confectife to the nightion\n",
      "As  rent Ron my hath iom\n",
      "the worse, my goth P would not?\n",
      "\n",
      "ANG:\n",
      "I h don: my fereigrts, word.\n",
      "\n",
      "DESS:\n",
      "Where exbinder: if not conflill, the confectife to the nightion\n",
      "As comest this?\n",
      "\n",
      "ANGOR I lot comest thou my plest near:\n",
      "Sou looks no virther\n",
      "But kill be he'l\n"
     ]
    }
   ],
   "source": [
    "def generate_sample(n=30, init_char=' '):\n",
    "    s = \"\"\n",
    "    hidden = model.init_hidden(batch_size=1)\n",
    "    input = Tensor(np.array([word2index[init_char]]))\n",
    "    for i in range(n):\n",
    "        rnn_input = embed.forward(input)\n",
    "        output, hidden = model.forward(input=rnn_input, hidden=hidden)\n",
    "        output.data *= 10\n",
    "        temp_dist = output.softmax()\n",
    "        temp_dist /= temp_dist.sum()\n",
    "\n",
    "        m = (temp_dist > np.random.rand()).argmax()\n",
    "#         m = output.data.argmax()\n",
    "        c = vocab[m]\n",
    "        input = Tensor(np.array([m]))\n",
    "        s += c\n",
    "    return s\n",
    "print(generate_sample(n=2000, init_char='\\n'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}