3_LSTM_for_POS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LSTM/Bi-LSTM for Part-of-Speech Tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F \n",
    "\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_data(training_file):\n",
    "    #  # Input data format is \"natural_JJ language_NN...\"\n",
    "    training_data = []\n",
    "    with open(training_file, 'r') as f:\n",
    "        for line in f:\n",
    "            words, tags = list(), list()\n",
    "            for word_tag in line.split():\n",
    "                w, t = tuple(word_tag.split('_'))\n",
    "                words.append(w)\n",
    "                tags.append(t)\n",
    "            training_data.append((words, tags))\n",
    "    return training_data\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_dict(training_data):\n",
    "    \n",
    "    word2index = {}\n",
    "    tag2index = {}\n",
    "    for sentence,tags in training_data:\n",
    "        for word in sentence:\n",
    "            if word not in word2index:\n",
    "                word2index[word] = len(word2index)\n",
    "        for tag in tags:\n",
    "            if tag not in tag2index:\n",
    "                tag2index[tag] = len(tag2index)\n",
    "    \n",
    "    return word2index, tag2index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "data download:\n",
    "https://github.com/llhthinker/nlptutorial-exercise/blob/master/data/wiki-en-train.norm_pos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_file = './data/wiki-en-train.norm_pos'\n",
    "data = load_data(training_file)\n",
    "word2index, tag2index = build_dict(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(['Natural', 'language', 'processing', '-LRB-', 'NLP', '-RRB-', 'is', 'a', 'field', 'of', 'computer', 'science', ',', 'artificial', 'intelligence', '-LRB-', 'also', 'called', 'machine', 'learning', '-RRB-', ',', 'and', 'linguistics', 'concerned', 'with', 'the', 'interactions', 'between', 'computers', 'and', 'human', '-LRB-', 'natural', '-RRB-', 'languages', '.'], ['JJ', 'NN', 'NN', '-LRB-', 'NN', '-RRB-', 'VBZ', 'DT', 'NN', 'IN', 'NN', 'NN', ',', 'JJ', 'NN', '-LRB-', 'RB', 'VBN', 'NN', 'NN', '-RRB-', ',', 'CC', 'NNS', 'VBN', 'IN', 'DT', 'NNS', 'IN', 'NNS', 'CC', 'JJ', '-LRB-', 'JJ', '-RRB-', 'NNS', '.'])\n",
      "1301\n"
     ]
    }
   ],
   "source": [
    "print(data[0])\n",
    "data_length = len(data)\n",
    "print(data_length)\n",
    "split = int(data_length * 0.7)\n",
    "training_data = data[:split]\n",
    "valid_data = data[split:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_seq(seq, seq2index):\n",
    "    indexs = [seq2index[w] for w in seq]\n",
    "    tensor = torch.LongTensor(indexs)\n",
    "    return Variable(tensor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM\n",
    "Ref: http://pytorch.org/tutorials/beginner/nlp/sequence_models_tutorial.html#lstm-s-in-pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def do_eval(model, valid_data):\n",
    "    acc = 0.0\n",
    "    for sentence, tags in valid_data:\n",
    "        sentence_in = prepare_seq(sentence, word2index)\n",
    "        targets = prepare_seq(tags, tag2index)\n",
    "        tag_scores = model(sentence_in)\n",
    "        _, predicted = torch.max(tag_scores.data, 1)\n",
    "        predicted = predicted.view(len(targets)).numpy()\n",
    "        targets = targets.data.numpy()\n",
    "        correct_num = np.sum((predicted == targets))\n",
    "        acc += correct_num / len(sentence)\n",
    "        \n",
    "    print(\"Valid set accuracy:\", acc / len(valid_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class Config():\n",
    "    def __init__(self):\n",
    "        self.vocab_size = len(word2index)\n",
    "        self.tagset_size = len(tag2index)\n",
    "        self.embedding_dim = 16\n",
    "        self.hidden_dim = 16\n",
    "\n",
    "config = Config()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class LSTMTagger(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super(LSTMTagger, self).__init__()\n",
    "        \n",
    "        self.embedding = nn.Embedding(config.vocab_size, config.embedding_dim)\n",
    "        \n",
    "        self.lstm = nn.LSTM(input_size=config.embedding_dim, hidden_size=config.hidden_dim)\n",
    "        \n",
    "        self.hidden2tag = nn.Linear(in_features=config.hidden_dim, out_features=config.tagset_size)\n",
    "    \n",
    "    def init_hidden(self):\n",
    "        # Before we've done anything, we dont have any hidden state.\n",
    "        # Refer to the Pytorch documentation to see exactly\n",
    "        # why they have this dimensionality.\n",
    "        # The axes semantics are (num_layers, minibatch_size, hidden_dim)\n",
    "        return (Variable(torch.zeros(1, 1, config.hidden_dim)),\n",
    "                Variable(torch.zeros(1, 1, config.hidden_dim)))\n",
    "    \n",
    "    \n",
    "    def forward(self, sentence):\n",
    "        embed = self.embedding(sentence)\n",
    "        # lstm input (seq_len, batch, input_size)\n",
    "        embed = embed.view(len(sentence), 1, -1)\n",
    "        lstm_out = self.lstm(embed)[0]\n",
    "#         print(lstm_out.size())\n",
    "        # output (seq_len, batch, hidden_size * num_directions)\n",
    "        # -> (seq_len, hidden_size * num_directions)\n",
    "        lstm_out_reshape = lstm_out.view(len(sentence), -1)\n",
    "#         print(lstm_out_reshape.size())\n",
    "        tag_space = self.hidden2tag(lstm_out_reshape)\n",
    "        tag_scores = F.log_softmax(tag_space)\n",
    "        \n",
    "        return tag_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,   200] loss: 0.324\n",
      "[1,   400] loss: 0.278\n",
      "[1,   600] loss: 0.255\n",
      "[1,   800] loss: 0.240\n",
      "Valid set accuracy: 0.402996500242\n",
      "[2,   200] loss: 0.210\n",
      "[2,   400] loss: 0.202\n",
      "[2,   600] loss: 0.193\n",
      "[2,   800] loss: 0.192\n",
      "Valid set accuracy: 0.515497941108\n",
      "[3,   200] loss: 0.172\n",
      "[3,   400] loss: 0.170\n",
      "[3,   600] loss: 0.167\n",
      "[3,   800] loss: 0.171\n",
      "Valid set accuracy: 0.572909377034\n",
      "[4,   200] loss: 0.154\n",
      "[4,   400] loss: 0.153\n",
      "[4,   600] loss: 0.152\n",
      "[4,   800] loss: 0.158\n",
      "Valid set accuracy: 0.598427265634\n",
      "[5,   200] loss: 0.142\n",
      "[5,   400] loss: 0.142\n",
      "[5,   600] loss: 0.141\n",
      "[5,   800] loss: 0.149\n",
      "Valid set accuracy: 0.617734514808\n",
      "[6,   200] loss: 0.133\n",
      "[6,   400] loss: 0.133\n",
      "[6,   600] loss: 0.132\n",
      "[6,   800] loss: 0.142\n",
      "Valid set accuracy: 0.635239141785\n",
      "[7,   200] loss: 0.126\n",
      "[7,   400] loss: 0.126\n",
      "[7,   600] loss: 0.125\n",
      "[7,   800] loss: 0.136\n",
      "Valid set accuracy: 0.648055034768\n",
      "[8,   200] loss: 0.119\n",
      "[8,   400] loss: 0.119\n",
      "[8,   600] loss: 0.118\n",
      "[8,   800] loss: 0.130\n",
      "Valid set accuracy: 0.658687795378\n",
      "[9,   200] loss: 0.113\n",
      "[9,   400] loss: 0.113\n",
      "[9,   600] loss: 0.112\n",
      "[9,   800] loss: 0.125\n",
      "Valid set accuracy: 0.665739946919\n",
      "[10,   200] loss: 0.109\n",
      "[10,   400] loss: 0.108\n",
      "[10,   600] loss: 0.107\n",
      "[10,   800] loss: 0.121\n",
      "Valid set accuracy: 0.673159377351\n"
     ]
    }
   ],
   "source": [
    "model = LSTMTagger(config)\n",
    "loss_function = nn.NLLLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.1)\n",
    "\n",
    "\n",
    "for epoch in range(10):\n",
    "    count = 0\n",
    "    running_loss = 0.0\n",
    "    for sentence, tags in training_data:\n",
    "        \n",
    "        # Step 1. Remember that Pytorch accumulates gradients.\n",
    "        # We need to clear them out before each instance\n",
    "        model.zero_grad()\n",
    "\n",
    "        # Also, we need to clear out the hidden state of the LSTM,\n",
    "        # detaching it from its history on the last instance.\n",
    "        model.hidden = model.init_hidden()\n",
    "\n",
    "        # Step 2. Get our inputs ready for the network, that is, turn them into\n",
    "        # Variables of word indices.\n",
    "        sentence_in = prepare_seq(sentence, word2index)\n",
    "        targets = prepare_seq(tags, tag2index)\n",
    "\n",
    "        # Step 3. Run our forward pass.\n",
    "        tag_scores = model(sentence_in)\n",
    "\n",
    "        # Step 4. Compute the loss, gradients, and update the parameters by\n",
    "        #  calling optimizer.step()\n",
    "        loss = loss_function(tag_scores, targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # print statistics\n",
    "        running_loss += loss.data[0]\n",
    "        if count % 200 == 199:\n",
    "            print('[%d, %5d] loss: %.3f' %\n",
    "                  (epoch + 1, count + 1, running_loss / 2000))\n",
    "            running_loss = 0.0\n",
    "        count += 1\n",
    "\n",
    "    do_eval(model, valid_data)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bi-LSTM\n",
    "- 实验结果显示使用Bi-LSTM有一定的提升效果, 0.673->0.692\n",
    "- 在较大数据集上的效果可能更好（Bi-LSTM参数比LSTM多）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class BiLSTMTagger(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super(BiLSTMTagger, self).__init__()\n",
    "        \n",
    "        self.embedding = nn.Embedding(config.vocab_size, config.embedding_dim)\n",
    "        \n",
    "        self.lstm = nn.LSTM(input_size=config.embedding_dim, \n",
    "                            hidden_size=config.hidden_dim, \n",
    "                            num_layers= 2,\n",
    "                            bidirectional=True)\n",
    "        \n",
    "        self.hidden2tag = nn.Linear(in_features=config.hidden_dim*2, out_features=config.tagset_size)\n",
    "    \n",
    "    def init_hidden(self):\n",
    "        # Before we've done anything, we dont have any hidden state.\n",
    "        # Refer to the Pytorch documentation to see exactly\n",
    "        # why they have this dimensionality.\n",
    "        # The axes semantics are (num_layers, minibatch_size, hidden_dim)\n",
    "        return (Variable(torch.zeros(1, 1, config.hidden_dim)),\n",
    "                Variable(torch.zeros(1, 1, config.hidden_dim)))\n",
    "    \n",
    "    \n",
    "    def forward(self, sentence):\n",
    "        embed = self.embedding(sentence)\n",
    "        # lstm input (seq_len, batch, input_size)\n",
    "        embed = embed.view(len(sentence), 1, -1)\n",
    "        lstm_out = self.lstm(embed)[0]\n",
    "#         print(lstm_out.size())\n",
    "        # output (seq_len, batch, hidden_size * num_directions)\n",
    "        # -> (seq_len, hidden_size * num_directions)\n",
    "        lstm_out_reshape = lstm_out.view(len(sentence), -1)\n",
    "#         print(lstm_out_reshape.size())\n",
    "        tag_space = self.hidden2tag(lstm_out_reshape)\n",
    "        tag_scores = F.log_softmax(tag_space)\n",
    "        \n",
    "        return tag_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,   200] loss: 0.320\n",
      "[1,   400] loss: 0.294\n",
      "[1,   600] loss: 0.284\n",
      "[1,   800] loss: 0.279\n",
      "Valid set accuracy: 0.239784096097\n",
      "[2,   200] loss: 0.267\n",
      "[2,   400] loss: 0.260\n",
      "[2,   600] loss: 0.251\n",
      "[2,   800] loss: 0.242\n",
      "Valid set accuracy: 0.394927813154\n",
      "[3,   200] loss: 0.215\n",
      "[3,   400] loss: 0.205\n",
      "[3,   600] loss: 0.195\n",
      "[3,   800] loss: 0.192\n",
      "Valid set accuracy: 0.508007925312\n",
      "[4,   200] loss: 0.173\n",
      "[4,   400] loss: 0.168\n",
      "[4,   600] loss: 0.161\n",
      "[4,   800] loss: 0.167\n",
      "Valid set accuracy: 0.554756792656\n",
      "[5,   200] loss: 0.150\n",
      "[5,   400] loss: 0.146\n",
      "[5,   600] loss: 0.140\n",
      "[5,   800] loss: 0.149\n",
      "Valid set accuracy: 0.607302061207\n",
      "[6,   200] loss: 0.133\n",
      "[6,   400] loss: 0.130\n",
      "[6,   600] loss: 0.125\n",
      "[6,   800] loss: 0.136\n",
      "Valid set accuracy: 0.628884194833\n",
      "[7,   200] loss: 0.120\n",
      "[7,   400] loss: 0.117\n",
      "[7,   600] loss: 0.114\n",
      "[7,   800] loss: 0.126\n",
      "Valid set accuracy: 0.647488870077\n",
      "[8,   200] loss: 0.109\n",
      "[8,   400] loss: 0.107\n",
      "[8,   600] loss: 0.104\n",
      "[8,   800] loss: 0.117\n",
      "Valid set accuracy: 0.66449194369\n",
      "[9,   200] loss: 0.101\n",
      "[9,   400] loss: 0.098\n",
      "[9,   600] loss: 0.095\n",
      "[9,   800] loss: 0.110\n",
      "Valid set accuracy: 0.679149682571\n",
      "[10,   200] loss: 0.093\n",
      "[10,   400] loss: 0.090\n",
      "[10,   600] loss: 0.088\n",
      "[10,   800] loss: 0.103\n",
      "Valid set accuracy: 0.692018188963\n"
     ]
    }
   ],
   "source": [
    "model = BiLSTMTagger(config)\n",
    "loss_function = nn.NLLLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.1)\n",
    "\n",
    "\n",
    "for epoch in range(10):\n",
    "    count = 0\n",
    "    running_loss = 0.0\n",
    "    for sentence, tags in training_data:\n",
    "        \n",
    "        # Step 1. Remember that Pytorch accumulates gradients.\n",
    "        # We need to clear them out before each instance\n",
    "        model.zero_grad()\n",
    "\n",
    "        # Also, we need to clear out the hidden state of the LSTM,\n",
    "        # detaching it from its history on the last instance.\n",
    "        model.hidden = model.init_hidden()\n",
    "\n",
    "        # Step 2. Get our inputs ready for the network, that is, turn them into\n",
    "        # Variables of word indices.\n",
    "        sentence_in = prepare_seq(sentence, word2index)\n",
    "        targets = prepare_seq(tags, tag2index)\n",
    "\n",
    "        # Step 3. Run our forward pass.\n",
    "        tag_scores = model(sentence_in)\n",
    "\n",
    "        # Step 4. Compute the loss, gradients, and update the parameters by\n",
    "        #  calling optimizer.step()\n",
    "        loss = loss_function(tag_scores, targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # print statistics\n",
    "        running_loss += loss.data[0]\n",
    "        if count % 200 == 199:\n",
    "            print('[%d, %5d] loss: %.3f' %\n",
    "                  (epoch + 1, count + 1, running_loss / 2000))\n",
    "            running_loss = 0.0\n",
    "        count += 1\n",
    "\n",
    "    do_eval(model, valid_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Augmenting the LSTM part-of-speech tagger with character-level features\n",
    "\n",
    "To do this, let $c_w$ be the character-level representation of\n",
    "word $w$. Let $x_w$ be the word embedding as before. Then\n",
    "the input to our sequence model is the concatenation of $x_w$ and\n",
    "$c_w$. So if $x_w$ has dimension 16, and $c_w$\n",
    "dimension 8, then our LSTM should accept an input of dimension 24.\n",
    "\n",
    "To get the character level representation, do an LSTM over the\n",
    "characters of a word, and let $c_w$ be the final hidden state of\n",
    "this LSTM. Hints:\n",
    "\n",
    "* There are going to be two LSTM's in your new model.\n",
    "  The original one that outputs POS tag scores, and the new one that\n",
    "  outputs a character-level representation of each word.\n",
    "* To do a sequence model over characters, you will have to embed characters.\n",
    "  The character embeddings will be the input to the character LSTM.\n",
    "\n",
    "---\n",
    "实验结果显示提升效果非常明显: 0.673 -> 0.782"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'-': 0, 'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8, 'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15, 'p': 16, 'q': 17, 'r': 18, 's': 19, 't': 20, 'u': 21, 'v': 22, 'w': 23, 'x': 24, 'y': 25, 'z': 26, 'A': 27, 'B': 28, 'C': 29, 'D': 30, 'E': 31, 'F': 32, 'G': 33, 'H': 34, 'I': 35, 'J': 36, 'K': 37, 'L': 38, 'M': 39, 'N': 40, 'O': 41, 'P': 42, 'Q': 43, 'R': 44, 'S': 45, 'T': 46, 'U': 47, 'V': 48, 'W': 49, 'X': 50, 'Y': 51, 'Z': 52}\n"
     ]
    }
   ],
   "source": [
    "char2index = dict()\n",
    "char2index['-'] = 0  # char = '-' if char is not ascii_letters\n",
    "for i, c in enumerate(string.ascii_letters):\n",
    "    char2index[c] = i+1\n",
    "print(char2index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_char_seq(word, char2index):\n",
    "    indexs = []\n",
    "    for c in word:\n",
    "        if c not in char2index:\n",
    "            indexs.append(0)\n",
    "        else:\n",
    "            indexs.append(char2index[c])\n",
    "    tensor = torch.LongTensor(indexs)\n",
    "    return Variable(tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Natural\n",
      "Variable containing:\n",
      " 40\n",
      "  1\n",
      " 20\n",
      " 21\n",
      " 18\n",
      "  1\n",
      " 12\n",
      "[torch.LongTensor of size 7]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(training_data[0][0][0])\n",
    "print(prepare_char_seq(training_data[0][0][0], char2index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConfigPlus():\n",
    "    def __init__(self):\n",
    "        self.charset_size = len(char2index)\n",
    "        self.vocab_size = len(word2index)\n",
    "        self.tagset_size = len(tag2index)\n",
    "        self.word_embedding_dim = 16\n",
    "        self.char_embedding_dim = 8\n",
    "        self.char_hidden_dim = 8\n",
    "        self.hidden_dim = 16\n",
    "\n",
    "config = ConfigPlus()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class CharLSTMTagger(nn.Module):\n",
    "    def __init__(self, config):\n",
    "        super(CharLSTMTagger, self).__init__()\n",
    "        \n",
    "        self.word_embedding = nn.Embedding(config.vocab_size, config.word_embedding_dim)\n",
    "        self.char_embedding = nn.Embedding(config.charset_size, config.char_embedding_dim)\n",
    "        \n",
    "        self.char_lstm = nn.LSTM(input_size=config.char_embedding_dim, \n",
    "                                 hidden_size=config.char_hidden_dim)\n",
    "        \n",
    "        self.lstm = nn.LSTM(input_size=config.word_embedding_dim+config.char_hidden_dim,\n",
    "                            hidden_size=config.hidden_dim)\n",
    "        \n",
    "        self.hidden2tag = nn.Linear(in_features=config.hidden_dim, out_features=config.tagset_size)\n",
    "    \n",
    "    def init_hidden(self):\n",
    "        # Before we've done anything, we dont have any hidden state.\n",
    "        # Refer to the Pytorch documentation to see exactly\n",
    "        # why they have this dimensionality.\n",
    "        # The axes semantics are (num_layers, minibatch_size, hidden_dim)\n",
    "        return (Variable(torch.zeros(1, 1, config.hidden_dim)),\n",
    "                Variable(torch.zeros(1, 1, config.hidden_dim)))\n",
    "    \n",
    "    \n",
    "    def forward(self, sentence, words):\n",
    "        word_embed = self.word_embedding(sentence)\n",
    "        # lstm input (seq_len, batch, input_size)\n",
    "        word_embed = word_embed.view(len(sentence), 1, -1)\n",
    "        \n",
    "        seg_char_embed = []\n",
    "        for word in words:\n",
    "            word_chars = prepare_char_seq(word, char2index)\n",
    "            char_embed = self.char_embedding(word_chars)\n",
    "            char_embed = char_embed.view(len(word_chars), 1, -1)\n",
    "            # character-level representation of each word is\n",
    "            # the final hidden state of char_lstm. \n",
    "            word_char_embed = self.char_lstm(char_embed)[1][0]  # h_n(the final hidden state of char_lstm.)\n",
    "            \n",
    "            seg_char_embed.append(word_char_embed)\n",
    "        #  (seq_len, batch, char_hidden_dim)\n",
    "        seg_char_embed = torch.cat(seg_char_embed, dim=0)\n",
    "        # new_embed_dim = word_embedding_dim + char_hidden_dim(the dim of character-level representation)\n",
    "        embed = torch.cat((word_embed, seg_char_embed), dim=2)\n",
    "        \n",
    "        lstm_out = self.lstm(embed)[0]\n",
    "#         print(lstm_out.size())\n",
    "        # output (seq_len, batch, hidden_size * num_directions)\n",
    "        # -> (seq_len, hidden_size * num_directions)\n",
    "        lstm_out_reshape = lstm_out.view(len(sentence), -1)\n",
    "#         print(lstm_out_reshape.size())\n",
    "        tag_space = self.hidden2tag(lstm_out_reshape)\n",
    "        tag_scores = F.log_softmax(tag_space)\n",
    "        \n",
    "        return tag_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def do_eval(model, valid_data):\n",
    "    acc = 0.0\n",
    "    for sentence, tags in valid_data:\n",
    "        sentence_in = prepare_seq(sentence, word2index)\n",
    "        targets = prepare_seq(tags, tag2index)\n",
    "        tag_scores = model(sentence_in, sentence)\n",
    "        _, predicted = torch.max(tag_scores.data, 1)\n",
    "        predicted = predicted.view(len(targets)).numpy()\n",
    "        targets = targets.data.numpy()\n",
    "        correct_num = np.sum((predicted == targets))\n",
    "        acc += correct_num / len(sentence)\n",
    "        \n",
    "    print(\"Valid set accuracy:\", acc / len(valid_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training_data size:  910\n",
      "valid_data size:  391\n",
      "[1,   200] loss: 0.323\n",
      "[1,   400] loss: 0.281\n",
      "[1,   600] loss: 0.253\n",
      "[1,   800] loss: 0.236\n",
      "Valid set accuracy: 0.433072208189\n",
      "[2,   200] loss: 0.206\n",
      "[2,   400] loss: 0.196\n",
      "[2,   600] loss: 0.186\n",
      "[2,   800] loss: 0.185\n",
      "Valid set accuracy: 0.552397303525\n",
      "[3,   200] loss: 0.160\n",
      "[3,   400] loss: 0.152\n",
      "[3,   600] loss: 0.142\n",
      "[3,   800] loss: 0.144\n",
      "Valid set accuracy: 0.646899055425\n",
      "[4,   200] loss: 0.125\n",
      "[4,   400] loss: 0.122\n",
      "[4,   600] loss: 0.118\n",
      "[4,   800] loss: 0.123\n",
      "Valid set accuracy: 0.682827064641\n",
      "[5,   200] loss: 0.109\n",
      "[5,   400] loss: 0.107\n",
      "[5,   600] loss: 0.104\n",
      "[5,   800] loss: 0.110\n",
      "Valid set accuracy: 0.70766459289\n",
      "[6,   200] loss: 0.096\n",
      "[6,   400] loss: 0.096\n",
      "[6,   600] loss: 0.092\n",
      "[6,   800] loss: 0.100\n",
      "Valid set accuracy: 0.731857723941\n",
      "[7,   200] loss: 0.086\n",
      "[7,   400] loss: 0.087\n",
      "[7,   600] loss: 0.083\n",
      "[7,   800] loss: 0.092\n",
      "Valid set accuracy: 0.749488303579\n",
      "[8,   200] loss: 0.079\n",
      "[8,   400] loss: 0.081\n",
      "[8,   600] loss: 0.076\n",
      "[8,   800] loss: 0.086\n",
      "Valid set accuracy: 0.762959056436\n",
      "[9,   200] loss: 0.074\n",
      "[9,   400] loss: 0.075\n",
      "[9,   600] loss: 0.071\n",
      "[9,   800] loss: 0.081\n",
      "Valid set accuracy: 0.771392478611\n",
      "[10,   200] loss: 0.068\n",
      "[10,   400] loss: 0.070\n",
      "[10,   600] loss: 0.066\n",
      "[10,   800] loss: 0.076\n",
      "Valid set accuracy: 0.781739832955\n"
     ]
    }
   ],
   "source": [
    "model = CharLSTMTagger(config)\n",
    "loss_function = nn.NLLLoss()\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.1)\n",
    "train_size = len(training_data)\n",
    "valid_size = len(valid_data)\n",
    "print(\"training_data size: \", train_size)\n",
    "print(\"valid_data size: \", valid_size)\n",
    "\n",
    "# training_data = training_data[:train_size//2]\n",
    "\n",
    "for epoch in range(10):\n",
    "    count = 0\n",
    "    running_loss = 0.0\n",
    "    for sentence, tags in training_data:\n",
    "        \n",
    "        # Step 1. Remember that Pytorch accumulates gradients.\n",
    "        # We need to clear them out before each instance\n",
    "        model.zero_grad()\n",
    "\n",
    "        # Also, we need to clear out the hidden state of the LSTM,\n",
    "        # detaching it from its history on the last instance.\n",
    "        model.hidden = model.init_hidden()\n",
    "\n",
    "        # Step 2. Get our inputs ready for the network, that is, turn them into\n",
    "        # Variables of word indices.\n",
    "        sentence_in = prepare_seq(sentence, word2index)\n",
    "        targets = prepare_seq(tags, tag2index)\n",
    "\n",
    "        # Step 3. Run our forward pass.\n",
    "        tag_scores = model(sentence_in, sentence)\n",
    "\n",
    "        # Step 4. Compute the loss, gradients, and update the parameters by\n",
    "        #  calling optimizer.step()\n",
    "        loss = loss_function(tag_scores, targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        # print statistics\n",
    "        running_loss += loss.data[0]\n",
    "        if count % 200 == 199:\n",
    "            print('[%d, %5d] loss: %.3f' %\n",
    "                  (epoch + 1, count + 1, running_loss / 2000))\n",
    "            running_loss = 0.0\n",
    "        count += 1\n",
    "    do_eval(model, valid_data)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}