basic_lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from string import punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv').fillna(\"Empty\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "404290\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>How can I increase the speed of my internet co...</td>\n",
       "      <td>How can Internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
       "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>Which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "2   2     5     6  How can I increase the speed of my internet co...   \n",
       "3   3     7     8  Why am I mentally very lonely? How can I solve...   \n",
       "4   4     9    10  Which one dissolve in water quikly sugar, salt...   \n",
       "\n",
       "                                           question2  is_duplicate  \n",
       "0  What is the step by step guide to invest in sh...             0  \n",
       "1  What would happen if the Indian government sto...             0  \n",
       "2  How can Internet speed be increased by hacking...             0  \n",
       "3  Find the remainder when [math]23^{24}[/math] i...             0  \n",
       "4            Which fish would survive in salt water?             0  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(train))\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "question_words = train['question1'].apply(lambda x: x.split()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "What      137930\n",
       "How        93409\n",
       "Why        32808\n",
       "Is         22485\n",
       "Which      17468\n",
       "Can        11605\n",
       "I           7906\n",
       "Who         7547\n",
       "Do          6789\n",
       "Where       6641\n",
       "If          5961\n",
       "What's      5638\n",
       "Does        5090\n",
       "Are         4542\n",
       "When        3227\n",
       "Should      3129\n",
       "Will        2981\n",
       "In          1967\n",
       "My          1723\n",
       "Did         1285\n",
       "I'm         1165\n",
       "Has         1100\n",
       "Would       1053\n",
       "Have         887\n",
       "Was          783\n",
       "Could        719\n",
       "As           582\n",
       "Name: question1, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "value_counts = question_words.value_counts()\n",
    "value_counts = value_counts[value_counts>500]\n",
    "value_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "404290\n",
      "404290\n",
      "Max sentence length 1169.00\n",
      "Min sentence length 1.00\n",
      "Average sentence length 59.82\n",
      "Median senctence length 51.00\n",
      "Standard deviation 31.96\n",
      "Mean plus 2 X STD 123.75\n",
      "Percent greater than max_words 0.10\n"
     ]
    }
   ],
   "source": [
    "sentence_lengths1 = train['question1'].apply(len)\n",
    "sentence_lengths2 = train['question2'].apply(len)\n",
    "print(len(sentence_lengths1))\n",
    "print(len(sentence_lengths2))\n",
    "sentence_lengths = sentence_lengths1.append(sentence_lengths2)\n",
    "print(\"Max sentence length %.2f\" % np.max(sentence_lengths))\n",
    "print(\"Min sentence length %.2f\" % np.min(sentence_lengths))\n",
    "print(\"Average sentence length %.2f\" % np.mean(sentence_lengths))\n",
    "print(\"Median senctence length %.2f\" % np.median(sentence_lengths))\n",
    "print(\"Standard deviation %.2f\" % np.std(sentence_lengths))\n",
    "print(\"Mean plus 2 X STD %.2f\" % (2 * np.std(sentence_lengths) + np.mean(sentence_lengths)))\n",
    "\n",
    "max_words = 100\n",
    "greater_than = sentence_lengths[sentence_lengths > max_words]\n",
    "print(\"Percent greater than max_words %.2f\" % (len(greater_than)/ len(sentence_lengths)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Train and Test Splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# train_data, test_data = train_test_split(train, test_size=0.25)\n",
    "train_questions1 = train['question1'].tolist()\n",
    "train_questions2 = train['question2'].tolist()\n",
    "train_data_labels = train['is_duplicate'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def convert_to_ints(questions, seq_len):\n",
    "    all_questions = ' '.join([c.translate(str.maketrans(\"\", \"\", punctuation)) for c in questions])\n",
    "    words = all_questions.split()\n",
    "    from collections import Counter\n",
    "    counts = Counter(words)\n",
    "    vocab = sorted(counts, key=counts.get, reverse=True)\n",
    "    vocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}\n",
    "    vocab_to_int[''] = len(vocab_to_int.keys()) + 1\n",
    "\n",
    "    questions_ints = []\n",
    "    for each in questions:\n",
    "        questions_ints.append([vocab_to_int[word.translate(str.maketrans(\"\", \"\", punctuation))] for word in each.split()])\n",
    "\n",
    "    features = np.zeros((len(questions_ints), seq_len), dtype=int)\n",
    "    for i, row in enumerate(questions_ints):\n",
    "        features[i, -len(row):] = np.array(row)[:seq_len]\n",
    "        \n",
    "    return features, vocab_to_int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "question_ints1, question1_vocab_to_int = convert_to_ints(train_questions1, max_words)\n",
    "question_ints2, question2_vocab_to_int = convert_to_ints(train_questions2, max_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t\tFeature Shapes:\n",
      "Train set: \t\t(323432, 100) \n",
      "Validation set: \t(40429, 100) \n",
      "Test set: \t\t(40429, 100)\n"
     ]
    }
   ],
   "source": [
    "split_frac = 0.8\n",
    "split_idx = int(len(question_ints1)*split_frac)\n",
    "train_q1, val_q1 = question_ints1[:split_idx], question_ints1[split_idx:]\n",
    "train_q2, val_q2 = question_ints2[:split_idx], question_ints2[split_idx:]\n",
    "train_y, val_y = train_data_labels[:split_idx], train_data_labels[split_idx:]\n",
    "train_y = np.array(train_y)\n",
    "val_y = np.array(val_y)\n",
    "\n",
    "test_idx = int(len(val_q1)*0.5)\n",
    "val_q1, test_q1 = val_q1[:test_idx], val_q1[test_idx:]\n",
    "val_q2, test_q2 = val_q2[:test_idx], val_q2[test_idx:]\n",
    "val_y, test_y = val_y[:test_idx], val_y[test_idx:]\n",
    "\n",
    "print(\"\\t\\t\\tFeature Shapes:\")\n",
    "print(\"Train set: \\t\\t{}\".format(train_q1.shape), \n",
    "      \"\\nValidation set: \\t{}\".format(val_q1.shape),\n",
    "      \"\\nTest set: \\t\\t{}\".format(test_q1.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "lstm_size = 256\n",
    "lstm_layers = 1\n",
    "batch_size = 512\n",
    "learning_rate = 0.001\n",
    "# Size of the embedding vectors (number of units in the embedding layer)\n",
    "embed_size = 300 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# n_words1 = len(question1_vocab_to_int)\n",
    "# n_words2 = len(question2_vocab_to_int)\n",
    "\n",
    "# # Create the graph object\n",
    "# graph = tf.Graph()\n",
    "# # Add nodes to the graph\n",
    "# with graph.as_default():\n",
    "#     inputs_1 = tf.placeholder(tf.int32, [None, None], name='inputs')\n",
    "#     inputs_2 = tf.placeholder(tf.int32, [None, None], name='inputs')\n",
    "#     labels_ = tf.placeholder(tf.int32, [None, None], name='labels')\n",
    "#     keep_prob = tf.placeholder(tf.float32, name='keep_prob')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n_words1 = len(question1_vocab_to_int) + 1\n",
    "n_words2 = len(question2_vocab_to_int) + 1\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# Create the graph object\n",
    "# graph = tf.Graph()\n",
    "# Add nodes to the graph\n",
    "with tf.variable_scope('questions', reuse=None):\n",
    "    inputs_1 = tf.placeholder(tf.int32, [None, None], name='inputs')\n",
    "    inputs_2 = tf.placeholder(tf.int32, [None, None], name='inputs')\n",
    "    labels_ = tf.placeholder(tf.int32, [None, None], name='labels')\n",
    "    keep_prob = tf.placeholder(tf.float32, name='keep_prob')\n",
    "    \n",
    "\n",
    "    embedding1 = tf.Variable(tf.random_uniform((n_words1, embed_size), -1, 1))\n",
    "    embed1 = tf.nn.embedding_lookup(embedding1, inputs_1)\n",
    "    \n",
    "    # Your basic LSTM cell\n",
    "    lstm1 = tf.contrib.rnn.BasicLSTMCell(lstm_size)\n",
    "    \n",
    "    # Add dropout to the cell\n",
    "    drop1 = tf.contrib.rnn.DropoutWrapper(lstm1, output_keep_prob=keep_prob)\n",
    "    \n",
    "    # Stack up multiple LSTM layers, for deep learning\n",
    "    cell1 = tf.contrib.rnn.MultiRNNCell([drop1] * lstm_layers)\n",
    "    \n",
    "    # Getting an initial state of all zeros\n",
    "    initial_state1 = cell1.zero_state(batch_size, tf.float32)\n",
    "\n",
    "    outputs1, final_state1 = tf.nn.dynamic_rnn(cell1, embed1,\n",
    "                                             initial_state=initial_state1)\n",
    "    \n",
    "with tf.variable_scope('questions', reuse=True):\n",
    "    embedding2 = tf.Variable(tf.random_uniform((n_words2, embed_size), -1, 1))\n",
    "    embed2 = tf.nn.embedding_lookup(embedding2, inputs_2)\n",
    "    \n",
    "    # Your basic LSTM cell\n",
    "    lstm2 = tf.contrib.rnn.BasicLSTMCell(lstm_size)\n",
    "    \n",
    "    # Add dropout to the cell\n",
    "    drop2 = tf.contrib.rnn.DropoutWrapper(lstm2, output_keep_prob=keep_prob)\n",
    "    \n",
    "    # Stack up multiple LSTM layers, for deep learning\n",
    "    cell2 = tf.contrib.rnn.MultiRNNCell([drop2] * lstm_layers)\n",
    "    \n",
    "    # Getting an initial state of all zeros\n",
    "    initial_state2 = cell2.zero_state(batch_size, tf.float32)\n",
    "    \n",
    "    outputs2, final_state2 = tf.nn.dynamic_rnn(cell2, embed2,\n",
    "                                             initial_state=initial_state2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope('questions', reuse=False):\n",
    "    outputs = tf.concat([outputs1[:, -1], outputs2[:,-1]], 1)\n",
    "    final_state = tf.concat([final_state1, final_state2], 1)\n",
    "    predictions = tf.contrib.layers.fully_connected(outputs, 1, activation_fn=tf.sigmoid)\n",
    "    cost = tf.losses.log_loss(labels_, predictions)\n",
    "    \n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope('questions', reuse=False):\n",
    "    correct_pred = tf.equal(tf.cast(tf.round(predictions), tf.int32), labels_)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_batches(x1, x2, y, batch_size=100):\n",
    "    \n",
    "    n_batches = len(x1)//batch_size\n",
    "    x1, x2, y = x1[:n_batches*batch_size], x2[:n_batches*batch_size], y[:n_batches*batch_size]\n",
    "    for ii in range(0, len(x1), batch_size):\n",
    "        yield x1[ii:ii+batch_size], x2[ii:ii+batch_size], y[ii:ii+batch_size]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 0/1 Iteration: 5 Train loss: 0.687\n",
      "Epoch: 0/1 Iteration: 10 Train loss: 0.663\n",
      "Epoch: 0/1 Iteration: 15 Train loss: 0.629\n",
      "Epoch: 0/1 Iteration: 20 Train loss: 0.631\n",
      "Epoch: 0/1 Iteration: 25 Train loss: 0.610\n",
      "Epoch: 0/1 Iteration: 30 Train loss: 0.637\n",
      "Epoch: 0/1 Iteration: 35 Train loss: 0.605\n",
      "Epoch: 0/1 Iteration: 40 Train loss: 0.616\n",
      "Epoch: 0/1 Iteration: 45 Train loss: 0.603\n",
      "Epoch: 0/1 Iteration: 50 Train loss: 0.600\n",
      "Val acc: 0.681\n",
      "Epoch: 0/1 Iteration: 55 Train loss: 0.567\n",
      "Epoch: 0/1 Iteration: 60 Train loss: 0.550\n",
      "Epoch: 0/1 Iteration: 65 Train loss: 0.623\n",
      "Epoch: 0/1 Iteration: 70 Train loss: 0.564\n",
      "Epoch: 0/1 Iteration: 75 Train loss: 0.588\n",
      "Epoch: 0/1 Iteration: 80 Train loss: 0.595\n",
      "Epoch: 0/1 Iteration: 85 Train loss: 0.560\n",
      "Epoch: 0/1 Iteration: 90 Train loss: 0.585\n",
      "Epoch: 0/1 Iteration: 95 Train loss: 0.536\n",
      "Epoch: 0/1 Iteration: 100 Train loss: 0.569\n",
      "Val acc: 0.716\n",
      "Epoch: 0/1 Iteration: 105 Train loss: 0.563\n",
      "Epoch: 0/1 Iteration: 110 Train loss: 0.535\n",
      "Epoch: 0/1 Iteration: 115 Train loss: 0.601\n",
      "Epoch: 0/1 Iteration: 120 Train loss: 0.531\n",
      "Epoch: 0/1 Iteration: 125 Train loss: 0.569\n"
     ]
    }
   ],
   "source": [
    "epochs = 1\n",
    "t_vars = tf.trainable_variables()\n",
    "saver = tf.train.Saver(var_list=t_vars)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    iteration = 1\n",
    "    for e in range(epochs):\n",
    "        state1 = sess.run(initial_state1)\n",
    "        state2 = sess.run(initial_state2)\n",
    "        \n",
    "        for ii, (x1, x2, y) in enumerate(get_batches(train_q1, train_q2, train_y, batch_size), 1):\n",
    "            feed = {inputs_1: x1,\n",
    "                    inputs_2: x2,\n",
    "                    labels_: y[:,None],\n",
    "                    keep_prob: 0.5,\n",
    "                    initial_state1: state1,\n",
    "                    initial_state2: state2}\n",
    "            loss, state, _ = sess.run([cost, final_state, optimizer], feed_dict=feed)\n",
    "            \n",
    "            if iteration%5==0:\n",
    "                print(\"Epoch: {}/{}\".format(e, epochs),\n",
    "                      \"Iteration: {}\".format(iteration),\n",
    "                      \"Train loss: {:.3f}\".format(loss))\n",
    "\n",
    "            if iteration%50==0:\n",
    "                val_acc = []\n",
    "                val_state1 = sess.run(cell1.zero_state(batch_size, tf.float32))\n",
    "                val_state2 = sess.run(cell2.zero_state(batch_size, tf.float32))\n",
    "                for x1, x2, y in get_batches(val_q1, val_q2, val_y, batch_size):\n",
    "                    feed = {inputs_1: x1,\n",
    "                            inputs_2: x2,\n",
    "                            labels_: y[:, None],\n",
    "                            keep_prob: 1,\n",
    "                            initial_state1: val_state1,\n",
    "                            initial_state1: val_state2}\n",
    "                    batch_acc, val_state = sess.run([accuracy, final_state], feed_dict=feed)\n",
    "                    val_acc.append(batch_acc)\n",
    "                print(\"Val acc: {:.3f}\".format(np.mean(val_acc)))\n",
    "            iteration +=1\n",
    "    saver.save(sess, \"checkpoints/duplicates.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_acc = []\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, tf.train.latest_checkpoint('checkpoints'))\n",
    "    test_state = sess.run(cell.zero_state(batch_size, tf.float32))\n",
    "    for ii, (x1, x2, y) in enumerate(get_batches(test_q1, test_q2, test_y, batch_size), 1):\n",
    "        feed = {inputs_1: x1,\n",
    "                inputs_2: x2,\n",
    "                labels_: y[:, None],\n",
    "                keep_prob: 1,\n",
    "                initial_state1: val_state1,\n",
    "                initial_state1: val_state2}\n",
    "        batch_acc, test_state = sess.run([cost, final_state], feed_dict=feed)\n",
    "        test_acc.append(batch_acc)\n",
    "    print(\"Test accuracy: {:.3f}\".format(np.mean(test_acc)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}