bidirectional_lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from string import punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TensorFlow Version: 1.0.0\n",
      "Default GPU Device: /gpu:0\n"
     ]
    }
   ],
   "source": [
    "from distutils.version import LooseVersion\n",
    "import warnings\n",
    "import tensorflow as tf\n",
    "\n",
    "# Check TensorFlow Version\n",
    "assert LooseVersion(tf.__version__) >= LooseVersion('1.0'), 'Please use TensorFlow version 1.0 or newer'\n",
    "print('TensorFlow Version: {}'.format(tf.__version__))\n",
    "\n",
    "# Check for a GPU\n",
    "if not tf.test.gpu_device_name():\n",
    "    warnings.warn('No GPU found. Please use a GPU to train your neural network.')\n",
    "else:\n",
    "    print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv').fillna(\"Empty\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "404290\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>How can I increase the speed of my internet co...</td>\n",
       "      <td>How can Internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
       "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>Which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "2   2     5     6  How can I increase the speed of my internet co...   \n",
       "3   3     7     8  Why am I mentally very lonely? How can I solve...   \n",
       "4   4     9    10  Which one dissolve in water quikly sugar, salt...   \n",
       "\n",
       "                                           question2  is_duplicate  \n",
       "0  What is the step by step guide to invest in sh...             0  \n",
       "1  What would happen if the Indian government sto...             0  \n",
       "2  How can Internet speed be increased by hacking...             0  \n",
       "3  Find the remainder when [math]23^{24}[/math] i...             0  \n",
       "4            Which fish would survive in salt water?             0  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(train))\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "question_words = train['question1'].apply(lambda x: x.split()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "What      137930\n",
       "How        93409\n",
       "Why        32808\n",
       "Is         22485\n",
       "Which      17468\n",
       "Can        11605\n",
       "I           7906\n",
       "Who         7547\n",
       "Do          6789\n",
       "Where       6641\n",
       "If          5961\n",
       "What's      5638\n",
       "Does        5090\n",
       "Are         4542\n",
       "When        3227\n",
       "Should      3129\n",
       "Will        2981\n",
       "In          1967\n",
       "My          1723\n",
       "Did         1285\n",
       "I'm         1165\n",
       "Has         1100\n",
       "Would       1053\n",
       "Have         887\n",
       "Was          783\n",
       "Could        719\n",
       "As           582\n",
       "Name: question1, dtype: int64"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "value_counts = question_words.value_counts()\n",
    "value_counts = value_counts[value_counts>500]\n",
    "value_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "404290\n",
      "404290\n",
      "Max sentence length 1169.00\n",
      "Min sentence length 1.00\n",
      "Average sentence length 59.82\n",
      "Median senctence length 51.00\n",
      "Standard deviation 31.96\n",
      "Mean plus 2 X STD 123.75\n",
      "Percent greater than max_words 0.72\n"
     ]
    }
   ],
   "source": [
    "sentence_lengths1 = train['question1'].apply(len)\n",
    "sentence_lengths2 = train['question2'].apply(len)\n",
    "print(len(sentence_lengths1))\n",
    "print(len(sentence_lengths2))\n",
    "sentence_lengths = sentence_lengths1.append(sentence_lengths2)\n",
    "print(\"Max sentence length %.2f\" % np.max(sentence_lengths))\n",
    "print(\"Min sentence length %.2f\" % np.min(sentence_lengths))\n",
    "print(\"Average sentence length %.2f\" % np.mean(sentence_lengths))\n",
    "print(\"Median senctence length %.2f\" % np.median(sentence_lengths))\n",
    "print(\"Standard deviation %.2f\" % np.std(sentence_lengths))\n",
    "print(\"Mean plus 2 X STD %.2f\" % (2 * np.std(sentence_lengths) + np.mean(sentence_lengths)))\n",
    "\n",
    "max_words = 40\n",
    "greater_than = sentence_lengths[sentence_lengths > max_words]\n",
    "print(\"Percent greater than max_words %.2f\" % (len(greater_than)/ len(sentence_lengths)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Train and Test Splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# train_data, test_data = train_test_split(train, test_size=0.25)\n",
    "train_questions1 = train['question1'].tolist()\n",
    "train_questions2 = train['question2'].tolist()\n",
    "train_data_labels = train['is_duplicate'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def build_int_dict(questions, max_vocab):\n",
    "    all_questions = ' '.join([c.lower().translate(str.maketrans(\"\", \"\", punctuation)) for c in questions])\n",
    "    words = all_questions.split()\n",
    "    from collections import Counter\n",
    "    vocab = Counter(words).most_common(max_vocab)\n",
    "    vocab_to_int = {word[0]: ii for ii, word in enumerate(vocab, 1)}\n",
    "    vocab_to_int[''] = len(vocab_to_int.keys()) + 1\n",
    "    vocab_to_int['<UNK>'] = len(vocab_to_int.keys()) + 1\n",
    "    \n",
    "    return vocab_to_int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "80002\n"
     ]
    }
   ],
   "source": [
    "max_vocab = 80000\n",
    "train_questions = train_questions1[:]\n",
    "train_questions.extend(train_questions2)\n",
    "\n",
    "vocab_to_int_dict = build_int_dict(train_questions, max_vocab)\n",
    "print(len(vocab_to_int_dict.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['What is the step by step guide to invest in share market in india?', 'What is the story of Kohinoor (Koh-i-Noor) Diamond?', 'How can I increase the speed of my internet connection while using a VPN?', 'Why am I mentally very lonely? How can I solve it?', 'Which one dissolve in water quikly sugar, salt, methane and carbon di oxide?', 'Astrology: I am a Capricorn Sun Cap moon and cap rising...what does that say about me?', 'Should I buy tiago?', 'How can I be a good geologist?', 'When do you use シ instead of し?', 'Motorola (company): Can I hack my Charter Motorolla DCX3400?']\n",
      "80002\n"
     ]
    }
   ],
   "source": [
    "print(train_questions1[0:10])\n",
    "print(len(vocab_to_int_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def convert_to_ints(questions, seq_len, vocab_to_int):\n",
    "    all_questions_ints = []\n",
    "    for i, each in enumerate(questions):\n",
    "        question_ints = []\n",
    "        if i % 100000 == 0:\n",
    "            print(i)\n",
    "        for word in each.split():\n",
    "            if word.lower().translate(str.maketrans(\"\",\"\",punctuation)) in vocab_to_int.keys():\n",
    "                question_ints.append(vocab_to_int[word.lower().translate(str.maketrans(\"\",\"\",punctuation))])\n",
    "            else:\n",
    "                question_ints.append(vocab_to_int['<UNK>'])\n",
    "        all_questions_ints.append(question_ints)\n",
    "    \n",
    "    print(\"Finished converting to ints. Now padding...\")\n",
    "    features = np.zeros((len(all_questions_ints), seq_len), dtype=int)\n",
    "    for i, row in enumerate(all_questions_ints):\n",
    "        features[i, -len(row):] = np.array(row)[:seq_len]\n",
    "        \n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100000\n",
      "200000\n",
      "300000\n",
      "400000\n",
      "Finished converting to ints. Now padding...\n",
      "0\n",
      "100000\n",
      "200000\n",
      "300000\n",
      "400000\n",
      "Finished converting to ints. Now padding...\n"
     ]
    }
   ],
   "source": [
    "question_ints1 = convert_to_ints(train_questions1, max_words, vocab_to_int_dict)\n",
    "question_ints2 = convert_to_ints(train_questions2, max_words, vocab_to_int_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# !mkdir checkpoints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\t\t\tFeature Shapes:\n",
      "Train set: \t\t(323432, 40) \n",
      "Validation set: \t(40429, 40) \n",
      "Test set: \t\t(40429, 40)\n"
     ]
    }
   ],
   "source": [
    "split_frac = 0.8\n",
    "split_idx = int(len(question_ints1)*split_frac)\n",
    "train_q1, val_q1 = question_ints1[:split_idx], question_ints1[split_idx:]\n",
    "train_q2, val_q2 = question_ints2[:split_idx], question_ints2[split_idx:]\n",
    "train_y, val_y = train_data_labels[:split_idx], train_data_labels[split_idx:]\n",
    "train_y = np.array(train_y)\n",
    "val_y = np.array(val_y)\n",
    "\n",
    "test_idx = int(len(val_q1)*0.5)\n",
    "val_q1, test_q1 = val_q1[:test_idx], val_q1[test_idx:]\n",
    "val_q2, test_q2 = val_q2[:test_idx], val_q2[test_idx:]\n",
    "val_y, test_y = val_y[:test_idx], val_y[test_idx:]\n",
    "\n",
    "print(\"\\t\\t\\tFeature Shapes:\")\n",
    "print(\"Train set: \\t\\t{}\".format(train_q1.shape), \n",
    "      \"\\nValidation set: \\t{}\".format(val_q1.shape),\n",
    "      \"\\nTest set: \\t\\t{}\".format(test_q1.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "lstm_size = 250\n",
    "lstm_layers = 1\n",
    "batch_size = 256\n",
    "learning_rate = 0.001\n",
    "# Size of the embedding vectors (number of units in the embedding layer)\n",
    "embed_size = 300 \n",
    "seq_length = max_words\n",
    "num_dense = 150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# n_words1 = len(question1_vocab_to_int)\n",
    "# n_words2 = len(question2_vocab_to_int)\n",
    "\n",
    "# # Create the graph object\n",
    "# graph = tf.Graph()\n",
    "# # Add nodes to the graph\n",
    "# with graph.as_default():\n",
    "#     inputs_1 = tf.placeholder(tf.int32, [None, None], name='inputs')\n",
    "#     inputs_2 = tf.placeholder(tf.int32, [None, None], name='inputs')\n",
    "#     labels_ = tf.placeholder(tf.int32, [None, None], name='labels')\n",
    "#     keep_prob = tf.placeholder(tf.float32, name='keep_prob')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "n_words = len(vocab_to_int_dict) + 1\n",
    "\n",
    "tf.reset_default_graph()\n",
    "\n",
    "# Create the graph object\n",
    "# graph = tf.Graph()\n",
    "# Add nodes to the graph\n",
    "with tf.variable_scope('questions', reuse=None):\n",
    "    inputs_1 = tf.placeholder(tf.int32, [None, None], name='inputs')\n",
    "    inputs_2 = tf.placeholder(tf.int32, [None, None], name='inputs')\n",
    "    labels_ = tf.placeholder(tf.int32, [None, None], name='labels')\n",
    "    keep_prob = tf.placeholder(tf.float32, name='keep_prob')\n",
    "    seq_len = tf.placeholder(tf.int32, [None])\n",
    "    \n",
    "\n",
    "    embedding1 = tf.Variable(tf.random_uniform((n_words, embed_size), -1, 1))\n",
    "    embed1 = tf.nn.embedding_lookup(embedding1, inputs_1)\n",
    "    \n",
    "    # Your basic LSTM cell\n",
    "    lstm_fwd1 = tf.contrib.rnn.BasicLSTMCell(lstm_size)\n",
    "    lstm_bwd1 = tf.contrib.rnn.BasicLSTMCell(lstm_size)\n",
    "    \n",
    "    # Add dropout to the cell\n",
    "    drop_fwd1 = tf.contrib.rnn.DropoutWrapper(lstm_fwd1, output_keep_prob=keep_prob)\n",
    "    drop_bwd1 = tf.contrib.rnn.DropoutWrapper(lstm_bwd1, output_keep_prob=keep_prob)\n",
    "    \n",
    "    # Stack up multiple LSTM layers, for deep learning\n",
    "    cell_fwd1 = tf.contrib.rnn.MultiRNNCell([drop_fwd1] * lstm_layers)\n",
    "    cell_bwd1 = tf.contrib.rnn.MultiRNNCell([drop_bwd1] * lstm_layers)\n",
    "    \n",
    "    # Getting an initial state of all zeros\n",
    "    initial_state_fwd1 = cell_fwd1.zero_state(batch_size, tf.float32)\n",
    "    initial_state_bwd1 = cell_bwd1.zero_state(batch_size, tf.float32)\n",
    "\n",
    "    outputs1, final_state1 = tf.nn.bidirectional_dynamic_rnn(cell_fwd1,cell_bwd1, embed1,\n",
    "                                             initial_state_fw=initial_state_fwd1,\n",
    "                                             initial_state_bw=initial_state_bwd1, sequence_length=seq_len)\n",
    "    \n",
    "with tf.variable_scope('questions', reuse=True):\n",
    "    embedding2 = tf.Variable(tf.random_uniform((n_words, embed_size), -1, 1))\n",
    "    embed2 = tf.nn.embedding_lookup(embedding2, inputs_2)\n",
    "    \n",
    "    # Your basic LSTM cell\n",
    "    lstm_fwd2 = tf.contrib.rnn.BasicLSTMCell(lstm_size)\n",
    "    lstm_bwd2 = tf.contrib.rnn.BasicLSTMCell(lstm_size)\n",
    "    \n",
    "    # Add dropout to the cell\n",
    "    drop_fwd2 = tf.contrib.rnn.DropoutWrapper(lstm_fwd2, output_keep_prob=keep_prob)\n",
    "    drop_bwd2 = tf.contrib.rnn.DropoutWrapper(lstm_bwd2, output_keep_prob=keep_prob)\n",
    "    \n",
    "    # Stack up multiple LSTM layers, for deep learning\n",
    "    cell_fwd2 = tf.contrib.rnn.MultiRNNCell([drop_fwd2] * lstm_layers)\n",
    "    cell_bwd2 = tf.contrib.rnn.MultiRNNCell([drop_bwd2] * lstm_layers)\n",
    "    \n",
    "    # Getting an initial state of all zeros\n",
    "    initial_state_fwd2 = cell_fwd2.zero_state(batch_size, tf.float32)\n",
    "    initial_state_bwd2 = cell_bwd2.zero_state(batch_size, tf.float32)\n",
    "    \n",
    "    outputs2, final_state2 = tf.nn.bidirectional_dynamic_rnn(cell_fwd2, cell_bwd2, embed2,\n",
    "                                             initial_state_fw=initial_state_fwd2,\n",
    "                                             initial_state_bw=initial_state_bwd2,sequence_length=seq_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope('questions', reuse=False):\n",
    "    outputs1 = tf.concat(outputs1, 2)\n",
    "    outputs2 = tf.concat(outputs2, 2)\n",
    "    outputs = tf.concat([outputs1[:,-1], outputs2[:,-1]], 1)\n",
    "    final_state = tf.concat([final_state1, final_state2], 1)\n",
    "    \n",
    "    dense_layer = tf.contrib.layers.fully_connected(outputs, num_dense, activation_fn=tf.nn.relu)\n",
    "    drop_dense = tf.layers.dropout(dense_layer, rate=keep_prob)\n",
    "    predictions = tf.contrib.layers.fully_connected(drop_dense, 1, activation_fn=tf.sigmoid)\n",
    "    cost = tf.losses.log_loss(labels_, predictions)\n",
    "    \n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate).minimize(cost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "with tf.variable_scope('questions', reuse=False):\n",
    "    correct_pred = tf.equal(tf.cast(tf.round(predictions), tf.int32), labels_)\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def get_batches(x1, x2, y, batch_size=100):\n",
    "    if y != None:\n",
    "        n_batches = len(x1)//batch_size\n",
    "        x1, x2, y = x1[:n_batches*batch_size], x2[:n_batches*batch_size], y[:n_batches*batch_size]\n",
    "        for ii in range(0, len(x1), batch_size):\n",
    "            yield x1[ii:ii+batch_size], x2[ii:ii+batch_size], y[ii:ii+batch_size]\n",
    "    else:\n",
    "        n_batches = len(x1)//batch_size\n",
    "        x1, x2, _ = x1[:n_batches*batch_size], x2[:n_batches*batch_size], None\n",
    "        for ii in range(0, len(x1), batch_size):\n",
    "            yield x1[ii:ii+batch_size], x2[ii:ii+batch_size], None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/carnd/anaconda3/envs/dl/lib/python3.5/site-packages/ipykernel/__main__.py:2: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1/3 Iteration: 25 Train loss: 0.610\n",
      "Epoch: 1/3 Iteration: 50 Train loss: 0.546\n",
      "Epoch: 1/3 Iteration: 75 Train loss: 0.550\n",
      "Epoch: 1/3 Iteration: 100 Train loss: 0.584\n",
      "Epoch: 1/3 Iteration: 125 Train loss: 0.541\n",
      "Epoch: 1/3 Iteration: 150 Train loss: 0.545\n",
      "Epoch: 1/3 Iteration: 175 Train loss: 0.540\n",
      "Epoch: 1/3 Iteration: 200 Train loss: 0.541\n",
      "Epoch: 1/3 Iteration: 225 Train loss: 0.574\n",
      "Epoch: 1/3 Iteration: 250 Train loss: 0.592\n",
      "Epoch: 1/3 Iteration: 275 Train loss: 0.563\n",
      "Epoch: 1/3 Iteration: 300 Train loss: 0.524\n",
      "Epoch: 1/3 Iteration: 325 Train loss: 0.499\n",
      "Epoch: 1/3 Iteration: 350 Train loss: 0.567\n",
      "Epoch: 1/3 Iteration: 375 Train loss: 0.478\n",
      "Epoch: 1/3 Iteration: 400 Train loss: 0.504\n",
      "Epoch: 1/3 Iteration: 425 Train loss: 0.516\n",
      "Epoch: 1/3 Iteration: 450 Train loss: 0.508\n",
      "Epoch: 1/3 Iteration: 475 Train loss: 0.581\n",
      "Epoch: 1/3 Iteration: 500 Train loss: 0.513\n",
      "Val acc: 0.504\n",
      "Epoch: 1/3 Iteration: 525 Train loss: 0.484\n",
      "Epoch: 1/3 Iteration: 550 Train loss: 0.474\n",
      "Epoch: 1/3 Iteration: 575 Train loss: 0.494\n",
      "Epoch: 1/3 Iteration: 600 Train loss: 0.450\n",
      "Epoch: 1/3 Iteration: 625 Train loss: 0.467\n",
      "Epoch: 1/3 Iteration: 650 Train loss: 0.454\n",
      "Epoch: 1/3 Iteration: 675 Train loss: 0.488\n",
      "Epoch: 1/3 Iteration: 700 Train loss: 0.466\n",
      "Epoch: 1/3 Iteration: 725 Train loss: 0.416\n",
      "Epoch: 1/3 Iteration: 750 Train loss: 0.489\n",
      "Epoch: 1/3 Iteration: 775 Train loss: 0.494\n",
      "Epoch: 1/3 Iteration: 800 Train loss: 0.545\n",
      "Epoch: 1/3 Iteration: 825 Train loss: 0.553\n",
      "Epoch: 1/3 Iteration: 850 Train loss: 0.497\n",
      "Epoch: 1/3 Iteration: 875 Train loss: 0.483\n",
      "Epoch: 1/3 Iteration: 900 Train loss: 0.511\n",
      "Epoch: 1/3 Iteration: 925 Train loss: 0.449\n",
      "Epoch: 1/3 Iteration: 950 Train loss: 0.461\n",
      "Epoch: 1/3 Iteration: 975 Train loss: 0.531\n",
      "Epoch: 1/3 Iteration: 1000 Train loss: 0.445\n",
      "Val acc: 0.481\n",
      "Epoch: 1/3 Iteration: 1025 Train loss: 0.468\n",
      "Epoch: 1/3 Iteration: 1050 Train loss: 0.503\n",
      "Epoch: 1/3 Iteration: 1075 Train loss: 0.464\n",
      "Epoch: 1/3 Iteration: 1100 Train loss: 0.490\n",
      "Epoch: 1/3 Iteration: 1125 Train loss: 0.453\n",
      "Epoch: 1/3 Iteration: 1150 Train loss: 0.480\n",
      "Epoch: 1/3 Iteration: 1175 Train loss: 0.472\n",
      "Epoch: 1/3 Iteration: 1200 Train loss: 0.427\n",
      "Epoch: 1/3 Iteration: 1225 Train loss: 0.442\n",
      "Epoch: 1/3 Iteration: 1250 Train loss: 0.508\n",
      "Epoch: 2/3 Iteration: 1275 Train loss: 0.499\n",
      "Epoch: 2/3 Iteration: 1300 Train loss: 0.437\n",
      "Epoch: 2/3 Iteration: 1325 Train loss: 0.430\n",
      "Epoch: 2/3 Iteration: 1350 Train loss: 0.442\n",
      "Epoch: 2/3 Iteration: 1375 Train loss: 0.419\n",
      "Epoch: 2/3 Iteration: 1400 Train loss: 0.417\n",
      "Epoch: 2/3 Iteration: 1425 Train loss: 0.486\n",
      "Epoch: 2/3 Iteration: 1450 Train loss: 0.390\n",
      "Epoch: 2/3 Iteration: 1475 Train loss: 0.467\n",
      "Epoch: 2/3 Iteration: 1500 Train loss: 0.475\n",
      "Val acc: 0.461\n",
      "Epoch: 2/3 Iteration: 1525 Train loss: 0.471\n",
      "Epoch: 2/3 Iteration: 1550 Train loss: 0.481\n",
      "Epoch: 2/3 Iteration: 1575 Train loss: 0.462\n",
      "Epoch: 2/3 Iteration: 1600 Train loss: 0.447\n",
      "Epoch: 2/3 Iteration: 1625 Train loss: 0.411\n",
      "Epoch: 2/3 Iteration: 1650 Train loss: 0.465\n",
      "Epoch: 2/3 Iteration: 1675 Train loss: 0.400\n",
      "Epoch: 2/3 Iteration: 1700 Train loss: 0.368\n",
      "Epoch: 2/3 Iteration: 1725 Train loss: 0.408\n",
      "Epoch: 2/3 Iteration: 1750 Train loss: 0.428\n",
      "Epoch: 2/3 Iteration: 1775 Train loss: 0.415\n",
      "Epoch: 2/3 Iteration: 1800 Train loss: 0.432\n",
      "Epoch: 2/3 Iteration: 1825 Train loss: 0.374\n",
      "Epoch: 2/3 Iteration: 1850 Train loss: 0.385\n",
      "Epoch: 2/3 Iteration: 1875 Train loss: 0.452\n",
      "Epoch: 2/3 Iteration: 1900 Train loss: 0.437\n",
      "Epoch: 2/3 Iteration: 1925 Train loss: 0.375\n",
      "Epoch: 2/3 Iteration: 1950 Train loss: 0.366\n",
      "Epoch: 2/3 Iteration: 1975 Train loss: 0.411\n",
      "Epoch: 2/3 Iteration: 2000 Train loss: 0.448\n",
      "Val acc: 0.457\n",
      "Epoch: 2/3 Iteration: 2025 Train loss: 0.403\n",
      "Epoch: 2/3 Iteration: 2050 Train loss: 0.458\n",
      "Epoch: 2/3 Iteration: 2075 Train loss: 0.376\n",
      "Epoch: 2/3 Iteration: 2100 Train loss: 0.394\n",
      "Epoch: 2/3 Iteration: 2125 Train loss: 0.426\n",
      "Epoch: 2/3 Iteration: 2150 Train loss: 0.426\n",
      "Epoch: 2/3 Iteration: 2175 Train loss: 0.444\n",
      "Epoch: 2/3 Iteration: 2200 Train loss: 0.418\n",
      "Epoch: 2/3 Iteration: 2225 Train loss: 0.413\n",
      "Epoch: 2/3 Iteration: 2250 Train loss: 0.345\n",
      "Epoch: 2/3 Iteration: 2275 Train loss: 0.389\n",
      "Epoch: 2/3 Iteration: 2300 Train loss: 0.459\n",
      "Epoch: 2/3 Iteration: 2325 Train loss: 0.425\n",
      "Epoch: 2/3 Iteration: 2350 Train loss: 0.445\n",
      "Epoch: 2/3 Iteration: 2375 Train loss: 0.353\n",
      "Epoch: 2/3 Iteration: 2400 Train loss: 0.434\n",
      "Epoch: 2/3 Iteration: 2425 Train loss: 0.407\n",
      "Epoch: 2/3 Iteration: 2450 Train loss: 0.401\n",
      "Epoch: 2/3 Iteration: 2475 Train loss: 0.399\n",
      "Epoch: 2/3 Iteration: 2500 Train loss: 0.386\n",
      "Val acc: 0.444\n",
      "Epoch: 2/3 Iteration: 2525 Train loss: 0.385\n",
      "Epoch: 3/3 Iteration: 2550 Train loss: 0.327\n",
      "Epoch: 3/3 Iteration: 2575 Train loss: 0.385\n",
      "Epoch: 3/3 Iteration: 2600 Train loss: 0.396\n",
      "Epoch: 3/3 Iteration: 2625 Train loss: 0.359\n",
      "Epoch: 3/3 Iteration: 2650 Train loss: 0.427\n",
      "Epoch: 3/3 Iteration: 2675 Train loss: 0.397\n",
      "Epoch: 3/3 Iteration: 2700 Train loss: 0.338\n",
      "Epoch: 3/3 Iteration: 2725 Train loss: 0.350\n",
      "Epoch: 3/3 Iteration: 2750 Train loss: 0.342\n",
      "Epoch: 3/3 Iteration: 2775 Train loss: 0.354\n",
      "Epoch: 3/3 Iteration: 2800 Train loss: 0.464\n",
      "Epoch: 3/3 Iteration: 2825 Train loss: 0.336\n",
      "Epoch: 3/3 Iteration: 2850 Train loss: 0.406\n",
      "Epoch: 3/3 Iteration: 2875 Train loss: 0.391\n",
      "Epoch: 3/3 Iteration: 2900 Train loss: 0.379\n",
      "Epoch: 3/3 Iteration: 2925 Train loss: 0.363\n",
      "Epoch: 3/3 Iteration: 2950 Train loss: 0.331\n",
      "Epoch: 3/3 Iteration: 2975 Train loss: 0.331\n",
      "Epoch: 3/3 Iteration: 3000 Train loss: 0.319\n",
      "Val acc: 0.466\n",
      "Epoch: 3/3 Iteration: 3025 Train loss: 0.261\n",
      "Epoch: 3/3 Iteration: 3050 Train loss: 0.350\n",
      "Epoch: 3/3 Iteration: 3075 Train loss: 0.350\n",
      "Epoch: 3/3 Iteration: 3100 Train loss: 0.323\n",
      "Epoch: 3/3 Iteration: 3125 Train loss: 0.272\n",
      "Epoch: 3/3 Iteration: 3150 Train loss: 0.290\n",
      "Epoch: 3/3 Iteration: 3175 Train loss: 0.338\n",
      "Epoch: 3/3 Iteration: 3200 Train loss: 0.387\n",
      "Epoch: 3/3 Iteration: 3225 Train loss: 0.318\n",
      "Epoch: 3/3 Iteration: 3250 Train loss: 0.308\n",
      "Epoch: 3/3 Iteration: 3275 Train loss: 0.287\n",
      "Epoch: 3/3 Iteration: 3300 Train loss: 0.380\n",
      "Epoch: 3/3 Iteration: 3325 Train loss: 0.342\n",
      "Epoch: 3/3 Iteration: 3350 Train loss: 0.361\n",
      "Epoch: 3/3 Iteration: 3375 Train loss: 0.285\n",
      "Epoch: 3/3 Iteration: 3400 Train loss: 0.353\n",
      "Epoch: 3/3 Iteration: 3425 Train loss: 0.351\n",
      "Epoch: 3/3 Iteration: 3450 Train loss: 0.283\n",
      "Epoch: 3/3 Iteration: 3475 Train loss: 0.392\n",
      "Epoch: 3/3 Iteration: 3500 Train loss: 0.279\n",
      "Val acc: 0.463\n",
      "Epoch: 3/3 Iteration: 3525 Train loss: 0.288\n",
      "Epoch: 3/3 Iteration: 3550 Train loss: 0.345\n",
      "Epoch: 3/3 Iteration: 3575 Train loss: 0.297\n",
      "Epoch: 3/3 Iteration: 3600 Train loss: 0.258\n",
      "Epoch: 3/3 Iteration: 3625 Train loss: 0.347\n",
      "Epoch: 3/3 Iteration: 3650 Train loss: 0.327\n",
      "Epoch: 3/3 Iteration: 3675 Train loss: 0.316\n",
      "Epoch: 3/3 Iteration: 3700 Train loss: 0.283\n",
      "Epoch: 3/3 Iteration: 3725 Train loss: 0.373\n",
      "Epoch: 3/3 Iteration: 3750 Train loss: 0.323\n",
      "Epoch: 3/3 Iteration: 3775 Train loss: 0.239\n"
     ]
    }
   ],
   "source": [
    "epochs = 3\n",
    "t_vars = tf.trainable_variables()\n",
    "saver = tf.train.Saver(var_list=t_vars)\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    iteration = 1\n",
    "    for e in range(epochs):\n",
    "        state_fwd1 = sess.run(initial_state_fwd1)\n",
    "        state_bwd1 = sess.run(initial_state_bwd1)\n",
    "        state_fwd2 = sess.run(initial_state_fwd2)\n",
    "        state_bwd2 = sess.run(initial_state_bwd2)\n",
    "        \n",
    "        for ii, (x1, x2, y) in enumerate(get_batches(train_q1, train_q2, train_y, batch_size), 1):\n",
    "            train_seq_len = np.ones(batch_size) * seq_length\n",
    "            feed = {inputs_1: x1,\n",
    "                    inputs_2: x2,\n",
    "                    labels_: y[:,None],\n",
    "                    keep_prob: 0.6,\n",
    "                    initial_state_fwd1: state_fwd1,\n",
    "                    initial_state_bwd1: state_bwd1,\n",
    "                    initial_state_fwd2: state_fwd2,\n",
    "                    initial_state_bwd2: state_bwd2,\n",
    "                    seq_len:train_seq_len}\n",
    "            loss, state1, state2,_ = sess.run([cost, final_state1, final_state2, optimizer], feed_dict=feed)\n",
    "            \n",
    "            if iteration%25==0:\n",
    "                print(\"Epoch: {}/{}\".format(e + 1, epochs),\n",
    "                      \"Iteration: {}\".format(iteration),\n",
    "                      \"Train loss: {:.3f}\".format(loss))\n",
    "\n",
    "            if iteration%500==0:\n",
    "                val_acc = []\n",
    "                val_state_fwd1 = sess.run(cell_fwd1.zero_state(batch_size, tf.float32))\n",
    "                val_state_bwd1 = sess.run(cell_bwd1.zero_state(batch_size, tf.float32))\n",
    "                val_state_fwd2 = sess.run(cell_fwd2.zero_state(batch_size, tf.float32))\n",
    "                val_state_bwd2 = sess.run(cell_bwd2.zero_state(batch_size, tf.float32))\n",
    "                for x1, x2, y in get_batches(val_q1, val_q2, val_y, batch_size):\n",
    "                    feed = {inputs_1: x1,\n",
    "                            inputs_2: x2,\n",
    "                            labels_: y[:, None],\n",
    "                            keep_prob: 1,\n",
    "                            initial_state_fwd1: val_state_fwd1,\n",
    "                            initial_state_fwd1: val_state_bwd1,\n",
    "                            initial_state_fwd2: val_state_fwd2,\n",
    "                            initial_state_fwd2: val_state_bwd2,\n",
    "                            seq_len:train_seq_len}\n",
    "                    batch_acc, val_state1, val_state2 = sess.run([cost, final_state1, final_state2], feed_dict=feed)\n",
    "                    val_acc.append(batch_acc)\n",
    "                print(\"Val acc: {:.3f}\".format(np.mean(val_acc)))\n",
    "            iteration +=1\n",
    "    saver.save(sess, \"checkpoints/duplicates2.ckpt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/carnd/anaconda3/envs/dl/lib/python3.5/site-packages/ipykernel/__main__.py:2: FutureWarning: comparison to `None` will result in an elementwise object comparison in the future.\n",
      "  from ipykernel import kernelapp as app\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy: 0.456\n"
     ]
    }
   ],
   "source": [
    "test_acc = []\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, tf.train.latest_checkpoint('checkpoints'))\n",
    "    test_state_fwd1 = sess.run(initial_state_fwd1)\n",
    "    test_state_bwd1 = sess.run(initial_state_bwd1)\n",
    "    test_state_fwd2 = sess.run(initial_state_fwd2)\n",
    "    test_state_bwd2 = sess.run(initial_state_bwd2)\n",
    "    for ii, (x1, x2, y) in enumerate(get_batches(test_q1, test_q2, test_y, batch_size), 1):\n",
    "        feed = {inputs_1: x1,\n",
    "                inputs_2: x2,\n",
    "                labels_: y[:, None],\n",
    "                keep_prob: 1,\n",
    "                initial_state_fwd1: test_state_fwd1,\n",
    "                initial_state_bwd1: test_state_bwd1,\n",
    "                initial_state_fwd2: test_state_fwd2,\n",
    "                initial_state_bwd2: test_state_bwd2,\n",
    "                seq_len:train_seq_len}\n",
    "        batch_acc, test_state1, test_state2 = sess.run([cost, final_state1, final_state2], feed_dict=feed)\n",
    "        test_acc.append(batch_acc)\n",
    "    print(\"Test accuracy: {:.3f}\".format(np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2345796\n"
     ]
    }
   ],
   "source": [
    "test_set = pd.read_csv('test.csv').fillna(\"Empty\")\n",
    "print(len(test_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100000\n",
      "200000\n",
      "300000\n",
      "400000\n",
      "500000\n",
      "600000\n",
      "700000\n",
      "800000\n",
      "900000\n",
      "1000000\n",
      "1100000\n",
      "1200000\n",
      "1300000\n",
      "1400000\n",
      "1500000\n",
      "1600000\n",
      "1700000\n",
      "1800000\n",
      "1900000\n",
      "2000000\n",
      "2100000\n",
      "2200000\n",
      "2300000\n",
      "Finished converting to ints. Now padding...\n",
      "0\n",
      "100000\n",
      "200000\n",
      "300000\n",
      "400000\n",
      "500000\n",
      "600000\n",
      "700000\n",
      "800000\n",
      "900000\n",
      "1000000\n",
      "1100000\n",
      "1200000\n",
      "1300000\n",
      "1400000\n",
      "1500000\n",
      "1600000\n",
      "1700000\n",
      "1800000\n",
      "1900000\n",
      "2000000\n",
      "2100000\n",
      "2200000\n",
      "2300000\n",
      "Finished converting to ints. Now padding...\n"
     ]
    }
   ],
   "source": [
    "test_questions1 = test_set['question1'].tolist()\n",
    "test_questions2 = test_set['question2'].tolist()\n",
    "test_question_ints1 = convert_to_ints(test_questions1, max_words, vocab_to_int_dict)\n",
    "test_question_ints2 = convert_to_ints(test_questions2, max_words, vocab_to_int_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Make predictions\n",
    "\n",
    "predictions = []\n",
    "with tf.Session() as sess:\n",
    "    saver.restore(sess, tf.train.latest_checkpoint('checkpoints'))\n",
    "    test_state1 = sess.run(initial_state1)\n",
    "    test_state2 = sess.run(initial_state2)\n",
    "    for ii, (x1, x2, _) in enumerate(get_batches(test_question_ints1, test_question_ints2, None, batch_size), 1):\n",
    "        feed = {inputs_1: x1,\n",
    "                inputs_2: x2,\n",
    "                keep_prob: 1}\n",
    "        prediction = sess.run([predictions], feed_dict=feed)\n",
    "        predictions.append(prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "print(len(test_set))\n",
    "print(len(predictions))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "widgets": {
   "state": {},
   "version": "1.1.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}