chatbot/37.capsule-lstm-seq2seq-greedy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from sklearn.utils import shuffle\n",
    "import re\n",
    "import time\n",
    "import collections\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words, atleast=1):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = open('movie_lines.txt', encoding='utf-8', errors='ignore').read().split('\\n')\n",
    "conv_lines = open('movie_conversations.txt', encoding='utf-8', errors='ignore').read().split('\\n')\n",
    "\n",
    "id2line = {}\n",
    "for line in lines:\n",
    "    _line = line.split(' +++$+++ ')\n",
    "    if len(_line) == 5:\n",
    "        id2line[_line[0]] = _line[4]\n",
    "        \n",
    "convs = [ ]\n",
    "for line in conv_lines[:-1]:\n",
    "    _line = line.split(' +++$+++ ')[-1][1:-1].replace(\"'\",\"\").replace(\" \",\"\")\n",
    "    convs.append(_line.split(','))\n",
    "    \n",
    "questions = []\n",
    "answers = []\n",
    "\n",
    "for conv in convs:\n",
    "    for i in range(len(conv)-1):\n",
    "        questions.append(id2line[conv[i]])\n",
    "        answers.append(id2line[conv[i+1]])\n",
    "        \n",
    "def clean_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"i'm\", \"i am\", text)\n",
    "    text = re.sub(r\"he's\", \"he is\", text)\n",
    "    text = re.sub(r\"she's\", \"she is\", text)\n",
    "    text = re.sub(r\"it's\", \"it is\", text)\n",
    "    text = re.sub(r\"that's\", \"that is\", text)\n",
    "    text = re.sub(r\"what's\", \"that is\", text)\n",
    "    text = re.sub(r\"where's\", \"where is\", text)\n",
    "    text = re.sub(r\"how's\", \"how is\", text)\n",
    "    text = re.sub(r\"\\'ll\", \" will\", text)\n",
    "    text = re.sub(r\"\\'ve\", \" have\", text)\n",
    "    text = re.sub(r\"\\'re\", \" are\", text)\n",
    "    text = re.sub(r\"\\'d\", \" would\", text)\n",
    "    text = re.sub(r\"\\'re\", \" are\", text)\n",
    "    text = re.sub(r\"won't\", \"will not\", text)\n",
    "    text = re.sub(r\"can't\", \"cannot\", text)\n",
    "    text = re.sub(r\"n't\", \" not\", text)\n",
    "    text = re.sub(r\"n'\", \"ng\", text)\n",
    "    text = re.sub(r\"'bout\", \"about\", text)\n",
    "    text = re.sub(r\"'til\", \"until\", text)\n",
    "    text = re.sub(r\"[-()\\\"#/@;:<>{}`+=~|.!?,]\", \"\", text)\n",
    "    return ' '.join([i.strip() for i in filter(None, text.split())])\n",
    "\n",
    "clean_questions = []\n",
    "for question in questions:\n",
    "    clean_questions.append(clean_text(question))\n",
    "    \n",
    "clean_answers = []    \n",
    "for answer in answers:\n",
    "    clean_answers.append(clean_text(answer))\n",
    "    \n",
    "min_line_length = 2\n",
    "max_line_length = 5\n",
    "short_questions_temp = []\n",
    "short_answers_temp = []\n",
    "\n",
    "i = 0\n",
    "for question in clean_questions:\n",
    "    if len(question.split()) >= min_line_length and len(question.split()) <= max_line_length:\n",
    "        short_questions_temp.append(question)\n",
    "        short_answers_temp.append(clean_answers[i])\n",
    "    i += 1\n",
    "\n",
    "short_questions = []\n",
    "short_answers = []\n",
    "\n",
    "i = 0\n",
    "for answer in short_answers_temp:\n",
    "    if len(answer.split()) >= min_line_length and len(answer.split()) <= max_line_length:\n",
    "        short_answers.append(answer)\n",
    "        short_questions.append(short_questions_temp[i])\n",
    "    i += 1\n",
    "\n",
    "question_test = short_questions[500:550]\n",
    "answer_test = short_answers[500:550]\n",
    "short_questions = short_questions[:500]\n",
    "short_answers = short_answers[:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_from = ' '.join(short_questions+question_test).split()\n",
    "vocabulary_size_from = len(list(set(concat_from)))\n",
    "data_from, count_from, dictionary_from, rev_dictionary_from = build_dataset(concat_from, vocabulary_size_from)\n",
    "print('vocab from size: %d'%(vocabulary_size_from))\n",
    "print('Most common words', count_from[4:10])\n",
    "print('Sample data', data_from[:10], [rev_dictionary_from[i] for i in data_from[:10]])\n",
    "print('filtered vocab size:',len(dictionary_from))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary_from)/vocabulary_size_from,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "concat_to = ' '.join(short_answers+answer_test).split()\n",
    "vocabulary_size_to = len(list(set(concat_to)))\n",
    "data_to, count_to, dictionary_to, rev_dictionary_to = build_dataset(concat_to, vocabulary_size_to)\n",
    "print('vocab from size: %d'%(vocabulary_size_to))\n",
    "print('Most common words', count_to[4:10])\n",
    "print('Sample data', data_to[:10], [rev_dictionary_to[i] for i in data_to[:10]])\n",
    "print('filtered vocab size:',len(dictionary_to))\n",
    "print(\"% of vocab used: {}%\".format(round(len(dictionary_to)/vocabulary_size_to,4)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary_from['GO']\n",
    "PAD = dictionary_from['PAD']\n",
    "EOS = dictionary_from['EOS']\n",
    "UNK = dictionary_from['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(short_answers)):\n",
    "    short_answers[i] += ' EOS'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def squash(X, epsilon = 1e-9):\n",
    "    vec_squared_norm = tf.reduce_sum(tf.square(X), -2, keep_dims=True)\n",
    "    scalar_factor = vec_squared_norm / (1 + vec_squared_norm) / tf.sqrt(vec_squared_norm + epsilon)\n",
    "    return scalar_factor * X\n",
    "\n",
    "def conv_layer(X, num_output, num_vector, kernel=None, stride=None):\n",
    "    capsules = tf.layers.conv1d(X, num_output * num_vector,\n",
    "                                kernel, stride, padding=\"VALID\", activation=tf.nn.relu)\n",
    "    capsules = tf.reshape(capsules, (tf.shape(X)[0], -1, num_vector, 1))\n",
    "    return squash(capsules)\n",
    "\n",
    "def routing(X, b_IJ, seq_len, dimension_out, routing_times = 2):\n",
    "    shape_X = tf.shape(X)[1]\n",
    "    w = tf.Variable(tf.truncated_normal([1, 1, seq_len, 4, dimension_out//2], stddev=1e-1))\n",
    "    X = tf.tile(X, [1, 1, seq_len, 1, dimension_out])\n",
    "    w = tf.tile(w, [tf.shape(X)[0], tf.shape(X)[1], 1, 1, routing_times])\n",
    "    print('X shape: %s, w shape: %s'%(str(X.shape), str(w.shape)))\n",
    "    u_hat = tf.matmul(w, X, transpose_a=True)\n",
    "    u_hat_stopped = tf.stop_gradient(u_hat)\n",
    "    print(u_hat,u_hat_stopped)\n",
    "    for i in range(routing_times):\n",
    "        c_IJ = tf.nn.softmax(b_IJ, dim=2)\n",
    "        print(c_IJ)\n",
    "        if i == routing_times - 1:\n",
    "            s_J = tf.multiply(c_IJ, u_hat)\n",
    "            s_J = tf.reduce_sum(s_J, axis=1, keep_dims=True)\n",
    "            v_J = squash(s_J)\n",
    "        else:\n",
    "            s_J = tf.multiply(c_IJ, u_hat_stopped)\n",
    "            s_J = tf.reduce_sum(s_J, axis=1, keep_dims=True)\n",
    "            v_J = squash(s_J)\n",
    "            v_J_tiled = tf.tile(v_J, [1, shape_X, 1, 1, 1])\n",
    "            u_produce_v = tf.matmul(u_hat_stopped, v_J_tiled, transpose_a=True)\n",
    "            b_IJ += u_produce_v\n",
    "    return v_J\n",
    "\n",
    "def fully_conn_layer(X, num_output, dimension_out):\n",
    "    batch_size = tf.shape(X)[1]\n",
    "    X_ = tf.reshape(X, shape=(tf.shape(X)[0], -1, 1, X.shape[-2].value, 1))\n",
    "    b_IJ = tf.fill([tf.shape(X)[0], tf.shape(X)[1], num_output, 1, 1], 0.0)\n",
    "    capsules = routing(X_, b_IJ, num_output, dimension_out, routing_times = 2)\n",
    "    capsules = tf.squeeze(capsules, axis=1)\n",
    "    return capsules\n",
    "\n",
    "class Chatbot:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size, seq_len, maxlen,\n",
    "                 from_dict_size, to_dict_size, learning_rate, batch_size,\n",
    "                kernels=[2, 4, 4],strides=[3,2,1],epsilon=1e-8):\n",
    "        \n",
    "        def cells(reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size_layer,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, maxlen])\n",
    "        self.Y = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        \n",
    "        encoder_embedding = tf.Variable(tf.random_uniform([from_dict_size, embedded_size], -1, 1))\n",
    "        decoder_embedding = tf.Variable(tf.random_uniform([to_dict_size, embedded_size], -1, 1))\n",
    "        encoder_embedded = tf.nn.embedding_lookup(encoder_embedding, self.X)\n",
    "        \n",
    "        results = []\n",
    "        for i in range(len(kernels)):\n",
    "            conv = tf.layers.conv1d(encoder_embedded, filters=32,\n",
    "                                    kernel_size=kernels[i], strides=strides[i],\n",
    "                                    padding='VALID')\n",
    "            caps1 = conv_layer(conv, 4, 4, kernels[i], strides[i])\n",
    "            caps2 = fully_conn_layer(caps1,seq_len,embedded_size)\n",
    "            print(caps2)\n",
    "            v_length = tf.sqrt(tf.reduce_sum(tf.square(caps2),axis=2, keep_dims=True) + epsilon)[:,:,0,:]\n",
    "            print('output shape: %s\\n'%(str(v_length.shape)))\n",
    "            results.append(v_length)\n",
    "        results = tf.concat(results,1)\n",
    "        self.X_seq_len = tf.fill([batch_size], seq_len * len(kernels))\n",
    "        \n",
    "        _, encoder_state = tf.nn.dynamic_rnn(\n",
    "            cell = tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(num_layers)]), \n",
    "            inputs = results,\n",
    "            sequence_length = self.X_seq_len,\n",
    "            dtype = tf.float32)\n",
    "        main = tf.strided_slice(self.Y, [0, 0], [batch_size, -1], [1, 1])\n",
    "        decoder_input = tf.concat([tf.fill([batch_size, 1], GO), main], 1)\n",
    "        dense = tf.layers.Dense(to_dict_size)\n",
    "        decoder_cells = tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(size_layer)])\n",
    "        \n",
    "        training_helper = tf.contrib.seq2seq.TrainingHelper(\n",
    "                inputs = tf.nn.embedding_lookup(decoder_embedding, decoder_input),\n",
    "                sequence_length = self.Y_seq_len,\n",
    "                time_major = False)\n",
    "        training_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell = decoder_cells,\n",
    "                helper = training_helper,\n",
    "                initial_state = encoder_state,\n",
    "                output_layer = dense)\n",
    "        training_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = training_decoder,\n",
    "                impute_finished = True,\n",
    "                maximum_iterations = tf.reduce_max(self.Y_seq_len))\n",
    "        self.training_logits = training_decoder_output.rnn_output\n",
    "        \n",
    "        predicting_helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(\n",
    "                embedding = decoder_embedding,\n",
    "                start_tokens = tf.tile(tf.constant([GO], dtype=tf.int32), [batch_size]),\n",
    "                end_token = EOS)\n",
    "        predicting_decoder = tf.contrib.seq2seq.BasicDecoder(\n",
    "                cell = decoder_cells,\n",
    "                helper = predicting_helper,\n",
    "                initial_state = encoder_state,\n",
    "                output_layer = dense)\n",
    "        predicting_decoder_output, _, _ = tf.contrib.seq2seq.dynamic_decode(\n",
    "                decoder = predicting_decoder,\n",
    "                impute_finished = True,\n",
    "                maximum_iterations = 3 * tf.reduce_max(self.X_seq_len))\n",
    "        self.predicting_ids = predicting_decoder_output.sample_id\n",
    "        \n",
    "        masks = tf.sequence_mask(self.Y_seq_len, tf.reduce_max(self.Y_seq_len), dtype=tf.float32)\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(logits = self.training_logits,\n",
    "                                                     targets = self.Y,\n",
    "                                                     weights = masks)\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        y_t = tf.argmax(self.training_logits,axis=2)\n",
    "        y_t = tf.cast(y_t, tf.int32)\n",
    "        self.prediction = tf.boolean_mask(y_t, masks)\n",
    "        mask_label = tf.boolean_mask(self.Y, masks)\n",
    "        correct_pred = tf.equal(self.prediction, mask_label)\n",
    "        correct_index = tf.cast(correct_pred, tf.float32)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "learning_rate = 1e-4\n",
    "batch_size = 16\n",
    "epoch = 20\n",
    "maxlen = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Chatbot(size_layer, num_layers, embedded_size, 5, maxlen, len(dictionary_from), \n",
    "                len(dictionary_to), learning_rate, batch_size)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def str_idx(corpus, dic):\n",
    "    X = []\n",
    "    for i in corpus:\n",
    "        ints = []\n",
    "        for k in i.split():\n",
    "            ints.append(dic.get(k,UNK))\n",
    "        X.append(ints)\n",
    "    return X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = str_idx(short_questions, dictionary_from)\n",
    "Y = str_idx(short_answers, dictionary_to)\n",
    "X_test = str_idx(question_test, dictionary_from)\n",
    "Y_test = str_idx(answer_test, dictionary_from)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens\n",
    "\n",
    "def pad_sentence_batch_static(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = maxlen\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(maxlen)\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(epoch):\n",
    "    total_loss, total_accuracy = 0, 0\n",
    "    X, Y = shuffle(X, Y)\n",
    "    for k in range(0, len(short_questions), batch_size):\n",
    "        index = min(k + batch_size, len(short_questions))\n",
    "        batch_x, _ = pad_sentence_batch_static(X[k: index], PAD)\n",
    "        batch_y, seq_y = pad_sentence_batch(Y[k: index], PAD)\n",
    "        predicted, loss, _, accuracy = sess.run([model.predicting_ids, model.cost, \n",
    "                                                 model.optimizer, model.accuracy], \n",
    "                                      feed_dict={model.X:batch_x,\n",
    "                                                model.Y:batch_y})\n",
    "        total_loss += loss\n",
    "        total_accuracy += accuracy\n",
    "    total_loss /= (len(short_questions) / batch_size)\n",
    "    total_accuracy /= (len(short_questions) / batch_size)\n",
    "    print('epoch: %d, avg loss: %f, avg accuracy: %f'%(i+1, total_loss, total_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(batch_x)):\n",
    "    print('row %d'%(i+1))\n",
    "    print('QUESTION:',' '.join([rev_dictionary_from[n] for n in batch_x[i] if n not in [0,1,2,3]]))\n",
    "    print('REAL ANSWER:',' '.join([rev_dictionary_to[n] for n in batch_y[i] if n not in[0,1,2,3]]))\n",
    "    print('PREDICTED ANSWER:',' '.join([rev_dictionary_to[n] for n in predicted[i] if n not in[0,1,2,3]]),'\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_x, _ = pad_sentence_batch_static(X_test[:batch_size], PAD)\n",
    "batch_y, seq_y = pad_sentence_batch(Y_test[:batch_size], PAD)\n",
    "predicted = sess.run(model.predicting_ids, feed_dict={model.X:batch_x})\n",
    "\n",
    "for i in range(len(batch_x)):\n",
    "    print('row %d'%(i+1))\n",
    "    print('QUESTION:',' '.join([rev_dictionary_from[n] for n in batch_x[i] if n not in [0,1,2,3]]))\n",
    "    print('REAL ANSWER:',' '.join([rev_dictionary_to[n] for n in batch_y[i] if n not in[0,1,2,3]]))\n",
    "    print('PREDICTED ANSWER:',' '.join([rev_dictionary_to[n] for n in predicted[i] if n not in[0,1,2,3]]),'\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```text\n",
    "epoch: 1, avg loss: 5.375520, avg accuracy: 0.239554\n",
    "epoch: 2, avg loss: 4.713192, avg accuracy: 0.262143\n",
    "epoch: 3, avg loss: 4.407076, avg accuracy: 0.277683\n",
    "epoch: 4, avg loss: 4.114979, avg accuracy: 0.294326\n",
    "epoch: 5, avg loss: 3.864636, avg accuracy: 0.302303\n",
    "epoch: 6, avg loss: 3.666699, avg accuracy: 0.312473\n",
    "epoch: 7, avg loss: 3.370519, avg accuracy: 0.343737\n",
    "epoch: 8, avg loss: 3.116906, avg accuracy: 0.372554\n",
    "epoch: 9, avg loss: 2.851404, avg accuracy: 0.403916\n",
    "epoch: 10, avg loss: 2.587815, avg accuracy: 0.466819\n",
    "epoch: 11, avg loss: 2.343917, avg accuracy: 0.517899\n",
    "epoch: 12, avg loss: 2.135444, avg accuracy: 0.561270\n",
    "epoch: 13, avg loss: 1.993201, avg accuracy: 0.600130\n",
    "epoch: 14, avg loss: 1.876830, avg accuracy: 0.617571\n",
    "epoch: 15, avg loss: 1.788884, avg accuracy: 0.632134\n",
    "epoch: 16, avg loss: 1.716430, avg accuracy: 0.641549\n",
    "epoch: 17, avg loss: 1.649575, avg accuracy: 0.655224\n",
    "epoch: 18, avg loss: 1.631061, avg accuracy: 0.656174\n",
    "epoch: 19, avg loss: 1.596259, avg accuracy: 0.656654\n",
    "epoch: 20, avg loss: 1.615984, avg accuracy: 0.655007\n",
    "row 1\n",
    "QUESTION: plenty twentysix minutes\n",
    "REAL ANSWER: we are not leaving\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 2\n",
    "QUESTION: no why\n",
    "REAL ANSWER: is david acting strangely\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 3\n",
    "QUESTION: cool pictures you a fan\n",
    "REAL ANSWER: yeah i guess\n",
    "PREDICTED ANSWER: i am so ashamed \n",
    "\n",
    "row 4\n",
    "QUESTION: ttthanks alice\n",
    "REAL ANSWER: earth to alice\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 1\n",
    "QUESTION: but david\n",
    "REAL ANSWER: is here that\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 2\n",
    "QUESTION: hopeless it is hopeless\n",
    "REAL ANSWER: tell ballet then back\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 3\n",
    "QUESTION: miss price\n",
    "REAL ANSWER: yes learning\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 4\n",
    "QUESTION: mr kessler wake up please\n",
    "REAL ANSWER: is here are\n",
    "PREDICTED ANSWER: i am so ashamed \n",
    "\n",
    "row 5\n",
    "QUESTION: there were witnesses\n",
    "REAL ANSWER: why she out\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 6\n",
    "QUESTION: what about it\n",
    "REAL ANSWER: not you are\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 7\n",
    "QUESTION: go on ask them\n",
    "REAL ANSWER: i just home\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 8\n",
    "QUESTION: beware the moon\n",
    "REAL ANSWER: seen hi is he\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 9\n",
    "QUESTION: did you hear that\n",
    "REAL ANSWER: is down what\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 10\n",
    "QUESTION: i heard that\n",
    "REAL ANSWER: it here not\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 11\n",
    "QUESTION: the hound of the baskervilles\n",
    "REAL ANSWER: heard\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 12\n",
    "QUESTION: it is moving\n",
    "REAL ANSWER: not you hear\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 13\n",
    "QUESTION: nice doggie good boy\n",
    "REAL ANSWER: bill stupid\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 14\n",
    "QUESTION: it sounds far away\n",
    "REAL ANSWER: that pecos baby seen hi\n",
    "PREDICTED ANSWER: i am fucked \n",
    "\n",
    "row 15\n",
    "QUESTION: debbie klein cried a lot\n",
    "REAL ANSWER: is will srai not\n",
    "PREDICTED ANSWER: i am so ashamed \n",
    "\n",
    "row 16\n",
    "QUESTION: what are you doing here\n",
    "REAL ANSWER: is know look i\n",
    "PREDICTED ANSWER: i am fucked \n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}