Jokes-Word-Stacked-LSTM-Keras.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import importlib\n",
    "from library import data_preprocess as dp\n",
    "importlib.reload(dp)\n",
    "import random\n",
    "from time import time\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import keras\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential, Input\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.layers import LSTM, Bidirectional\n",
    "from keras.layers import Embedding, TimeDistributed, Flatten, Merge, Concatenate\n",
    "from keras import regularizers\n",
    "from keras.metrics import sparse_categorical_accuracy, sparse_categorical_crossentropy\n",
    "from keras.models import load_model\n",
    "from keras.optimizers import Adam, RMSprop\n",
    "from keras.models import Model\n",
    "\n",
    "from keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint\n",
    "import tensorflow as tf\n",
    "from keras import backend as K\n",
    "from keras.utils import multi_gpu_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['/job:localhost/replica:0/task:0/device:GPU:0']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K.tensorflow_backend._get_available_gpus()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = './datasets/jokes.pickle'\n",
    "VOCAB_PATH = './datasets/jokes_vocabulary.pickle'\n",
    "MODELS_PATH = './models/'\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 13\n",
    "VALIDATION_SPLIT = 0.2\n",
    "\n",
    "MODEL_PREFIX = 'jokes_stacked_lstm'\n",
    "\n",
    "EMBEDDING_DIM = 512\n",
    "HIDDEN_DIM1 = 1024\n",
    "HIDDEN_DIM2 = 512\n",
    "DEEPER_DIM = 512\n",
    "DROPOUT_FACTOR = 0.2\n",
    "REGULARIZATION = 0.00001\n",
    "LEARNING_RATE = 0.003\n",
    "\n",
    "DATA_PERCENT = 0.1\n",
    "\n",
    "RUN_INDEX = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sentences =  96910\n",
      "['sos i finally told her those three magical words every woman wish to hear threedots i give up ! eos', 'sos which cheeseburger makes a big hit in baseball ? a double ! eos']\n",
      "Vocab size =  8922\n",
      "['sos', 'did', 'you', 'hear', 'about', 'the', 'new', 'corduroy', 'pillows', '?']\n"
     ]
    }
   ],
   "source": [
    "with open(DATA_PATH, 'rb') as pickleFile:\n",
    "    sentences = pickle.load(pickleFile)\n",
    "\n",
    "with open(VOCAB_PATH, 'rb') as pickleFile:\n",
    "    vocab = pickle.load(pickleFile)\n",
    "    \n",
    "random.shuffle(sentences)\n",
    "\n",
    "print(\"Number of sentences = \", len(sentences))\n",
    "print(sentences[:2])\n",
    "print(\"Vocab size = \", len(vocab))\n",
    "print(vocab[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 7, 583, 123, 61, 320, 253, 4222, 522, 174, 138, 474, 8, 98, 15, 7, 181, 55, 21, 2], [1, 278, 4349, 304, 4, 229, 339, 14, 1245, 6, 4, 1065, 21, 2], [1, 32, 39, 536, 12, 2075, 1941, 6, 37, 39, 192, 293, 122, 313, 42, 3927, 1701, 3, 2], [1, 85, 3395, 86, 8, 4, 8479, 1286, 2], [1, 38, 13, 265, 2314, 24, 4, 270, 15, 15, 7, 95, 54, 5, 4019, 2]]\n",
      "8923\n"
     ]
    }
   ],
   "source": [
    "# tokenize data\n",
    "num_words = len(vocab)\n",
    "\n",
    "tokenizer = Tokenizer(num_words=None, filters='', lower=True, split=' ', \n",
    "                      char_level=False, oov_token=None)\n",
    "tokenizer.fit_on_texts(sentences)\n",
    "assert num_words == len(tokenizer.word_index)\n",
    "\n",
    "encoded_sentences = tokenizer.texts_to_sequences(sentences)\n",
    "print(encoded_sentences[:5])\n",
    "\n",
    "VOCAB_SIZE = len(tokenizer.word_index) + 1\n",
    "print(VOCAB_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving\n",
    "with open(MODELS_PATH + MODEL_PREFIX + '_tokenizer_' + str(RUN_INDEX) + '.pickle', 'wb') as handle:\n",
    "    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total training data size =  833131\n",
      "Max seq len =  13\n",
      "(833131, 13)\n",
      "[[   1    7  583  123   61  320  253 4222  522  174  138  474    8]\n",
      " [   7  583  123   61  320  253 4222  522  174  138  474    8   98]]\n",
      "(833131, 13, 1)\n",
      "[[[   7]\n",
      "  [ 583]\n",
      "  [ 123]\n",
      "  [  61]\n",
      "  [ 320]\n",
      "  [ 253]\n",
      "  [4222]\n",
      "  [ 522]\n",
      "  [ 174]\n",
      "  [ 138]\n",
      "  [ 474]\n",
      "  [   8]\n",
      "  [  98]]\n",
      "\n",
      " [[ 583]\n",
      "  [ 123]\n",
      "  [  61]\n",
      "  [ 320]\n",
      "  [ 253]\n",
      "  [4222]\n",
      "  [ 522]\n",
      "  [ 174]\n",
      "  [ 138]\n",
      "  [ 474]\n",
      "  [   8]\n",
      "  [  98]\n",
      "  [  15]]]\n"
     ]
    }
   ],
   "source": [
    "X_data = []\n",
    "y_data = []\n",
    "for sentence in encoded_sentences:\n",
    "    l = len(sentence)\n",
    "    sliding_window_length = min(l-3, MAX_SEQUENCE_LENGTH)\n",
    "    step_size = 1\n",
    "    for i in range(0, l - sliding_window_length, step_size):\n",
    "        X_data.append(sentence[i:i+sliding_window_length])\n",
    "        y_data.append(sentence[i+1:i+sliding_window_length+1])\n",
    "        \n",
    "print(\"Total training data size = \", len(X_data))\n",
    "MAX_SEQ_LEN = max([len(seq) for seq in X_data])\n",
    "print(\"Max seq len = \", MAX_SEQ_LEN)\n",
    "X_data = pad_sequences(X_data, maxlen=MAX_SEQ_LEN, padding='pre')\n",
    "y_data = pad_sequences(y_data, maxlen=MAX_SEQ_LEN, padding='pre').reshape(-1, MAX_SEQ_LEN, 1)\n",
    "#y_data = np.array(y_data).reshape(-1,1)\n",
    "print(X_data.shape)\n",
    "print(X_data[:2])\n",
    "print(y_data.shape)\n",
    "print(y_data[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define model\n",
    "def StackedLSTM(vocab_size, embedding_dim, hidden_dim1, hidden_dim2, deeper_dim, max_seq_len, \n",
    "           dropout_factor=0.5, regularization=0.00001, learning_rate=0.001):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Embedding(vocab_size, embedding_dim, #input_length=max_seq_len, \n",
    "                        mask_zero=True, embeddings_regularizer=regularizers.l2(regularization)))\n",
    "    model.add(LSTM(hidden_dim1, activation='tanh', \n",
    "                   kernel_regularizer=regularizers.l2(regularization), \n",
    "                   recurrent_regularizer=regularizers.l2(regularization), #unroll=True, \n",
    "                   return_sequences = True, dropout=dropout_factor, recurrent_dropout=dropout_factor))\n",
    "    model.add(LSTM(hidden_dim2, activation='tanh', \n",
    "                   kernel_regularizer=regularizers.l2(regularization), \n",
    "                   recurrent_regularizer=regularizers.l2(regularization), #unroll=True, \n",
    "                   return_sequences = True, dropout=dropout_factor, recurrent_dropout=dropout_factor))\n",
    "    model.add(TimeDistributed(Dropout(dropout_factor)))\n",
    "    model.add(Dense(units=deeper_dim, activation='tanh', kernel_regularizer=regularizers.l2(regularization)))\n",
    "    model.add(Dropout(dropout_factor))\n",
    "    model.add(Dense(units=vocab_size, activation='softmax', \n",
    "              kernel_regularizer=regularizers.l2(regularization)))\n",
    "    #model = multi_gpu_model(model)\n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer=RMSprop(lr=learning_rate),\n",
    "                  metrics=[sparse_categorical_crossentropy, sparse_categorical_accuracy], sample_weight_mode='temporal')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_1 (Embedding)      (None, None, 512)         4568576   \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, None, 1024)        6295552   \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, None, 512)         3147776   \n",
      "_________________________________________________________________\n",
      "time_distributed_1 (TimeDist (None, None, 512)         0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, None, 512)         262656    \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, None, 512)         0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, None, 8923)        4577499   \n",
      "=================================================================\n",
      "Total params: 18,852,059\n",
      "Trainable params: 18,852,059\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "K.clear_session()\n",
    "sess = tf.Session()\n",
    "K.set_session(sess)\n",
    "model = StackedLSTM(vocab_size=VOCAB_SIZE, embedding_dim=EMBEDDING_DIM, hidden_dim1=HIDDEN_DIM1, hidden_dim2=HIDDEN_DIM2, \n",
    "                    deeper_dim=DEEPER_DIM, max_seq_len=MAX_SEQ_LEN, dropout_factor=DROPOUT_FACTOR, \n",
    "                    regularization=REGULARIZATION, learning_rate=LEARNING_RATE)\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TB(TensorBoard):\n",
    "    def __init__(self, log_every=1, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.log_every = log_every\n",
    "        self.counter = 0\n",
    "    \n",
    "    def on_batch_end(self, batch, logs=None):\n",
    "        self.counter+=1\n",
    "        if self.counter%self.log_every==0:\n",
    "            for name, value in logs.items():\n",
    "                if name in ['batch', 'size']:\n",
    "                    continue\n",
    "                summary = tf.Summary()\n",
    "                summary_value = summary.value.add()\n",
    "                summary_value.simple_value = value.item()\n",
    "                summary_value.tag = name\n",
    "                self.writer.add_summary(summary, self.counter)\n",
    "            self.writer.flush()\n",
    "        \n",
    "        super().on_batch_end(batch, logs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 666504 samples, validate on 166627 samples\n",
      "Epoch 1/25\n",
      "666504/666504 [==============================] - 821s 1ms/step - loss: 5.1358 - sparse_categorical_crossentropy: 4.9671 - sparse_categorical_accuracy: 0.1787 - val_loss: 4.5949 - val_sparse_categorical_crossentropy: 4.3189 - val_sparse_categorical_accuracy: 0.2544\n",
      "\n",
      "Epoch 00001: saving model to ./models/checkpoints/jokes_stacked_lstm_gen6.01-4.59.hdf5\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 4.59488, saving model to ./models/jokes_stacked_lstm_gen6.hdf5\n",
      "Epoch 2/25\n",
      "666504/666504 [==============================] - 822s 1ms/step - loss: 4.4131 - sparse_categorical_crossentropy: 4.0814 - sparse_categorical_accuracy: 0.2667 - val_loss: 4.4985 - val_sparse_categorical_crossentropy: 4.1253 - val_sparse_categorical_accuracy: 0.2812\n",
      "\n",
      "Epoch 00002: saving model to ./models/checkpoints/jokes_stacked_lstm_gen6.02-4.50.hdf5\n",
      "\n",
      "Epoch 00002: val_loss improved from 4.59488 to 4.49845, saving model to ./models/jokes_stacked_lstm_gen6.hdf5\n",
      "Epoch 3/25\n",
      "173056/666504 [======>.......................] - ETA: 9:22 - loss: 4.3036 - sparse_categorical_crossentropy: 3.9191 - sparse_categorical_accuracy: 0.2845"
     ]
    }
   ],
   "source": [
    "start_time = time()\n",
    "tensorboard = TB(log_dir=\"./logs/\" + MODEL_PREFIX + \"/{}\".format(time()), \n",
    "                          histogram_freq=0, write_graph=True, write_images=False, log_every=10)\n",
    "\n",
    "callbacks=[tensorboard, \n",
    "           EarlyStopping(patience=5, monitor='val_loss'),\n",
    "           ModelCheckpoint(filepath=MODELS_PATH + 'checkpoints/'+ MODEL_PREFIX + '_gen'+str(RUN_INDEX)+'.{epoch:02d}-{val_loss:.2f}.hdf5', \n",
    "                           monitor='val_loss', verbose=1, mode='auto', period=1), \n",
    "           ModelCheckpoint(filepath=MODELS_PATH + MODEL_PREFIX + '_gen'+str(RUN_INDEX)+'.hdf5', \n",
    "                           monitor='val_loss', verbose=1, mode='auto', period=1, save_best_only=True)]\n",
    "\n",
    "model.fit(X_data, y_data, epochs=25, batch_size=1024, shuffle=True, verbose=1, validation_split=0.2, callbacks=callbacks)\n",
    "\n",
    "print(\"Total elapsed time: \", time()-start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate a sequence from a language model\n",
    "def generate(model, tokenizer, seed_text, maxlen, probabilistic=False, exploration_factor=0.0):\n",
    "    \n",
    "    reverse_word_map = dict(map(reversed, tokenizer.word_index.items()))\n",
    "    seq = tokenizer.texts_to_sequences([seed_text])[0]\n",
    "    print(seq)\n",
    "    \n",
    "    while True:\n",
    "        encoded_seq = seq\n",
    "        if len(seq) > MAX_SEQ_LEN:\n",
    "            encoded_seq = encoded_seq[-1*MAX_SEQ_LEN:]\n",
    "            \n",
    "        #padded_seq = pad_sequences([encoded_seq], maxlen=MAX_SEQ_LEN, padding='pre')\n",
    "        padded_seq = np.array([seq])\n",
    "        y_prob = model.predict(padded_seq)[0][-1].reshape(1,-1)#[3:].reshape(-1,1)\n",
    "        \n",
    "        if random.random() <= exploration_factor:\n",
    "            probabilistic = True\n",
    "        else:\n",
    "            probabilistic = False\n",
    "            \n",
    "        if probabilistic:\n",
    "            y_class = np.argmax(np.random.multinomial(1,y_prob[0]/(np.sum(y_prob[0])+1e-5),1))\n",
    "        else:\n",
    "            y_class = y_prob.argmax(axis=-1)[0]\n",
    "        \n",
    "        if y_class == 0:\n",
    "            break\n",
    "        out_word = reverse_word_map[y_class]\n",
    "        seq.append(y_class)\n",
    "        if out_word == 'eos' or len(seq) > maxlen or out_word == 'sos':\n",
    "            break\n",
    "    \n",
    "    words = [reverse_word_map[idx] for idx in seq]\n",
    "    \n",
    "    return ' '.join(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sos i finally told her those three magical words every woman wish to hear threedots i give up ! eos', 'sos which cheeseburger makes a big hit in baseball ? a double ! eos', \"sos how are america and mcdonald's similar ? they are both run by red - headed clowns . eos\", 'sos two toilets go to a potty lol eos', 'sos if my friends circle was a pizza threedots threedots i would be the crust eos', 'sos my buddy told me this one : \" people think i\\'m ugly until they see my bank account \" \" then they think i\\'m ugly and poor . \" eos', 'sos my dog always walked backwards threedots so i named him god . eos', 'sos i dated this muslim girl once threedots she was the bomb ! eos', \"sos trying to make a list of things that are worse than mondays and all i've got so far is hitler and christian rock . eos\", \"sos after sex i always get out my phone and order my wife a dozen roses . my girlfriend thinks i'm hilarious . eos\"]\n"
     ]
    }
   ],
   "source": [
    "print(sentences[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = load_model('models/jokes_stacked_lstm_gen6.hdf5')\n",
    "with open('models/jokes_stacked_lstm_tokenizer_6.pickle', 'rb') as pickleFile:\n",
    "    tokenizer = pickle.load(pickleFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 14, 5, 571]\n",
      "sos in the zoo , the only animal in the entire zoo is a dog . it's a shitzu . eos\n"
     ]
    }
   ],
   "source": [
    "joke = generate(model, tokenizer, \"sos i like \", maxlen=40, exploration_factor=0.2)\n",
    "print(joke)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sos hello', 'hello ,', \", i'm\", \"i'm a\", 'a dinosaur', 'dinosaur .', '. eos']\n"
     ]
    }
   ],
   "source": [
    "def bigrams_list(sentence):\n",
    "    words = sentence.split(' ')\n",
    "    bigrams = []\n",
    "    for i in range(0, len(words)-1):\n",
    "        bigrams.append(words[i]+' '+words[i+1])\n",
    "    return bigrams\n",
    "\n",
    "print(bigrams_list(\"sos hello , i'm a dinosaur . eos\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['sos i', 'i finally', 'finally told', 'told her', 'her those', 'those three', 'three magical', 'magical words', 'words every', 'every woman', 'woman wish', 'wish to', 'to hear', 'hear threedots', 'threedots i', 'i give', 'give up', 'up !', '! eos'], ['sos which', 'which cheeseburger', 'cheeseburger makes', 'makes a', 'a big', 'big hit', 'hit in', 'in baseball', 'baseball ?', '? a', 'a double', 'double !', '! eos']]\n"
     ]
    }
   ],
   "source": [
    "sentence_bigrams = [bigrams_list(s) for s in sentences]\n",
    "print(sentence_bigrams[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def intersection(lst1, lst2):\n",
    "    temp = set(lst2)\n",
    "    lst3 = [value for value in lst1 if value in temp]\n",
    "    return lst3\n",
    "\n",
    "def similarity_score(lst1, lst2):\n",
    "    intersection_len = len(intersection(lst1, lst2))\n",
    "    return (1.0*intersection_len)/len(lst1)#+len(lst2)-intersection_len)\n",
    " \n",
    "def print_closest_sentences(sentence, sentence_bigrams, top_k=3):\n",
    "    bigrams = bigrams_list(sentence)\n",
    "    scores = np.array([similarity_score(bigrams, sbigrams)\n",
    "                       for sbigrams in sentence_bigrams])\n",
    "    top_k_indices = scores.argsort()[-1*top_k:][::-1]\n",
    "    top_k_scores = scores[top_k_indices]\n",
    "    for k in range(top_k):\n",
    "        print(top_k_scores[k], \" -> \", sentences[top_k_indices[k]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3181818181818182  ->  sos my neighbour complained to me about my dog chasing people on a bike . my dog can't even ride a bike . eos\n",
      "0.2727272727272727  ->  sos my dog used to chase people on a bike a lot . it got so bad that i finally had to take his bike away . eos\n",
      "0.2727272727272727  ->  sos a man walks into a zoo threedots the only animal in the entire zoo is a dog . it's a shitzu . eos\n",
      "0.2727272727272727  ->  sos my dog used to chase people on a bike a lot it got so bad , finally i had to take his bike away . eos\n",
      "0.2727272727272727  ->  sos my best friend won't talk to me . it's because he is a dog . eos\n",
      "0.2727272727272727  ->  sos a man walks into a zoo threedots threedots the only animal in the whole entire zoo is a dog . it's a shitzu . eos\n",
      "0.2727272727272727  ->  sos a man walks into a zoo only to find out the only animal in the entire zoo is a dog . it's a shitzu . eos\n",
      "0.2727272727272727  ->  sos a man goes to a zoo but the only animal there is a dog . it's a shitzu . eos\n",
      "0.2727272727272727  ->  sos a man walks into a zoo . the only animal in the entire zoo is a dog . it's a shitzu . eos\n",
      "0.2727272727272727  ->  sos my wife is a lot like a facebook post threedots lots of people like her , she gets lots of comments , she has been shared by thousands of people , and she may be viral . eos\n"
     ]
    }
   ],
   "source": [
    "print_closest_sentences(joke, sentence_bigrams, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 17, 20, 9, 66]\n",
      "sos what do you call he when he just wheel it ? a carrot . eos\n",
      "0.4666666666666667  ->  sos what do you call the sound a ghost makes when he calls you ? a phone moan . eos\n",
      "0.4666666666666667  ->  sos what do you call a porno with aliens in it ? a science friction movie . eos\n",
      "0.4666666666666667  ->  sos what do you call an immature carrot ? a baby carrot . eos\n",
      "0.4666666666666667  ->  sos what do you call a black brick with three holes in it ? a polish bowling ball . eos\n",
      "0.4666666666666667  ->  sos what do you call it when you sit on a high chair to see if you want to buy it ? a stool sample . eos\n",
      "0.4666666666666667  ->  sos what do you call keanu reeves when he served under hitler ? a neo nazi . eos\n",
      "0.4  ->  sos what do you call a human that's now a cactus ? a transplant . eos\n",
      "0.4  ->  sos what do you call a baby donkey in spanish ? a burrito . eos\n",
      "0.4  ->  sos what do you call a person who drinks way too much soda ? a coke - o - nut . eos\n",
      "0.4  ->  sos what do you call a jewish baby in the womb ? a bun in the oven . eos\n"
     ]
    }
   ],
   "source": [
    "joke = generate(model, tokenizer, \"sos what do you call\", maxlen=40, exploration_factor=0.3)\n",
    "print(joke)\n",
    "print_closest_sentences(joke, sentence_bigrams, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sos save the whales . collect the whole set . eos', 'sos mickey mouse q : what kind of mouse walks on two legs ? a : mickey mouse q : what kind of duck walks on two legs ? a : they all do ! eos', 'sos did you know diarrhoea is hereditary ? it runs in your jeans . eos', 'sos i thought i saw jesus in a cookie . but i was wrong . it was just a guy who looked like him . eos', \"sos i'm a feminist so is my wife , but because i'm a man , i'm better at it . eos\", \"sos what's the difference between a good joke and click bait ? this isn't a good joke . eos\", 'sos when jokes go to parties , where do they wait for drinks ? in the punchlines . eos', 'sos when i found out top gear was being cancelled threedots i was so shocked it was like a punch in the face . eos', \"sos waiter asked if i wanted the soup or salad me : yes , i'll take the super salad eos\", 'sos president trump ! what about the aliens from space ? we need a roof ! eos']\n"
     ]
    }
   ],
   "source": [
    "print(sentences[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 7, 33, 4, 124]\n",
      "sos i like a girl with a lazy eye . i just can't find her pencil . eos\n",
      "0.5294117647058824  ->  sos i used to date a girl with a lazy eye . i had to break up with her . she was seeing somebody on the side . eos\n",
      "0.47058823529411764  ->  sos i was dating a girl with a lazy eye . unfortunately i had to break up with her because she was seeing someone else on the side . eos\n",
      "0.4117647058823529  ->  sos i dated a girl with a lazy eye once threedots caught her seeing someone on the side though . eos\n",
      "0.4117647058823529  ->  sos i once dated a girl with a lazy eye , it was going well for a few months until i realized she was seeing someone else . eos\n",
      "0.4117647058823529  ->  sos i dated a girl with a lazy eye , but it didn't work out . she was seeing someone else on the side . eos\n",
      "0.4117647058823529  ->  sos i used to date a girl with a lazy eye threedots turns out she was seeing someone else the whole time . eos\n",
      "0.4117647058823529  ->  sos i used to date a girl with a lazy eye threedots but i dumped her because she kept seeing people on the side . eos\n",
      "0.4117647058823529  ->  sos i used to date a girl with a lazy eye but i had to break it off because she was seeing someone on the side . eos\n",
      "0.4117647058823529  ->  sos i dated a girl with a lazy eye once . turns out she was seeing someone else the whole time . eos\n",
      "0.4117647058823529  ->  sos i used to date a girl with a lazy eye , i had to dump her because she kept seeing guys on the side . eos\n"
     ]
    }
   ],
   "source": [
    "joke = generate(model, tokenizer, \"sos i \", maxlen=40, exploration_factor=0.0)\n",
    "print(joke)\n",
    "print_closest_sentences(joke, sentence_bigrams, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-33-6cad3ef22911>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel1\u001b[0m \u001b[0;34m=\u001b[0m  \u001b[0mload_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'models/checkpoints/jokes_bilstm_gen2.08-4.39.hdf5'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/keras/models.py\u001b[0m in \u001b[0;36mload_model\u001b[0;34m(filepath, custom_objects, compile)\u001b[0m\n\u001b[1;32m    244\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    245\u001b[0m         \u001b[0;31m# set weights\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 246\u001b[0;31m         \u001b[0mtopology\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_weights_from_hdf5_group\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'model_weights'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    247\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    248\u001b[0m         \u001b[0;31m# Early return if compilation is not required.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/keras/engine/topology.py\u001b[0m in \u001b[0;36mload_weights_from_hdf5_group\u001b[0;34m(f, layers, reshape)\u001b[0m\n\u001b[1;32m   3187\u001b[0m                              ' elements.')\n\u001b[1;32m   3188\u001b[0m         \u001b[0mweight_value_tuples\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msymbolic_weights\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight_values\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3189\u001b[0;31m     \u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_set_value\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mweight_value_tuples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3190\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3191\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36mbatch_set_value\u001b[0;34m(tuples)\u001b[0m\n\u001b[1;32m   2355\u001b[0m         \u001b[0mfeed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2356\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtuples\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2357\u001b[0;31m             \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2358\u001b[0m             \u001b[0mtf_dtype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_dtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'_'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2359\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'_assign_placeholder'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/numpy/core/numeric.py\u001b[0m in \u001b[0;36masarray\u001b[0;34m(a, dtype, order)\u001b[0m\n\u001b[1;32m    490\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    491\u001b[0m     \"\"\"\n\u001b[0;32m--> 492\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0morder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0morder\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    493\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    494\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mh5py/_objects.pyx\u001b[0m in \u001b[0;36mh5py._objects.with_phil.wrapper\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/h5py/_hl/dataset.py\u001b[0m in \u001b[0;36m__array__\u001b[0;34m(self, dtype)\u001b[0m\n\u001b[1;32m    694\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    695\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 696\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_direct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    697\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0marr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    698\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/h5py/_hl/dataset.py\u001b[0m in \u001b[0;36mread_direct\u001b[0;34m(self, dest, source_sel, dest_sel)\u001b[0m\n\u001b[1;32m    655\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    656\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mmspace\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdest_sel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbroadcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msource_sel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 657\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmspace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfspace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdest\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdxpl\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dxpl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    658\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    659\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwrite_direct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msource_sel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdest_sel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "model1 =  load_model('models/checkpoints/jokes_bilstm_gen2.08-4.39.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 4, 107, 1832]\n",
      "sos a guy finds a rectal thermometer in his pocket and says \" i don't know what he laced them with , but i've been tripping all day . eos\n",
      "0.5517241379310345  ->  sos i recently bought some shoes from a drug dealer . i don't know what he laced them with , but i've been tripping all day . eos\n",
      "0.5517241379310345  ->  sos i recently bought shoes from a drug dealer i don't know what he laced them with , but i've been tripping all day . eos\n",
      "0.5517241379310345  ->  sos my drug dealer gave me new shoes today . i don't know what he laced them with , but i've been tripping all day . eos\n",
      "0.5517241379310345  ->  sos i bought some shoes from a drug dealer . i don't know what he laced them with , but i've been tripping all day . eos\n",
      "0.5517241379310345  ->  sos i bought shoes from my drug dealer today i don't know what he laced them with , but i've been tripping all day . eos\n",
      "0.5517241379310345  ->  sos shoes from a drug dealer i bought some shoes from a drug dealer . i don't know what he laced them with , but i've been tripping all day . eos\n",
      "0.5517241379310345  ->  sos i just bought shoes from a drug dealer threedots threedots i don't know what he laced them with , but i've been tripping all day . eos\n",
      "0.5517241379310345  ->  sos i bought some shoes from my dealer . i don't know what he laced them with , but i've been tripping all day . eos\n",
      "0.5517241379310345  ->  sos so i bought some shoes from a drug dealer today . i don't know what he laced them with , but i've been tripping all day . eos\n",
      "0.5517241379310345  ->  sos i bought a pair of shoes from a drug dealer today . i don't know what he laced them with , but i've been tripping all day . eos\n"
     ]
    }
   ],
   "source": [
    "joke = generate(model, tokenizer, \"sos a guy finds\", maxlen=40)\n",
    "print(joke)\n",
    "print_closest_sentences(joke, sentence_bigrams, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}