Jokes-Word-Controlled-Stacked-LSTM-Keras.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train a bilstm to generate jokes in forward and reverse based on a controlled bit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import importlib\n",
    "from library import data_preprocess as dp\n",
    "importlib.reload(dp)\n",
    "import random\n",
    "from time import time\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import keras\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential, Input\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.layers import LSTM, Bidirectional\n",
    "from keras.layers import Embedding, TimeDistributed, Flatten, Merge, Concatenate\n",
    "from keras import regularizers\n",
    "from keras.metrics import sparse_categorical_accuracy, sparse_categorical_crossentropy\n",
    "from keras.models import load_model\n",
    "from keras.optimizers import Adam\n",
    "from keras.models import Model\n",
    "\n",
    "from keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint\n",
    "import tensorflow as tf\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# http://nlp.stanford.edu/data/glove.6B.zip\n",
    "DATA_PATH = './datasets/jokes.pickle'\n",
    "VOCAB_PATH = './datasets/jokes_vocabulary.pickle'\n",
    "MODELS_PATH = './models/'\n",
    "\n",
    "GLOVE_PATH = './data/glove.6B.200d.txt'\n",
    "\n",
    "MODEL_PREFIX = 'jokes_controlled_stacked_lstm_glove'\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 13\n",
    "VALIDATION_SPLIT = 0.2\n",
    "\n",
    "GLOVE_EMBEDDING_DIM = 200\n",
    "EMBEDDING_DIM1 = 256\n",
    "EMBEDDING_DIM2 = 256\n",
    "HIDDEN_DIM1 = 512\n",
    "HIDDEN_DIM2 = 256\n",
    "DEEPER_DIM = 256\n",
    "DROPOUT_FACTOR = 0.2\n",
    "REGULARIZATION = 0.00001\n",
    "LEARNING_RATE = 0.003\n",
    "\n",
    "DATA_PERCENT = 0.1\n",
    "\n",
    "RUN_INDEX = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sentences =  96910\n",
      "['sos if god made everything threedots is god chinese ? eos', 'sos before you refer to someone as your ex , make sure they know you dated . eos']\n",
      "Vocab size =  8922\n",
      "['sos', 'did', 'you', 'hear', 'about', 'the', 'new', 'corduroy', 'pillows', '?']\n"
     ]
    }
   ],
   "source": [
    "with open(DATA_PATH, 'rb') as pickleFile:\n",
    "    sentences = pickle.load(pickleFile)\n",
    "\n",
    "with open(VOCAB_PATH, 'rb') as pickleFile:\n",
    "    vocab = pickle.load(pickleFile)\n",
    "    \n",
    "random.shuffle(sentences)\n",
    "\n",
    "print(\"Number of sentences = \", len(sentences))\n",
    "print(sentences[:2])\n",
    "print(\"Vocab size = \", len(vocab))\n",
    "print(vocab[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 38, 256, 205, 332, 15, 18, 256, 541, 6, 2], [1, 159, 9, 5211, 8, 125, 93, 35, 493, 10, 88, 249, 37, 67, 9, 2747, 3, 2], [1, 38, 9, 140, 752, 30, 983, 26, 4, 559, 15, 15, 37, 95, 27, 8, 1406, 9, 29, 904, 42, 5353, 2], [1, 217, 23, 17, 36, 5, 372, 60, 34, 61, 201, 123, 61, 28, 70, 24, 451, 6, 4, 23, 11, 18, 16, 602, 6, 11, 2], [1, 5, 704, 1294, 12, 924, 15, 5, 704, 1294, 12, 924, 188, 72, 4, 97, 3, 16, 24, 1761, 3, 2]]\n",
      "8923\n"
     ]
    }
   ],
   "source": [
    "# tokenize data\n",
    "num_words = len(vocab)\n",
    "\n",
    "tokenizer = Tokenizer(num_words=None, filters='', lower=True, split=' ', \n",
    "                      char_level=False, oov_token=None)\n",
    "tokenizer.fit_on_texts(sentences)\n",
    "assert num_words == len(tokenizer.word_index)\n",
    "\n",
    "encoded_sentences = tokenizer.texts_to_sequences(sentences)\n",
    "print(encoded_sentences[:5])\n",
    "\n",
    "VOCAB_SIZE = len(tokenizer.word_index) + 1\n",
    "print(VOCAB_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving\n",
    "with open(MODELS_PATH + MODEL_PREFIX + '_tokenizer_' + str(RUN_INDEX) + '.pickle', 'wb') as handle:\n",
    "    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total training data size =  833131\n",
      "Max seq len =  13\n",
      "(1666262, 13)\n",
      "[[ 150  246    5  194  128 2544  297   95   16   41   15   82  560]\n",
      " [ 169    4   24   31   15 7256 5006  243  726    5    8  190  633]\n",
      " [   0   36    9   98    6   51    5  663   28 2007   53  141    6]\n",
      " [ 636    7  134   46    5   11   23   22   11    6 4588   17   11]]\n",
      "(1666262, 13, 1)\n",
      "[[[ 246]\n",
      "  [   5]\n",
      "  [ 194]\n",
      "  [ 128]\n",
      "  [2544]\n",
      "  [ 297]\n",
      "  [  95]\n",
      "  [  16]\n",
      "  [  41]\n",
      "  [  15]\n",
      "  [  82]\n",
      "  [ 560]\n",
      "  [   4]]\n",
      "\n",
      " [[   4]\n",
      "  [  24]\n",
      "  [  31]\n",
      "  [  15]\n",
      "  [7256]\n",
      "  [5006]\n",
      "  [ 243]\n",
      "  [ 726]\n",
      "  [   5]\n",
      "  [   8]\n",
      "  [ 190]\n",
      "  [ 633]\n",
      "  [   4]]\n",
      "\n",
      " [[   0]\n",
      "  [   9]\n",
      "  [  98]\n",
      "  [   6]\n",
      "  [  51]\n",
      "  [   5]\n",
      "  [ 663]\n",
      "  [  28]\n",
      "  [2007]\n",
      "  [  53]\n",
      "  [ 141]\n",
      "  [   6]\n",
      "  [  15]]\n",
      "\n",
      " [[   7]\n",
      "  [ 134]\n",
      "  [  46]\n",
      "  [   5]\n",
      "  [  11]\n",
      "  [  23]\n",
      "  [  22]\n",
      "  [  11]\n",
      "  [   6]\n",
      "  [4588]\n",
      "  [  17]\n",
      "  [  11]\n",
      "  [  23]]]\n",
      "(1666262, 13, 2)\n",
      "[[[0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]]\n",
      "\n",
      " [[0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]]\n",
      "\n",
      " [[1. 0.]\n",
      "  [1. 0.]\n",
      "  [1. 0.]\n",
      "  [1. 0.]\n",
      "  [1. 0.]\n",
      "  [1. 0.]\n",
      "  [1. 0.]\n",
      "  [1. 0.]\n",
      "  [1. 0.]\n",
      "  [1. 0.]\n",
      "  [1. 0.]\n",
      "  [1. 0.]\n",
      "  [1. 0.]]\n",
      "\n",
      " [[0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]\n",
      "  [0. 1.]]]\n"
     ]
    }
   ],
   "source": [
    "TAG_SIZE = 2\n",
    "X_data = []\n",
    "y_data = []\n",
    "for sentence in encoded_sentences:\n",
    "    l = len(sentence)\n",
    "    sliding_window_length = min(l-3, MAX_SEQUENCE_LENGTH)\n",
    "    step_size = 1\n",
    "    for i in range(0, l - sliding_window_length, step_size):\n",
    "        X_data.append(sentence[i:i+sliding_window_length])\n",
    "        y_data.append(sentence[i+1:i+sliding_window_length+1])\n",
    "        \n",
    "print(\"Total training data size = \", len(X_data))\n",
    "MAX_SEQ_LEN = max([len(seq) for seq in X_data])\n",
    "print(\"Max seq len = \", MAX_SEQ_LEN)\n",
    "\n",
    "forward_X_data = pad_sequences(X_data, maxlen=MAX_SEQ_LEN, padding='pre')\n",
    "forward_y_data = pad_sequences(y_data, maxlen=MAX_SEQ_LEN, padding='pre').reshape(-1, MAX_SEQ_LEN, 1)\n",
    "forward_tag = to_categorical(np.full((forward_X_data.shape[0], MAX_SEQ_LEN), 0), TAG_SIZE)\n",
    "\n",
    "#print(forward_X_data.shape)\n",
    "#print(forward_X_data[:2])\n",
    "#print(forward_y_data.shape)\n",
    "#print(forward_y_data[:2])\n",
    "#print(forward_tag.shape)\n",
    "\n",
    "reverse_X_data = pad_sequences([item[::-1] for item in y_data], maxlen=MAX_SEQ_LEN, padding='pre')\n",
    "reverse_y_data = pad_sequences([item[::-1] for item in X_data], maxlen=MAX_SEQ_LEN, padding='pre').reshape(-1, MAX_SEQ_LEN, 1)\n",
    "reverse_tag = to_categorical(np.full((reverse_X_data.shape[0], MAX_SEQ_LEN), 1), TAG_SIZE)\n",
    "\n",
    "#print(reverse_X_data.shape)\n",
    "#print(reverse_X_data[:2])\n",
    "#print(reverse_y_data.shape)\n",
    "#print(reverse_y_data[:2])\n",
    "#print(reverse_tag.shape)\n",
    "\n",
    "X_data = np.concatenate((forward_X_data, reverse_X_data), axis=0)\n",
    "y_data = np.concatenate((forward_y_data, reverse_y_data), axis=0)\n",
    "tag_data = np.concatenate((forward_tag, reverse_tag), axis=0)\n",
    "\n",
    "# shuffle\n",
    "perm = np.random.permutation(X_data.shape[0])\n",
    "X_data = X_data[perm]\n",
    "y_data = y_data[perm]\n",
    "tag_data = tag_data[perm]\n",
    "print(X_data.shape)\n",
    "print(X_data[:4])\n",
    "print(y_data.shape)\n",
    "print(y_data[:4])\n",
    "print(tag_data.shape)\n",
    "print(tag_data[:4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing glove word vectors\n",
      "Total 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "print('Indexing glove word vectors')\n",
    "#Glove Vectors\n",
    "glove_embeddings_index = {}\n",
    "f = open(GLOVE_PATH)\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    glove_embeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Total %s word vectors.' % len(glove_embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing glove embedding matrix\n",
      "Null word embeddings: 255\n",
      "(8923, 200)\n"
     ]
    }
   ],
   "source": [
    "print('Preparing glove embedding matrix')\n",
    "glove_embedding_matrix = np.zeros((VOCAB_SIZE, GLOVE_EMBEDDING_DIM))\n",
    "for word,i in tokenizer.word_index.items():\n",
    "    embedding_vector = glove_embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        glove_embedding_matrix[i] = embedding_vector\n",
    "print('Null word embeddings: %d' % np.sum(np.sum(glove_embedding_matrix, axis=1) == 0))\n",
    "print(glove_embedding_matrix.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define model\n",
    "def StackedLSTM(vocab_size, glove_embedding_dim, glove_embedding_matrix, embedding_dim1, embedding_dim2,\n",
    "           hidden_dim1, hidden_dim2, deeper_dim, max_seq_len, tag_size,\n",
    "           dropout_factor=0.5, regularization=0.00001, learning_rate=0.001):\n",
    "    \n",
    "    inputs = Input(shape=(None,))\n",
    "    tag_inputs = Input(shape=(None,tag_size))\n",
    "    \n",
    "    glove_embedding = Embedding(vocab_size, glove_embedding_dim, #input_length=max_seq_len,\n",
    "                                  weights=[glove_embedding_matrix],\n",
    "                                  mask_zero=True,trainable=False)(inputs)\n",
    "    \n",
    "    word_embedding = Embedding(vocab_size, embedding_dim1, #input_length=max_seq_len, \n",
    "                               mask_zero=True, embeddings_regularizer=regularizers.l2(regularization))(inputs)\n",
    "    \n",
    "    #tag_embedding = Embedding(tag_size, tag_size, embeddings_regularizer=regularizers.l2(regularization))(tag_inputs)\n",
    "    \n",
    "    concat_embeds = Concatenate(axis=-1)([glove_embedding, word_embedding, tag_inputs])\n",
    "    \n",
    "    final_embed = Dense(units=embedding_dim2, activation='tanh',\n",
    "                        kernel_regularizer=regularizers.l2(regularization))(concat_embeds)\n",
    "    \n",
    "    lstm1 = LSTM(hidden_dim1, activation='tanh', \n",
    "                   kernel_regularizer=regularizers.l2(regularization), \n",
    "                   recurrent_regularizer=regularizers.l2(regularization), #unroll=True, \n",
    "                   return_sequences = True, dropout=dropout_factor, recurrent_dropout=dropout_factor)(final_embed)\n",
    "    \n",
    "    lstm2 = LSTM(hidden_dim2, activation='tanh', \n",
    "                   kernel_regularizer=regularizers.l2(regularization), \n",
    "                   recurrent_regularizer=regularizers.l2(regularization), #unroll=True, \n",
    "                   return_sequences = True, dropout=dropout_factor, recurrent_dropout=dropout_factor)(lstm1)\n",
    "    \n",
    "    timedist_dropout = TimeDistributed(Dropout(dropout_factor))(lstm2)\n",
    "    \n",
    "    deep_dense = Dense(units=deeper_dim, activation='tanh', \n",
    "                       kernel_regularizer=regularizers.l2(regularization))(timedist_dropout)\n",
    "    \n",
    "    dropout_layer1 = Dropout(dropout_factor)(deep_dense)\n",
    "    \n",
    "    outputs = Dense(units=vocab_size, activation='softmax', \n",
    "                    kernel_regularizer=regularizers.l2(regularization))(dropout_layer1)\n",
    "    \n",
    "    model = Model(inputs=[inputs,tag_inputs], outputs=outputs)\n",
    "    \n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam(lr=learning_rate),\n",
    "                  metrics=[sparse_categorical_crossentropy, sparse_categorical_accuracy], sample_weight_mode='temporal')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'K' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-57957770243e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclear_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mconfig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mConfigProto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mallow_soft_placement\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0msess\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSession\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mK\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'K' is not defined"
     ]
    }
   ],
   "source": [
    "K.clear_session()\n",
    "sess = tf.Session()\n",
    "K.set_session(sess)\n",
    "\n",
    "model = StackedLSTM(vocab_size=VOCAB_SIZE, glove_embedding_dim=GLOVE_EMBEDDING_DIM,\n",
    "                    glove_embedding_matrix=glove_embedding_matrix, \n",
    "                    embedding_dim1=EMBEDDING_DIM1, embedding_dim2=EMBEDDING_DIM2,\n",
    "                    hidden_dim1=HIDDEN_DIM1, hidden_dim2=HIDDEN_DIM2,\n",
    "                    deeper_dim=DEEPER_DIM, max_seq_len=MAX_SEQ_LEN, dropout_factor=DROPOUT_FACTOR, \n",
    "                    regularization=REGULARIZATION, learning_rate=LEARNING_RATE, tag_size=TAG_SIZE)\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TB(TensorBoard):\n",
    "    def __init__(self, log_every=1, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.log_every = log_every\n",
    "        self.counter = 0\n",
    "    \n",
    "    def on_batch_end(self, batch, logs=None):\n",
    "        self.counter+=1\n",
    "        if self.counter%self.log_every==0:\n",
    "            for name, value in logs.items():\n",
    "                if name in ['batch', 'size']:\n",
    "                    continue\n",
    "                summary = tf.Summary()\n",
    "                summary_value = summary.value.add()\n",
    "                summary_value.simple_value = value.item()\n",
    "                summary_value.tag = name\n",
    "                self.writer.add_summary(summary, self.counter)\n",
    "            self.writer.flush()\n",
    "        \n",
    "        super().on_batch_end(batch, logs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1333009 samples, validate on 333253 samples\n",
      "Epoch 1/10\n",
      "1333009/1333009 [==============================] - 968s 726us/step - loss: 5.4862 - sparse_categorical_crossentropy: 5.3589 - sparse_categorical_accuracy: 0.1469 - val_loss: 4.5909 - val_sparse_categorical_crossentropy: 4.3533 - val_sparse_categorical_accuracy: 0.2517\n",
      "\n",
      "Epoch 00001: saving model to ./models/checkpoints/jokes_controlled_stacked_lstm_glove_gen1.01-4.59.hdf5\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 4.59085, saving model to ./models/jokes_controlled_stacked_lstm_glove_gen1.hdf5\n",
      "Epoch 2/10\n",
      "1333009/1333009 [==============================] - 969s 727us/step - loss: 4.6670 - sparse_categorical_crossentropy: 4.3963 - sparse_categorical_accuracy: 0.2463 - val_loss: 4.3883 - val_sparse_categorical_crossentropy: 4.0964 - val_sparse_categorical_accuracy: 0.2820\n",
      "\n",
      "Epoch 00002: saving model to ./models/checkpoints/jokes_controlled_stacked_lstm_glove_gen1.02-4.39.hdf5\n",
      "\n",
      "Epoch 00002: val_loss improved from 4.59085 to 4.38835, saving model to ./models/jokes_controlled_stacked_lstm_glove_gen1.hdf5\n",
      "Epoch 3/10\n",
      "1333009/1333009 [==============================] - 969s 727us/step - loss: 4.5716 - sparse_categorical_crossentropy: 4.2674 - sparse_categorical_accuracy: 0.2612 - val_loss: 4.3114 - val_sparse_categorical_crossentropy: 3.9990 - val_sparse_categorical_accuracy: 0.2936\n",
      "\n",
      "Epoch 00003: saving model to ./models/checkpoints/jokes_controlled_stacked_lstm_glove_gen1.03-4.31.hdf5\n",
      "\n",
      "Epoch 00003: val_loss improved from 4.38835 to 4.31136, saving model to ./models/jokes_controlled_stacked_lstm_glove_gen1.hdf5\n",
      "Epoch 4/10\n",
      "1333009/1333009 [==============================] - 970s 728us/step - loss: 4.5265 - sparse_categorical_crossentropy: 4.2072 - sparse_categorical_accuracy: 0.2681 - val_loss: 4.2710 - val_sparse_categorical_crossentropy: 3.9478 - val_sparse_categorical_accuracy: 0.2997\n",
      "\n",
      "Epoch 00004: saving model to ./models/checkpoints/jokes_controlled_stacked_lstm_glove_gen1.04-4.27.hdf5\n",
      "\n",
      "Epoch 00004: val_loss improved from 4.31136 to 4.27095, saving model to ./models/jokes_controlled_stacked_lstm_glove_gen1.hdf5\n",
      "Epoch 5/10\n",
      " 844800/1333009 [==================>...........] - ETA: 5:27 - loss: 4.5008 - sparse_categorical_crossentropy: 4.1737 - sparse_categorical_accuracy: 0.2718"
     ]
    }
   ],
   "source": [
    "start_time = time()\n",
    "tensorboard = TB(log_dir=\"./logs/\" + MODEL_PREFIX + \"/{}\".format(time()), \n",
    "                          histogram_freq=0, write_graph=True, write_images=False, log_every=10)\n",
    "\n",
    "callbacks=[tensorboard, \n",
    "           EarlyStopping(patience=5, monitor='val_loss'),\n",
    "           ModelCheckpoint(filepath=MODELS_PATH + 'checkpoints/'+ MODEL_PREFIX + '_gen' + str(RUN_INDEX) + '.{epoch:02d}-{val_loss:.2f}.hdf5', \n",
    "                           monitor='val_loss', verbose=1, mode='auto', period=1), \n",
    "           ModelCheckpoint(filepath=MODELS_PATH + MODEL_PREFIX + '_gen'+str(RUN_INDEX)+'.hdf5', \n",
    "                           monitor='val_loss', verbose=1, mode='auto', period=1, save_best_only=True)]\n",
    "\n",
    "model.fit([X_data, tag_data], y_data, epochs=10, batch_size=1024, shuffle=True, verbose=1, validation_split=0.2, callbacks=callbacks)\n",
    "\n",
    "print(\"Total elapsed time: \", time()-start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate a sequence from a language model\n",
    "def generate_categorical(model, tokenizer, seed_text, maxlen, probabilistic=False, exploration_factor=1.0, tag=0):\n",
    "    \n",
    "    reverse_word_map = dict(map(reversed, tokenizer.word_index.items()))\n",
    "    seq = tokenizer.texts_to_sequences([seed_text])[0]\n",
    "    print(seq)\n",
    "    \n",
    "    while True:\n",
    "        encoded_seq = seq\n",
    "        if len(seq) > MAX_SEQ_LEN:\n",
    "            encoded_seq = encoded_seq[-1*MAX_SEQ_LEN:]\n",
    "            \n",
    "        #padded_seq = pad_sequences([encoded_seq], maxlen=MAX_SEQ_LEN, padding='pre')\n",
    "        padded_seq = np.array([seq])\n",
    "        tags = to_categorical(np.full((1, padded_seq[0].shape[0]), tag), TAG_SIZE)\n",
    "        y_prob = model.predict([padded_seq,tags])[0][-1].reshape(1,-1)#[3:].reshape(-1,1)\n",
    "        \n",
    "        if random.random() <= exploration_factor:\n",
    "            probabilistic = True\n",
    "        else:\n",
    "            probabilistic = False\n",
    "            \n",
    "        if probabilistic:\n",
    "            y_class = np.argmax(np.random.multinomial(1,y_prob[0]/(np.sum(y_prob[0])+1e-5),1))\n",
    "        else:\n",
    "            y_class = y_prob.argmax(axis=-1)[0]\n",
    "        \n",
    "        if y_class == 0:\n",
    "            break\n",
    "        out_word = reverse_word_map[y_class]\n",
    "        seq.append(y_class)\n",
    "        if out_word == 'eos' or len(seq) > maxlen or out_word == 'sos':\n",
    "            break\n",
    "    \n",
    "    words = [reverse_word_map[idx] for idx in seq]\n",
    "    \n",
    "    return ' '.join(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "K.clear_session()\n",
    "sess = tf.Session()\n",
    "K.set_session(sess)\n",
    "model = load_model('models/jokes_controlled_stacked_lstm_glove_gen1.hdf5')\n",
    "with open('models/jokes_controlled_stacked_lstm_glove_tokenizer_1.pickle', 'rb') as pickleFile:\n",
    "    tokenizer = pickle.load(pickleFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sos if god made everything threedots is god chinese ? eos', 'sos before you refer to someone as your ex , make sure they know you dated . eos', 'sos if you were arrested for masturbating on a plane threedots threedots they would have to charge you with hi - jacking eos', 'sos q : what did the blonde say when her doctor told her that she was pregnant ? a : \" is it mine ? \" eos', 'sos the past present and future threedots the past present and future walk into a bar . it was tense . eos', \"sos knock knock who's there ? pill . pill who ? pill cosby . eos\", \"sos i went on one of those once in a lifetime holidays last week threedots i won't be doing that again . joke by tim vine eos\", 'sos when your boss asks you \" do i look stupid to you ? \" it\\'s a rhetorical question i know this now eos', \"sos i ate an optimist once threedots but i couldn't keep him down . eos\", 'sos reddit accounts should be treated like underwear . if you can keep them clean you could keep the same one for a while , but if you flood them with shit you should get a new one . eos']\n"
     ]
    }
   ],
   "source": [
    "print(sentences[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 13, 500, 720]\n",
      "sos my boss likes in the bathroom and i don't know what that working they are . eos\n"
     ]
    }
   ],
   "source": [
    "MAX_SEQ_LEN = 13\n",
    "TAG_SIZE = 2\n",
    "joke = generate_categorical(model, tokenizer, \"sos my boss likes\", maxlen=40, tag=0, exploration_factor=0.1)\n",
    "print(joke)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sos hello', 'hello ,', \", i'm\", \"i'm a\", 'a dinosaur', 'dinosaur .', '. eos']\n"
     ]
    }
   ],
   "source": [
    "def bigrams_list(sentence):\n",
    "    words = sentence.split(' ')\n",
    "    bigrams = []\n",
    "    for i in range(0, len(words)-1):\n",
    "        bigrams.append(words[i]+' '+words[i+1])\n",
    "    return bigrams\n",
    "\n",
    "print(bigrams_list(\"sos hello , i'm a dinosaur . eos\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[\"sos what's\", \"what's the\", 'the difference', 'difference between', 'between a', 'a strawberry', 'strawberry and', 'and a', 'a pencil', 'pencil ?', \"? one's\", \"one's a\", 'a fruit', 'fruit ,', ', you', 'you idiot', 'idiot .', '. eos'], ['sos my', 'my bank', 'bank says', 'says my', 'my password', \"password isn't\", \"isn't strong\", 'strong enough', 'enough .', '. did', 'did it', 'it ever', 'ever stop', 'stop and', 'and think', 'think that', 'that my', 'my password', 'password has', 'has a', 'a lot', 'lot going', 'going on', 'on right', 'right now', 'now ?', '? eos']]\n"
     ]
    }
   ],
   "source": [
    "sentence_bigrams = [bigrams_list(s) for s in sentences]\n",
    "print(sentence_bigrams[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def intersection(lst1, lst2):\n",
    "    temp = set(lst2)\n",
    "    lst3 = [value for value in lst1 if value in temp]\n",
    "    return lst3\n",
    "\n",
    "def similarity_score(lst1, lst2):\n",
    "    intersection_len = len(intersection(lst1, lst2))\n",
    "    return (1.0*intersection_len)/len(lst1)#+len(lst2)-intersection_len)\n",
    " \n",
    "def print_closest_sentences(sentence, sentence_bigrams, top_k=3):\n",
    "    bigrams = bigrams_list(sentence)\n",
    "    scores = np.array([similarity_score(bigrams, sbigrams)\n",
    "                       for sbigrams in sentence_bigrams])\n",
    "    top_k_indices = scores.argsort()[-1*top_k:][::-1]\n",
    "    top_k_scores = scores[top_k_indices]\n",
    "    for k in range(top_k):\n",
    "        print(top_k_scores[k], \" -> \", sentences[top_k_indices[k]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.16666666666666666  ->  sos i really want to make a period joke . but it's just not that time of the month for me eos\n",
      "0.1111111111111111  ->  sos \" was i adopted ? \" \" yes . but they brought you back . \" eos\n",
      "0.1111111111111111  ->  sos why did the blonde freeze a pot of boiling water ? because you never know when you're going to need some boiling water . eos\n",
      "0.1111111111111111  ->  sos a good time to keep your mouth shut is when you're in deep water . eos\n",
      "0.1111111111111111  ->  sos i better have a baby soon before my mom gets too old to raise it for me eos\n",
      "0.1111111111111111  ->  sos when you're sad , hug a kid . but make sure it's yours cuz that shit would be weird . eos\n",
      "0.1111111111111111  ->  sos if i'm ever being chased by a giraffe i'm gonna run into a place with low ceiling fans sorry giraffe but i gotta do what's best for me eos\n",
      "0.1111111111111111  ->  sos he was a real gentlemen and always opened the fridge door for me eos\n",
      "0.1111111111111111  ->  sos i wanted to see lot of animals so i went to the zoo . but they only had one small dog . it was a shih - tzu . eos\n",
      "0.1111111111111111  ->  sos blood is thicker than water . but maple syrup is thicker than blood . so pancakes are more important than family . i said it . eos\n"
     ]
    }
   ],
   "source": [
    "print_closest_sentences(joke, sentence_bigrams, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 195, 4, 110]\n",
      "sos i wanted a coming elton so hard to neighbors , \" i'm not having god \" so he engineers off a baby eos\n",
      "0.17391304347826086  ->  sos i crashed into a dwarf at some traffic lights he got out of his car and said \" i'm not happy . \" i replied , \" well , which one are you then ? \" eos\n",
      "0.17391304347826086  ->  sos i had a parrot that talked . . . but it never said , \" i'm hungry . \" so it died . eos\n",
      "0.17391304347826086  ->  sos i texted my boss , \" what's the difference between this morning and your daughter ? \" he answered , \" i don't know . \" i replied , \" i'm not coming in this morning . \" eos\n",
      "0.17391304347826086  ->  sos i had a parrot . the parrot talked , but it did not say \" i'm hungry , \" so it died . eos\n",
      "0.17391304347826086  ->  sos i told the barber , \" a little off the top . \" so he gave me a circumcision . eos\n",
      "0.17391304347826086  ->  sos i rang up a local builder and said , \" i want a skip outside my house . \" he said , \" i'm not stopping you . \" * tim vine * eos\n",
      "0.13043478260869565  ->  sos i went to the doctor today . he told me i was fat . i said i wanted a second opinion . he says , okay , you're ugly . eos\n",
      "0.13043478260869565  ->  sos i found a new way of making popcorn threedots just give an ear a baby eos\n",
      "0.13043478260869565  ->  sos i ask myself , \" how did i get here ? , \" i'm sure my neighbors ask the same question every time they catch me in their house threedots taking a shower . eos\n",
      "0.13043478260869565  ->  sos i knew this guy who was so dumb threedots he saw a road sign that said , \" disney land left \" , so he turned around and went home . eos\n"
     ]
    }
   ],
   "source": [
    "joke = generate_categorical(model, tokenizer, \"eos baby a off\", maxlen=40, tag=1)\n",
    "joke = ' '.join(joke.split(' ')[::-1])\n",
    "print(joke)\n",
    "print_closest_sentences(joke, sentence_bigrams, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2, 3, 2082, 1698]\n",
      "sos what do you call a bunch of sauce on a plane ? the whiskey contact sport . eos\n",
      "0.5555555555555556  ->  sos what do you call a bunch of male pornstars on a flight together ? snakes on a plane . eos\n",
      "0.5555555555555556  ->  sos what do you call a bunch of white guys sitting on a bench ? the nba . eos\n",
      "0.5  ->  sos what do you call a black man flying a plane ? the pilot you racist fuck . eos\n",
      "0.5  ->  sos what do you call a middle eastern man flying a plane ? the pilot , you racist . eos\n",
      "0.5  ->  sos what do you call a bunch of white guys on a bench ? the nba eos\n",
      "0.5  ->  sos what do you call a bunch of white guys sitting on a bench ? the nba eos\n",
      "0.5  ->  sos what do you call a gay guy flying a plane ? the pilot . eos\n",
      "0.5  ->  sos what do you call a muslim on a plane ? a passenger threedots you racist bastard . eos\n",
      "0.5  ->  sos what do you call a black man flying a plane ? the pilot . eos\n",
      "0.5  ->  sos what do you call a bunch of white dudes sitting on a bench ? the nba eos\n"
     ]
    }
   ],
   "source": [
    "joke = generate_categorical(model, tokenizer, \"eos . sport contact\", maxlen=40, tag=1)\n",
    "joke = ' '.join(joke.split(' ')[::-1])\n",
    "print(joke)\n",
    "print_closest_sentences(joke, sentence_bigrams, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 7, 237, 58, 121]\n",
      "sos i am not going to be a comedian . i just say \" i don't know what i know , but i don't know what , but i don't know what i think about . \" eos\n",
      "0.40540540540540543  ->  sos i bought some shoes from my pill dealer on friday . i don't know what he laced them with , but i was tripping all weekend . eos\n",
      "0.40540540540540543  ->  sos i bought shoes from a drug dealer once . i don't know what he laced them with , but i was tripping all day . eos\n",
      "0.40540540540540543  ->  sos i don't know what you do when you come across a bear , but i just wipe it off and apologize . eos\n",
      "0.3783783783783784  ->  sos i bought shoes from a drug dealer i don't know what he laced them with , but i was tripping all day eos\n",
      "0.3783783783783784  ->  sos this is the last pair of shoes i buy from a drug dealer threedots i don't know what he laced them with , but i just keep tripping . eos\n",
      "0.3783783783783784  ->  sos i bought shoes off a drug dealer i don't know what he laced them with , but i was tripping all day . eos\n",
      "0.3783783783783784  ->  sos i don't know what i drank last night , but the vacuum is stuck on top of the house . eos\n",
      "0.3783783783783784  ->  sos i don't know about you , but i can't wait to be ashamed about what i do this weekend . eos\n",
      "0.3783783783783784  ->  sos i bought shoes from a drug dealer i don't know what he laced them with , but i ' ve been tripping all day ! eos\n",
      "0.3783783783783784  ->  sos i bought shoes from a drug dealer once threedots i don't know what he laced them with , but i was tripping all day . eos\n"
     ]
    }
   ],
   "source": [
    "joke = generate_categorical(model, tokenizer, \"sos i am not going\", maxlen=40, tag=0, exploration_factor=0.3)\n",
    "print(joke)\n",
    "print_closest_sentences(joke, sentence_bigrams, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}