Combined-Word-Glove-BiLSTM-Keras.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import importlib\n",
    "from library import data_preprocess as dp\n",
    "importlib.reload(dp)\n",
    "import random\n",
    "from time import time\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import keras\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Activation,Input\n",
    "from keras.layers import LSTM, Bidirectional\n",
    "from keras.layers import Embedding\n",
    "from keras import regularizers\n",
    "from keras.metrics import sparse_categorical_accuracy, sparse_categorical_crossentropy\n",
    "from keras.models import load_model\n",
    "from keras.layers.merge import Concatenate\n",
    "from keras.utils.np_utils import to_categorical   \n",
    "from keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint\n",
    "import tensorflow as tf\n",
    "\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "#from keras import initializations\n",
    "from keras import initializers, regularizers, constraints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        \"\"\"\n",
    "        Keras Layer that implements an Attention mechanism for temporal data.\n",
    "        Supports Masking.\n",
    "        Follows the work of Raffel et al. [https://arxiv.org/abs/1512.08756]\n",
    "        # Input shape\n",
    "            3D tensor with shape: `(samples, steps, features)`.\n",
    "        # Output shape\n",
    "            2D tensor with shape: `(samples, features)`.\n",
    "        :param kwargs:\n",
    "        Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n",
    "        The dimensions are inferred based on the output shape of the RNN.\n",
    "        Example:\n",
    "            model.add(LSTM(64, return_sequences=True))\n",
    "            model.add(Attention())\n",
    "        \"\"\"\n",
    "        self.supports_masking = True\n",
    "        #self.init = initializations.get('glorot_uniform')\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        # do not pass the mask to the next layers\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        # eij = K.dot(x, self.W) TF backend doesn't support it\n",
    "\n",
    "        # features_dim = self.W.shape[0]\n",
    "        # step_dim = x._keras_shape[1]\n",
    "\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        # apply mask after the exp. will be re-normalized next\n",
    "        if mask is not None:\n",
    "            # Cast the mask to floatX to avoid float64 upcasting in theano\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        # in some cases especially in the early stages of training the sum may be almost zero\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "    #print weigthted_input.shape\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        #return input_shape[0], input_shape[-1]\n",
    "        return input_shape[0],  self.features_dim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = './datasets/combined.pickle'\n",
    "VOCAB_PATH = './datasets/combined_vocabulary.pickle'\n",
    "MODELS_PATH = './models/'\n",
    "EMBEDDING_FILE = \"./data/glove/glove.6B.100d.txt\"\n",
    "MAX_SEQ_LEN = 13\n",
    "VALIDATION_SPLIT = 0.1\n",
    "\n",
    "EMBEDDING_DIM = 100\n",
    "\n",
    "HIDDEN_DIM = 512\n",
    "DROPOUT_FACTOR = 0.333\n",
    "REGULARIZATION = 0.00001\n",
    "DEEPER_DIM = 256\n",
    "DATA_PERCENT = 0.1\n",
    "RUN_INDEX = 4\n",
    "TAG_DIM = 3\n",
    "# EMBEDDING_FILE = \"./data/glove/glove.6B.100d.txt\"\n",
    "# TRAIN_DATA_FILE = \"./datasets/combined.pickle\"\n",
    "# VOCABULARY_FILE = \"./datasets/combined_vocabulary.pickle\"\n",
    "# MAX_SEQUENCE_LENGTH = 10\n",
    "MAX_NB_WORDS = 200000\n",
    "# EMBEDDING_DIM = 100\n",
    "SEQUENCE_STEP = 1\n",
    "# #VALIDATION_SPLIT = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sentences =  270543\n",
      "[[2, \"sos thinking about all the guys who have screwed me over . i just want to find someone who's real . eos\"], [0, 'sos what comes after l ? bow eos']]\n",
      "Vocab size =  12614\n",
      "['liberate', 'savings', 'clip', 'rhino', 'cross', 'encouragement', 'viewed', 'womens', 'lap', 'challenges']\n"
     ]
    }
   ],
   "source": [
    "with open(DATA_PATH, 'rb') as pickleFile:\n",
    "    tags_and_sentences = pickle.load(pickleFile)\n",
    "\n",
    "with open(VOCAB_PATH, 'rb') as pickleFile:\n",
    "    vocab = pickle.load(pickleFile)\n",
    "    \n",
    "random.shuffle(sentences)\n",
    "\n",
    "print(\"Number of sentences = \", len(sentences))\n",
    "print(sentences[:2])\n",
    "print(\"Vocab size = \", len(vocab))\n",
    "print(vocab[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(270543, 2)\n",
      "(270543,)\n",
      "(270543, 3)\n"
     ]
    }
   ],
   "source": [
    "#Specially process combined data (separate texts and index)\n",
    "combined_data = np.array(tags_and_sentences)\n",
    "print(combined_data.shape)\n",
    "sentences = np.array(combined_data[:,1])\n",
    "tags = to_categorical(np.array(combined_data[:,0]))\n",
    "print(sentences.shape)\n",
    "print(tags.shape)\n",
    "#print(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 72, 14, 205, 56, 4, 104, 4254, 4975, 13, 282, 309, 4114, 1263, 16, 2], [1, 7, 1704, 6, 3990, 8, 2731, 8, 9, 3468, 3, 323, 53, 101, 716, 3, 2], [1, 49, 1195, 11, 2086, 12, 12, 113, 11, 973, 1490, 13, 2], [1, 5, 1567, 2822, 85, 40, 4, 1948, 28, 46, 858, 7, 945, 3, 2], [1, 50, 34, 4, 3231, 2841, 6, 4, 367, 13, 71, 296, 7, 4384, 3, 2]]\n",
      "12615\n"
     ]
    }
   ],
   "source": [
    "# tokenize data\n",
    "num_words = len(vocab)\n",
    "\n",
    "tokenizer = Tokenizer(num_words=None, filters='', lower=True, split=' ', \n",
    "                      char_level=False, oov_token=None)\n",
    "tokenizer.fit_on_texts(sentences)\n",
    "assert num_words == len(tokenizer.word_index)\n",
    "#word to index\n",
    "word_index = tokenizer.word_index\n",
    "encoded_sentences = tokenizer.texts_to_sequences(sentences)\n",
    "print(encoded_sentences[:5])\n",
    "\n",
    "VOCAB_SIZE = len(tokenizer.word_index) + 1\n",
    "print(VOCAB_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving\n",
    "with open(MODELS_PATH + 'combined_tokenizer_' + str(RUN_INDEX) + '.pickle', 'wb') as handle:\n",
    "    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total training data size =  833131\n",
      "Max seq len =  13\n",
      "(833131, 13)\n",
      "[[   1  217   23   25   36  256 3028   73  159  138    6    4   23]\n",
      " [ 217   23   25   36  256 3028   73  159  138    6    4   23   31]]\n",
      "(833131, 1)\n",
      "[[ 31]\n",
      " [135]]\n"
     ]
    }
   ],
   "source": [
    "X_data = []\n",
    "y_data = []\n",
    "for sentence in encoded_sentences:\n",
    "    l = len(sentence)\n",
    "    sliding_window_length = min(l-3, MAX_SEQUENCE_LENGTH)\n",
    "    step_size = 1\n",
    "    for i in range(0, l - sliding_window_length, step_size):\n",
    "        X_data.append(sentence[i:i+sliding_window_length])\n",
    "        y_data.append(sentence[i+sliding_window_length])\n",
    "        \n",
    "print(\"Total training data size = \", len(X_data))\n",
    "MAX_SEQ_LEN = max([len(seq) for seq in X_data])\n",
    "print(\"Max seq len = \", MAX_SEQ_LEN)\n",
    "X_data = pad_sequences(X_data, maxlen=MAX_SEQ_LEN, padding='pre')\n",
    "y_data = np.array(y_data).reshape(-1,1)\n",
    "print(X_data.shape)\n",
    "print(X_data[:2])\n",
    "print(y_data.shape)\n",
    "print(y_data[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing word vectors\n",
      "Total 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "print('Indexing word vectors')\n",
    "#Glove Vectors\n",
    "embeddings_index = {}\n",
    "f = open(EMBEDDING_FILE)\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Total %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing embedding matrix\n",
      "Null word embeddings: 298\n"
     ]
    }
   ],
   "source": [
    "print('Preparing embedding matrix')\n",
    "embedding_matrix = np.zeros((VOCAB_SIZE, EMBEDDING_DIM))\n",
    "for word,i in word_index.items():\n",
    "    if i >= MAX_NB_WORDS:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "print('Null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1) == 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define model\n",
    "def BiLSTM(vocab_size, embedding_dim, hidden1_dim,hidden_dim2,hidden_dim3 max_seq_len, \n",
    "           tag_dim,dropout_factor=0.2,regularization=0.00001):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Embedding(vocab_size,embedding_dim,input_length=max_seq_len,weights=[embedding_matrix],mask_zero=True,trainable=False))\n",
    "    tag_input = Input(shape=(tag_dim,), name='aux_input')\n",
    "    model.add(Concatenate([model, tag_input]))\n",
    "    model.add(LSTM(hidden_dim1, \n",
    "                                 activation='tanh',\n",
    "                                 kernel_regularizer=regularizers.l2(regularization),\n",
    "                                 recurrent_regularizer=regularizers.l2(regularization), unroll=True, return_sequences = True\n",
    "                                ))\n",
    "    model.add(LSTM(hidden_dim2, activation='tanh', \n",
    "                   kernel_regularizer=regularizers.l2(regularization), \n",
    "                   recurrent_regularizer=regularizers.l2(regularization), #unroll=True, \n",
    "                   return_sequences = True, dropout=dropout_factor, recurrent_dropout=dropout_factor))\n",
    "    \n",
    "     model.add(LSTM(hidden_dim3, activation='tanh', \n",
    "                   kernel_regularizer=regularizers.l2(regularization), \n",
    "                   recurrent_regularizer=regularizers.l2(regularization), #unroll=True, \n",
    "                   return_sequences = True, dropout=dropout_factor, recurrent_dropout=dropout_factor))\n",
    "    \n",
    "    \n",
    "    model.add(Dropout(dropout_factor))\n",
    "    model.add(Attention(MAX_SEQUENCE_LENGTH))\n",
    "    model.add(Dense(units=vocab_size, activation='softmax', \n",
    "              kernel_regularizer=regularizers.l2(regularization)))\n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', \n",
    "                  metrics=[sparse_categorical_crossentropy, sparse_categorical_accuracy])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "A `Concatenate` layer should be called on a list of at least 2 inputs",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-36-f6dacfda0e63>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m model = BiLSTM(vocab_size=VOCAB_SIZE, embedding_dim=EMBEDDING_DIM, hidden_dim=HIDDEN_DIM, deeper_dim=DEEPER_DIM,\n\u001b[0;32m----> 2\u001b[0;31m               max_seq_len=MAX_SEQ_LEN,tag_dim=TAG_DIM,dropout_factor=DROPOUT_FACTOR, regularization=REGULARIZATION)\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msummary\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-35-2e007224fc80>\u001b[0m in \u001b[0;36mBiLSTM\u001b[0;34m(vocab_size, embedding_dim, hidden_dim, deeper_dim, max_seq_len, tag_dim, dropout_factor, regularization)\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEmbedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvocab_size\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0membedding_dim\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0minput_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmax_seq_len\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mweights\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0membedding_matrix\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mmask_zero\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mtrainable\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0mtag_input\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mInput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtag_dim\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'aux_input'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mConcatenate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtag_input\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m     model.add(Bidirectional(LSTM(hidden_dim, \n\u001b[1;32m      9\u001b[0m                                  \u001b[0mactivation\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'tanh'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/keras/models.py\u001b[0m in \u001b[0;36madd\u001b[0;34m(self, layer)\u001b[0m\n\u001b[1;32m    490\u001b[0m                           output_shapes=[self.outputs[0]._keras_shape])\n\u001b[1;32m    491\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 492\u001b[0;31m             \u001b[0moutput_tensor\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlayer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    493\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_tensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    494\u001b[0m                 raise TypeError('All layers in a Sequential model '\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/keras/engine/topology.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs, **kwargs)\u001b[0m\n\u001b[1;32m    588\u001b[0m                                          '`layer.build(batch_input_shape)`')\n\u001b[1;32m    589\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shapes\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 590\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shapes\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    591\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    592\u001b[0m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuild\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shapes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/merge.py\u001b[0m in \u001b[0;36mbuild\u001b[0;34m(self, input_shape)\u001b[0m\n\u001b[1;32m    339\u001b[0m         \u001b[0;31m# Used purely for shape validation.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    340\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_shape\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 341\u001b[0;31m             raise ValueError('A `Concatenate` layer should be called '\n\u001b[0m\u001b[1;32m    342\u001b[0m                              'on a list of at least 2 inputs')\n\u001b[1;32m    343\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mshape\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mshape\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: A `Concatenate` layer should be called on a list of at least 2 inputs"
     ]
    }
   ],
   "source": [
    "model = BiLSTM(vocab_size=VOCAB_SIZE, embedding_dim=EMBEDDING_DIM, hidden_dim1=512,hidden_dim2=256,hidden_dim3=128,\n",
    "              max_seq_len=MAX_SEQ_LEN,tag_dim=TAG_DIM,dropout_factor=DROPOUT_FACTOR, regularization=REGULARIZATION)\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TB(TensorBoard):\n",
    "    def __init__(self, log_every=1, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.log_every = log_every\n",
    "        self.counter = 0\n",
    "    \n",
    "    def on_batch_end(self, batch, logs=None):\n",
    "        self.counter+=1\n",
    "        if self.counter%self.log_every==0:\n",
    "            for name, value in logs.items():\n",
    "                if name in ['batch', 'size']:\n",
    "                    continue\n",
    "                summary = tf.Summary()\n",
    "                summary_value = summary.value.add()\n",
    "                summary_value.simple_value = value.item()\n",
    "                summary_value.tag = name\n",
    "                self.writer.add_summary(summary, self.counter)\n",
    "            self.writer.flush()\n",
    "        \n",
    "        super().on_batch_end(batch, logs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 666504 samples, validate on 166627 samples\n",
      "Epoch 1/50\n",
      " 81920/666504 [==>...........................] - ETA: 2:11 - loss: 5.1130 - sparse_categorical_crossentropy: 5.0824 - sparse_categorical_accuracy: 0.2048"
     ]
    }
   ],
   "source": [
    "start_time = time()\n",
    "tensorboard = TB(log_dir=\"./logs/jokes_bilstm/{}\".format(time()), \n",
    "                          histogram_freq=0, write_graph=True, write_images=False, log_every=10)\n",
    "\n",
    "callbacks=[tensorboard, \n",
    "           EarlyStopping(patience=5, monitor='val_loss'),\n",
    "           ModelCheckpoint(filepath=MODELS_PATH + 'checkpoints/jokes_bilstm_gen'+str(RUN_INDEX)+'.{epoch:02d}-{val_loss:.2f}.hdf5', \n",
    "                           monitor='val_loss', verbose=1, mode='auto', period=1), \n",
    "           ModelCheckpoint(filepath=MODELS_PATH + 'jokes_bilstm_gen'+str(RUN_INDEX)+'.hdf5', \n",
    "                           monitor='val_loss', verbose=1, mode='auto', period=1, save_best_only=True)]\n",
    "\n",
    "model.fit(X_data, y_data, epochs=50, batch_size=2048, shuffle=True, verbose=1, validation_split=0.2, \n",
    "          callbacks=callbacks)\n",
    "\n",
    "print(\"Total elapsed time: \",time()-start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate a sequence from a language model\n",
    "def generate(model, tokenizer, seed_text, maxlen):\n",
    "    \n",
    "    reverse_word_map = dict(map(reversed, tokenizer.word_index.items()))\n",
    "    seq = tokenizer.texts_to_sequences([seed_text])[0]\n",
    "    print(seq)\n",
    "    \n",
    "    while True:\n",
    "        if len(seq) > MAX_SEQ_LEN:\n",
    "            encoded_seq = seq[-1*MAX_SEQ_LEN:]\n",
    "        else:\n",
    "            encoded_seq = seq\n",
    "        padded_seq = pad_sequences([encoded_seq], maxlen=MAX_SEQ_LEN, padding='pre')\n",
    "        #padded_seq = np.array([seq])\n",
    "        y_prob = model.predict(padded_seq)\n",
    "        #y_class = y_prob.argmax(axis=-1)[0]\n",
    "        y_class_max = y_prob.argmax(axis=-1)[0]\n",
    "        y_class_sample = np.argmax(np.random.multinomial(1,y_prob.squeeze(axis=0),1))\n",
    "        y_class = y_class_sample\n",
    "        if y_class == 0:\n",
    "            break\n",
    "        out_word = reverse_word_map[y_class]\n",
    "        seq.append(y_class)\n",
    "        if out_word == 'eos' or len(seq) > maxlen:\n",
    "            break\n",
    "    words = [reverse_word_map[idx] for idx in seq]\n",
    "    return ' '.join(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "model1 =  load_model('models/checkpoints/jokes_bilstm_gen2.08-4.39.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-b5f7d2b53109>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mjoke\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgenerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"sos a guy finds\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmaxlen\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m40\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mjoke\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "joke = generate(model, tokenizer, \"sos a guy finds\", maxlen=40)\n",
    "print(joke)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Document Similarity "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 159, 92]\n",
      "sos before we ipad most dyslexics tang blankets beyonce sings restroom hugs presidency vision frame opinion daily itch monkeys care daughter's harry quietly burns archaeologists glove recipe bruno honour minor wimbledon insensitive answers jewish teenagers motherfucking rousey fowl judged marvel la\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'sos phew phew phew phew phew phew phew phew phew phew phew phew phew phew phew phew phew phew phew phew phew phew the chosen phew eos'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_distances\n",
    "joke = generate(model, tokenizer,' '.join(sentences[12].split()[:3]), maxlen=40)\n",
    "print(joke)\n",
    "tfidf_vectorizer = TfidfVectorizer()\n",
    "X_tf_idf_data = tfidf_vectorizer.fit_transform(sentences)\n",
    "joke_vector = tfidf_vectorizer.transform([joke])\n",
    "d = cosine_distances(X_tf_idf_data,joke_vector)\n",
    "sentences[np.argmax(d)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}