Jokes-Quotes-Tweets-Controlled-Word-Stacked-LSTM-Keras.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train a bilstm to generate jokes in forward and reverse based on a controlled bit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import importlib\n",
    "from library import data_preprocess as dp\n",
    "importlib.reload(dp)\n",
    "import random\n",
    "from time import time\n",
    "\n",
    "import numpy as np\n",
    "import keras\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils import to_categorical\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential, Input\n",
    "from keras.layers import Dense, Dropout, Activation\n",
    "from keras.layers import LSTM, Bidirectional\n",
    "from keras.layers import Embedding, TimeDistributed, Flatten, Merge, Concatenate\n",
    "from keras import regularizers\n",
    "from keras.metrics import sparse_categorical_accuracy, sparse_categorical_crossentropy\n",
    "from keras.models import load_model\n",
    "from keras.optimizers import Adam\n",
    "from keras.models import Model\n",
    "\n",
    "from keras.callbacks import TensorBoard, EarlyStopping, ModelCheckpoint\n",
    "import tensorflow as tf\n",
    "from keras import backend as K\n",
    "from keras.utils import multi_gpu_model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# http://nlp.stanford.edu/data/glove.6B.zip\n",
    "DATA_PATH = './datasets/combined.pickle'\n",
    "VOCAB_PATH = './datasets/combined_vocabulary.pickle'\n",
    "MODELS_PATH = './models/'\n",
    "\n",
    "GLOVE_PATH = './data/glove/glove.6B.200d.txt'\n",
    "\n",
    "MODEL_PREFIX = 'combined_controlled_stacked_lstm_glove'\n",
    "\n",
    "MAX_SEQUENCE_LENGTH = 13\n",
    "VALIDATION_SPLIT = 0.2\n",
    "\n",
    "GLOVE_EMBEDDING_DIM = 200\n",
    "EMBEDDING_DIM1 = 512\n",
    "EMBEDDING_DIM2 = 512\n",
    "HIDDEN_DIM1 = 1024\n",
    "HIDDEN_DIM2 = 512\n",
    "DEEPER_DIM = 512\n",
    "DROPOUT_FACTOR = 0.2\n",
    "REGULARIZATION = 0.00001\n",
    "LEARNING_RATE = 0.003\n",
    "\n",
    "DATA_PERCENT = 0.1\n",
    "\n",
    "RUN_INDEX = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of sentences =  270543\n",
      "[[0, 'sos hillary and trump crash in a plane , who survives ? the united states eos'], [0, 'sos keep your friends close , and your enemies closer . marry your enemy . grow old together . watch your enemy die . eos']]\n",
      "Vocab size =  12614\n",
      "['liberate', 'savings', 'clip', 'rhino', 'cross', 'encouragement', 'viewed', 'womens', 'lap', 'challenges']\n"
     ]
    }
   ],
   "source": [
    "with open(DATA_PATH, 'rb') as pickleFile:\n",
    "    sentences = pickle.load(pickleFile)\n",
    "\n",
    "with open(VOCAB_PATH, 'rb') as pickleFile:\n",
    "    vocab = pickle.load(pickleFile)\n",
    "    \n",
    "random.shuffle(sentences)\n",
    "\n",
    "print(\"Number of sentences = \", len(sentences))\n",
    "print(sentences[:2])\n",
    "print(\"Vocab size = \", len(vocab))\n",
    "print(vocab[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of Jokes:  96910\n",
      "Number of Quotes:  43383\n",
      "Number of Tweets:  130250\n",
      "JOKES_WEIGHT:  2.7916933237024044\n",
      "QUOTES_WEIGHT:  6.236152409930157\n",
      "TWEETS_WEIGHT:  2.07710556621881\n"
     ]
    }
   ],
   "source": [
    "NUM_JOKES = len([0 for item in sentences if item[0] == 0])\n",
    "NUM_QUOTES = len([1 for item in sentences if item[0] == 1])\n",
    "NUM_TWEETS = len([2 for item in sentences if item[0] == 2])\n",
    "print(\"Number of Jokes: \", NUM_JOKES)\n",
    "print(\"Number of Quotes: \", NUM_QUOTES)\n",
    "print(\"Number of Tweets: \", NUM_TWEETS)\n",
    "TOTAL_SENTENCES = NUM_JOKES + NUM_QUOTES + NUM_TWEETS\n",
    "JOKES_WEIGHT = TOTAL_SENTENCES/NUM_JOKES\n",
    "QUOTES_WEIGHT = TOTAL_SENTENCES/NUM_QUOTES\n",
    "TWEETS_WEIGHT = TOTAL_SENTENCES/NUM_TWEETS\n",
    "print(\"JOKES_WEIGHT: \", JOKES_WEIGHT)\n",
    "print(\"QUOTES_WEIGHT: \", QUOTES_WEIGHT)\n",
    "print(\"TWEETS_WEIGHT: \", TWEETS_WEIGHT)\n",
    "\n",
    "CLASS_WEIGHTS = [JOKES_WEIGHT, QUOTES_WEIGHT, TWEETS_WEIGHT]\n",
    "CLASSES = [item[0] for item in sentences]\n",
    "sentences = [item[1] for item in sentences]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1, 1549, 9, 641, 2013, 17, 7, 824, 8, 83, 5318, 13, 4, 1466, 1272, 2], [1, 248, 54, 184, 661, 8, 9, 54, 2597, 1795, 3, 1667, 54, 2159, 3, 866, 244, 514, 3, 286, 54, 2159, 442, 3, 2], [1, 152, 36, 59, 2073, 41, 59, 19, 426, 16, 407, 2], [1, 90, 5, 75, 6, 52, 6, 81, 90, 11, 4, 157, 640, 17, 565, 3, 179, 332, 18, 74, 55, 24, 8, 170, 255, 5, 291, 17, 7336, 8, 44, 4, 67, 7007, 2], [1, 5, 32, 352, 7, 129, 4081, 1194, 56, 51, 4573, 38, 1117, 514, 3, 18, 34, 9623, 16, 2]]\n",
      "12615\n"
     ]
    }
   ],
   "source": [
    "# tokenize data\n",
    "num_words = len(vocab)\n",
    "\n",
    "with open(\"../combined_controlled_stacked_lstm_glove_tokenizer_1.pickle\",\"rb\") as fip:\n",
    "    tokenizer = pickle.load(fip)\n",
    "# tokenizer = Tokenizer(num_words=None, filters='', lower=True, split=' ', \n",
    "#                       char_level=False, oov_token=None)\n",
    "tokenizer.fit_on_texts(sentences)\n",
    "assert num_words == len(tokenizer.word_index)\n",
    "\n",
    "encoded_sentences = tokenizer.texts_to_sequences(sentences)\n",
    "print(encoded_sentences[:5])\n",
    "\n",
    "VOCAB_SIZE = len(tokenizer.word_index) + 1\n",
    "print(VOCAB_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# saving\n",
    "with open(MODELS_PATH + MODEL_PREFIX + '_tokenizer_' + str(RUN_INDEX) + '.pickle', 'wb') as handle:\n",
    "    pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing Training Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total training data size =  2244907\n",
      "Max seq len =  13\n",
      "(2244907, 13)\n",
      "[[  14  213  285   24    4  324    8   28   18  310  297   49  153]\n",
      " [   0    0    0    0   36   91  207  128  682   16   33   61   16]\n",
      " [   9    7 1085  865  493   13    7 1085  865  493   60 1421    7]\n",
      " [  21    6  123  452   15   10  214 7845  388    5  154  132  663]\n",
      " [2712    8 9635   20    5  105   73  163   92  131  927   39 1884]]\n",
      "(2244907, 13, 1)\n",
      "[[[ 213]\n",
      "  [ 285]\n",
      "  [  24]\n",
      "  [   4]\n",
      "  [ 324]\n",
      "  [   8]\n",
      "  [  28]\n",
      "  [  18]\n",
      "  [ 310]\n",
      "  [ 297]\n",
      "  [  49]\n",
      "  [ 153]\n",
      "  [ 862]]\n",
      "\n",
      " [[   0]\n",
      "  [   0]\n",
      "  [   0]\n",
      "  [   0]\n",
      "  [  91]\n",
      "  [ 207]\n",
      "  [ 128]\n",
      "  [ 682]\n",
      "  [  16]\n",
      "  [  33]\n",
      "  [  61]\n",
      "  [  16]\n",
      "  [   2]]\n",
      "\n",
      " [[   7]\n",
      "  [1085]\n",
      "  [ 865]\n",
      "  [ 493]\n",
      "  [  13]\n",
      "  [   7]\n",
      "  [1085]\n",
      "  [ 865]\n",
      "  [ 493]\n",
      "  [  60]\n",
      "  [1421]\n",
      "  [   7]\n",
      "  [ 240]]\n",
      "\n",
      " [[   6]\n",
      "  [ 123]\n",
      "  [ 452]\n",
      "  [  15]\n",
      "  [  10]\n",
      "  [ 214]\n",
      "  [7845]\n",
      "  [ 388]\n",
      "  [   5]\n",
      "  [ 154]\n",
      "  [ 132]\n",
      "  [ 663]\n",
      "  [  24]]\n",
      "\n",
      " [[   8]\n",
      "  [9635]\n",
      "  [  20]\n",
      "  [   5]\n",
      "  [ 105]\n",
      "  [  73]\n",
      "  [ 163]\n",
      "  [  92]\n",
      "  [ 131]\n",
      "  [ 927]\n",
      "  [  39]\n",
      "  [1884]\n",
      "  [ 191]]]\n",
      "(2244907, 13, 3)\n",
      "[[[1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]]\n",
      "\n",
      " [[0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]]\n",
      "\n",
      " [[1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]\n",
      "  [1. 0. 0.]]\n",
      "\n",
      " [[0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]\n",
      "  [0. 0. 1.]]\n",
      "\n",
      " [[0. 1. 0.]\n",
      "  [0. 1. 0.]\n",
      "  [0. 1. 0.]\n",
      "  [0. 1. 0.]\n",
      "  [0. 1. 0.]\n",
      "  [0. 1. 0.]\n",
      "  [0. 1. 0.]\n",
      "  [0. 1. 0.]\n",
      "  [0. 1. 0.]\n",
      "  [0. 1. 0.]\n",
      "  [0. 1. 0.]\n",
      "  [0. 1. 0.]\n",
      "  [0. 1. 0.]]]\n",
      "(2244907,)\n",
      "[2.79169332 2.07710557 2.79169332 2.07710557 6.23615241]\n"
     ]
    }
   ],
   "source": [
    "TAG_SIZE = 3\n",
    "X_data = []\n",
    "y_data = []\n",
    "tag_data = []\n",
    "weight_data = []\n",
    "\n",
    "for idx, sentence in enumerate(encoded_sentences):\n",
    "    l = len(sentence)\n",
    "    sliding_window_length = min(l-3, MAX_SEQUENCE_LENGTH)\n",
    "    step_size = 1\n",
    "    for i in range(0, l - sliding_window_length, step_size):\n",
    "        X_data.append(sentence[i:i+sliding_window_length])\n",
    "        y_data.append(sentence[i+1:i+sliding_window_length+1])\n",
    "        tag_data.append(CLASSES[idx])\n",
    "        weight_data.append(CLASS_WEIGHTS[CLASSES[idx]])\n",
    "        \n",
    "print(\"Total training data size = \", len(X_data))\n",
    "MAX_SEQ_LEN = max([len(seq) for seq in X_data])\n",
    "print(\"Max seq len = \", MAX_SEQ_LEN)\n",
    "\n",
    "X_data = pad_sequences(X_data, maxlen=MAX_SEQ_LEN, padding='pre')\n",
    "y_data = pad_sequences(y_data, maxlen=MAX_SEQ_LEN, padding='pre').reshape(-1, MAX_SEQ_LEN, 1)\n",
    "tag_data = to_categorical([[[tag]]*MAX_SEQ_LEN for tag in tag_data], TAG_SIZE)\n",
    "weight_data = np.array(weight_data).reshape(-1,)\n",
    "#y_data = np.array(y_data).reshape(-1,1)\n",
    "\n",
    "# shuffle\n",
    "perm = np.random.permutation(X_data.shape[0])\n",
    "X_data = X_data[perm]\n",
    "y_data = y_data[perm]\n",
    "tag_data = tag_data[perm]\n",
    "weight_data = weight_data[perm]\n",
    "print(X_data.shape)\n",
    "print(X_data[:5])\n",
    "print(y_data.shape)\n",
    "print(y_data[:5])\n",
    "print(tag_data.shape)\n",
    "print(tag_data[:5])\n",
    "print(weight_data.shape)\n",
    "print(weight_data[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing glove word vectors\n",
      "Total 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "print('Indexing glove word vectors')\n",
    "#Glove Vectors\n",
    "glove_embeddings_index = {}\n",
    "f = open(GLOVE_PATH)\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    glove_embeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Total %s word vectors.' % len(glove_embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing glove embedding matrix\n",
      "Null word embeddings: 298\n",
      "(12615, 200)\n"
     ]
    }
   ],
   "source": [
    "print('Preparing glove embedding matrix')\n",
    "glove_embedding_matrix = np.zeros((VOCAB_SIZE, GLOVE_EMBEDDING_DIM))\n",
    "for word,i in tokenizer.word_index.items():\n",
    "    embedding_vector = glove_embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        glove_embedding_matrix[i] = embedding_vector\n",
    "print('Null word embeddings: %d' % np.sum(np.sum(glove_embedding_matrix, axis=1) == 0))\n",
    "print(glove_embedding_matrix.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import merge\n",
    "from keras.layers.core import Lambda\n",
    "from keras.models import Model\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "def make_parallel(model, gpu_count):\n",
    "    def get_slice(data, idx, parts):\n",
    "        shape = tf.shape(data)\n",
    "        size = tf.concat([ shape[:1] // parts, shape[1:] ],axis=0)\n",
    "        stride = tf.concat([ shape[:1] // parts, shape[1:]*0 ],axis=0)\n",
    "        start = stride * idx\n",
    "        return tf.slice(data, start, size)\n",
    "\n",
    "    outputs_all = []\n",
    "    for i in range(len(model.outputs)):\n",
    "        outputs_all.append([])\n",
    "\n",
    "    #Place a copy of the model on each GPU, each getting a slice of the batch\n",
    "    for i in range(gpu_count):\n",
    "        with tf.device('/gpu:%d' % i):\n",
    "            with tf.name_scope('tower_%d' % i) as scope:\n",
    "\n",
    "                inputs = []\n",
    "                #Slice each input into a piece for processing on this GPU\n",
    "                for x in model.inputs:\n",
    "                    input_shape = tuple(x.get_shape().as_list())[1:]\n",
    "                    slice_n = Lambda(get_slice, output_shape=input_shape, arguments={'idx':i,'parts':gpu_count})(x)\n",
    "                    inputs.append(slice_n)                \n",
    "\n",
    "                outputs = model(inputs)\n",
    "                \n",
    "                if not isinstance(outputs, list):\n",
    "                    outputs = [outputs]\n",
    "                \n",
    "                #Save all the outputs for merging back together later\n",
    "                for l in range(len(outputs)):\n",
    "                    outputs_all[l].append(outputs[l])\n",
    "\n",
    "    # merge outputs on CPU\n",
    "    with tf.device('/cpu:0'):\n",
    "        merged = []\n",
    "        for outputs in outputs_all:\n",
    "            merged.append(merge(outputs, mode='concat', concat_axis=0))\n",
    "            \n",
    "        return Model(input=model.inputs, output=merged)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define model\n",
    "def StackedLSTM(vocab_size, glove_embedding_dim, glove_embedding_matrix, embedding_dim1, embedding_dim2,\n",
    "           hidden_dim1, hidden_dim2, deeper_dim, max_seq_len, tag_size,\n",
    "           dropout_factor=0.5, regularization=0.00001, learning_rate=0.001):\n",
    "    \n",
    "    inputs = Input(shape=(None,))\n",
    "    tag_inputs = Input(shape=(None,tag_size))\n",
    "    \n",
    "    glove_embedding = Embedding(vocab_size, glove_embedding_dim, #input_length=max_seq_len,\n",
    "                                  weights=[glove_embedding_matrix],\n",
    "                                  mask_zero=True,trainable=False)(inputs)\n",
    "    \n",
    "    word_embedding = Embedding(vocab_size, embedding_dim1, #input_length=max_seq_len, \n",
    "                               mask_zero=True, embeddings_regularizer=regularizers.l2(regularization))(inputs)\n",
    "    \n",
    "    #tag_embedding = Embedding(tag_size, tag_size, embeddings_regularizer=regularizers.l2(regularization))(tag_inputs)\n",
    "    \n",
    "    concat_embeds = Concatenate(axis=-1)([glove_embedding, word_embedding, tag_inputs])\n",
    "    \n",
    "    final_embed = Dense(units=embedding_dim2, activation='tanh',\n",
    "                        kernel_regularizer=regularizers.l2(regularization))(concat_embeds)\n",
    "    \n",
    "    lstm1 = LSTM(hidden_dim1, activation='tanh', \n",
    "                   kernel_regularizer=regularizers.l2(regularization), \n",
    "                   recurrent_regularizer=regularizers.l2(regularization), #unroll=True, \n",
    "                   return_sequences = True, dropout=dropout_factor, recurrent_dropout=dropout_factor)(final_embed)\n",
    "    \n",
    "    lstm2 = LSTM(hidden_dim2, activation='tanh', \n",
    "                   kernel_regularizer=regularizers.l2(regularization), \n",
    "                   recurrent_regularizer=regularizers.l2(regularization), #unroll=True, \n",
    "                   return_sequences = True, dropout=dropout_factor, recurrent_dropout=dropout_factor)(lstm1)\n",
    "    \n",
    "    timedist_dropout = TimeDistributed(Dropout(dropout_factor))(lstm2)\n",
    "    \n",
    "    deep_dense = Dense(units=deeper_dim, activation='tanh', \n",
    "                       kernel_regularizer=regularizers.l2(regularization))(timedist_dropout)\n",
    "    \n",
    "    dropout_layer1 = Dropout(dropout_factor)(deep_dense)\n",
    "    \n",
    "    outputs = Dense(units=vocab_size, activation='softmax', \n",
    "                    kernel_regularizer=regularizers.l2(regularization))(dropout_layer1)\n",
    "    \n",
    "    model = Model(inputs=[inputs,tag_inputs], outputs=outputs)\n",
    "    #model = make_parallel(Model(inputs=[inputs,tag_inputs], outputs=outputs), 2)\n",
    "    #model = multi_gpu_model(model, gpus=2)\n",
    "    model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam(lr=learning_rate),\n",
    "                  metrics=[sparse_categorical_crossentropy, sparse_categorical_accuracy]#, sample_weight_mode='temporal'\n",
    "                 )\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, None)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_1 (Embedding)         (None, None, 200)    2523000     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, None, 512)    6458880     input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, None, 3)      0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, None, 715)    0           embedding_1[0][0]                \n",
      "                                                                 embedding_2[0][0]                \n",
      "                                                                 input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, None, 512)    366592      concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, None, 1024)   6295552     dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                   (None, None, 512)    3147776     lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_1 (TimeDistrib (None, None, 512)    0           lstm_2[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, None, 512)    262656      time_distributed_1[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, None, 512)    0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, None, 12615)  6471495     dropout_2[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 25,525,951\n",
      "Trainable params: 23,002,951\n",
      "Non-trainable params: 2,523,000\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "K.clear_session()\n",
    "config = tf.ConfigProto(allow_soft_placement=True)\n",
    "sess = tf.Session(config=config)\n",
    "K.set_session(sess)\n",
    "\n",
    "model = StackedLSTM(vocab_size=VOCAB_SIZE, glove_embedding_dim=GLOVE_EMBEDDING_DIM,\n",
    "                    glove_embedding_matrix=glove_embedding_matrix, \n",
    "                    embedding_dim1=EMBEDDING_DIM1, embedding_dim2=EMBEDDING_DIM2,\n",
    "                    hidden_dim1=HIDDEN_DIM1, hidden_dim2=HIDDEN_DIM2,\n",
    "                    deeper_dim=DEEPER_DIM, max_seq_len=MAX_SEQ_LEN, dropout_factor=DROPOUT_FACTOR, \n",
    "                    regularization=REGULARIZATION, learning_rate=LEARNING_RATE, tag_size=TAG_SIZE)\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TB(TensorBoard):\n",
    "    def __init__(self, log_every=1, **kwargs):\n",
    "        super().__init__(**kwargs)\n",
    "        self.log_every = log_every\n",
    "        self.counter = 0\n",
    "    \n",
    "    def on_batch_end(self, batch, logs=None):\n",
    "        self.counter+=1\n",
    "        if self.counter%self.log_every==0:\n",
    "            for name, value in logs.items():\n",
    "                if name in ['batch', 'size']:\n",
    "                    continue\n",
    "                summary = tf.Summary()\n",
    "                summary_value = summary.value.add()\n",
    "                summary_value.simple_value = value.item()\n",
    "                summary_value.tag = name\n",
    "                self.writer.add_summary(summary, self.counter)\n",
    "            self.writer.flush()\n",
    "        \n",
    "        super().on_batch_end(batch, logs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1795925 samples, validate on 448982 samples\n",
      "Epoch 1/10\n",
      "1795925/1795925 [==============================] - 2751s 2ms/step - loss: 18.5659 - sparse_categorical_crossentropy: 5.3421 - sparse_categorical_accuracy: 0.1383 - val_loss: 15.3502 - val_sparse_categorical_crossentropy: 4.2757 - val_sparse_categorical_accuracy: 0.2304\n",
      "\n",
      "Epoch 00001: saving model to ./models/checkpoints/combined_controlled_stacked_lstm_glove_gen1.01-15.35.hdf5\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 15.35020, saving model to ./models/combined_controlled_stacked_lstm_glove_gen1.hdf5\n",
      "Epoch 2/10\n",
      "1795925/1795925 [==============================] - 2749s 2ms/step - loss: 15.5209 - sparse_categorical_crossentropy: 4.2997 - sparse_categorical_accuracy: 0.2262 - val_loss: 14.6837 - val_sparse_categorical_crossentropy: 4.0368 - val_sparse_categorical_accuracy: 0.2536\n",
      "\n",
      "Epoch 00002: saving model to ./models/checkpoints/combined_controlled_stacked_lstm_glove_gen1.02-14.68.hdf5\n",
      "\n",
      "Epoch 00002: val_loss improved from 15.35020 to 14.68375, saving model to ./models/combined_controlled_stacked_lstm_glove_gen1.hdf5\n",
      "Epoch 3/10\n",
      "  10240/1795925 [..............................] - ETA: 42:08 - loss: 15.1677 - sparse_categorical_crossentropy: 4.1939 - sparse_categorical_accuracy: 0.2334"
     ]
    }
   ],
   "source": [
    "start_time = time()\n",
    "tensorboard = TB(log_dir=\"./logs/\" + MODEL_PREFIX + \"/{}\".format(time()), \n",
    "                          histogram_freq=0, write_graph=True, write_images=False, log_every=10)\n",
    "\n",
    "callbacks=[tensorboard, \n",
    "           EarlyStopping(patience=5, monitor='val_loss'),\n",
    "           ModelCheckpoint(filepath=MODELS_PATH + 'checkpoints/'+ MODEL_PREFIX + '_gen' + str(RUN_INDEX) + '.{epoch:02d}-{val_loss:.2f}.hdf5', \n",
    "                           monitor='val_loss', verbose=1, mode='auto', period=1), \n",
    "           ModelCheckpoint(filepath=MODELS_PATH + MODEL_PREFIX + '_gen'+str(RUN_INDEX)+'.hdf5', \n",
    "                           monitor='val_loss', verbose=1, mode='auto', period=1, save_best_only=True)]\n",
    "\n",
    "model.fit([X_data, tag_data], y_data, epochs=10, batch_size=1024, shuffle=True, \n",
    "          verbose=1, validation_split=0.2, callbacks=callbacks, sample_weight=weight_data)\n",
    "\n",
    "print(\"Total elapsed time: \", time()-start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model.save_weights(MODELS_PATH + 'checkpoints/'+ MODEL_PREFIX + '_gen' + str(RUN_INDEX)+'_epoch1.hdf5')\n",
    "model = load_model('../combined_controlled_stacked_lstm_glove_gen1.hdf5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate a sequence from a language model\n",
    "def generate_categorical(model, tokenizer, seed_text, maxlen, probabilistic=False, exploration_factor=1.0, tag=0):\n",
    "    \n",
    "    reverse_word_map = dict(map(reversed, tokenizer.word_index.items()))\n",
    "    seq = tokenizer.texts_to_sequences([seed_text])[0]\n",
    "    print(seq)\n",
    "    \n",
    "    while True:\n",
    "        encoded_seq = seq\n",
    "        if len(seq) > MAX_SEQUENCE_LENGTH:\n",
    "            encoded_seq = encoded_seq[-1*MAX_SEQUENCE_LENGTH:]\n",
    "            \n",
    "        #padded_seq = pad_sequences([encoded_seq], maxlen=MAX_SEQ_LEN, padding='pre')\n",
    "        padded_seq = np.array([seq])\n",
    "        tags = to_categorical(np.full((1, padded_seq[0].shape[0]), tag), TAG_SIZE)\n",
    "        y_prob = model.predict([padded_seq,tags])[0][-1].reshape(1,-1)#[3:].reshape(-1,1)\n",
    "        \n",
    "        if np.random.rand() <= exploration_factor:\n",
    "            probabilistic = True\n",
    "        else:\n",
    "            probabilistic = False\n",
    "            \n",
    "        if probabilistic:\n",
    "            y_class = np.argmax(np.random.multinomial(1,y_prob[0]/(np.sum(y_prob[0])+1e-5),1))\n",
    "        else:\n",
    "            y_class = y_prob.argmax(axis=-1)[0]\n",
    "        \n",
    "        if y_class == 0:\n",
    "            break\n",
    "        out_word = reverse_word_map[y_class]\n",
    "        seq.append(y_class)\n",
    "        if out_word == 'eos' or len(seq) > maxlen or out_word == 'sos':\n",
    "            break\n",
    "    \n",
    "    words = [reverse_word_map[idx] for idx in seq]\n",
    "    \n",
    "    return ' '.join(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 7, 397, 251, 4, 403]\n",
      "sos a doctor told the boy i was going to get appointment today . i miss him so much . eos\n"
     ]
    }
   ],
   "source": [
    "joke = generate_categorical(model, tokenizer, \"sos a doctor told the boy \", maxlen=40, tag=2, exploration_factor=0.3)\n",
    "print(joke)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100,)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random = np.random.randint(0,len(sentences),100)\n",
    "random.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sos hello', 'hello ,', \", i'm\", \"i'm a\", 'a dinosaur', 'dinosaur .', '. eos']\n",
      "['sos hello ,', \"hello , i'm\", \", i'm a\", \"i'm a dinosaur\", 'a dinosaur .', 'dinosaur . eos']\n",
      "[\"sos hello ,i'm\", \"hello , i'ma\", \", i'm adinosaur\", \"i'm a dinosaur.\", 'a dinosaur .eos']\n"
     ]
    }
   ],
   "source": [
    "def unigram_list(sentence):\n",
    "    return sentence.split(' ')\n",
    "def bigrams_list(sentence):\n",
    "    words = sentence.split(' ')\n",
    "    bigrams = []\n",
    "    for i in range(0, len(words)-1):\n",
    "        bigrams.append(words[i]+' '+words[i+1])\n",
    "    return bigrams\n",
    "\n",
    "print(bigrams_list(\"sos hello , i'm a dinosaur . eos\"))\n",
    "def trigrams_list(sentence):\n",
    "    words = sentence.split(' ')\n",
    "    trigrams = []\n",
    "    for i in range(0, len(words)-2):\n",
    "        trigrams.append(words[i]+' '+words[i+1]+' '+words[i+2])\n",
    "    return trigrams\n",
    "\n",
    "print(trigrams_list(\"sos hello , i'm a dinosaur . eos\"))\n",
    "def fourgrams_list(sentence):\n",
    "    words = sentence.split(' ')\n",
    "    fourgrams = []\n",
    "    for i in range(0, len(words)-3):\n",
    "        fourgrams.append(words[i]+' '+words[i+1]+' '+words[i+2]+words[i+3])\n",
    "    return fourgrams\n",
    "\n",
    "print(fourgrams_list(\"sos hello , i'm a dinosaur . eos\"))\n",
    "\n",
    "def phraseOverlap(sent1,sent2,flist):\n",
    "    score = 0\n",
    "    for i in range(0,len(flist),1):\n",
    "        l1 = flist[i](sent1)\n",
    "        l2 = set(flist[i](sent2))\n",
    "        for l in l1:\n",
    "            if l in l2:\n",
    "                #print(l)\n",
    "                score += (i+1)**2\n",
    "        #print(score)\n",
    "                \n",
    "    return np.tanh(float(score)/(len(sent1)+len(sent2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "flist = [unigram_list,bigrams_list,trigrams_list,fourgrams_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "8\n",
      "8\n",
      "8\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.21863508368712128"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "phraseOverlap(\"this is good stuff\",\"is this good stuff\",flist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sos what do you do when you lose your voice ? i don't know what to do threedots then hit me before it ever hurt eos ==> voice ? i don't know what to do threedots then hit me before it ever hurt eos\n",
      "sos what do you do when you lose your === sos what do you do when you lose your voice ? i don't know what to do threedots then hit me before it ever hurt eos\n",
      "0.7979789325668545 sos i don't know what to do threedots i love him eos\n",
      "sos i'm what you might call ' internet pretty ' threedots eos ==> ' threedots eos\n",
      "sos i'm what you might call ' internet pretty === sos i'm what you might call ' internet pretty ' threedots eos\n",
      "0.2744686218100928 sos i want to listen to something ' sexy ' threedots eos\n",
      "sos how many feminists does it take to change a lightbulb ? i am in new home . i fills it all alone eos ==> a lightbulb ? i am in new home . i fills it all alone eos\n",
      "sos how many feminists does it take to change === sos how many feminists does it take to change a lightbulb ? i am in new home . i fills it all alone eos\n",
      "0.3676596827481631 sos how many women does it take to change a lightbulb ? i don't care . you pick eos\n",
      "sos not ready for another week to begin threedots just not for any results ? two weeks of school . eos ==> not for any results ? two weeks of school . eos\n",
      "sos not ready for another week to begin threedots just === sos not ready for another week to begin threedots just not for any results ? two weeks of school . eos\n",
      "0.46607983322593444 sos i'm sick . i missed the last day of school . eos\n",
      "sos it's sad going to mcdonald's and finding out cheese shop left . i want a new face in bed ! eos ==> cheese shop left . i want a new face in bed ! eos\n",
      "sos it's sad going to mcdonald's and finding out === sos it's sad going to mcdonald's and finding out cheese shop left . i want a new face in bed ! eos\n",
      "0.6435154482335124 sos thank god camera fixed itself . i want a new ipod eos\n",
      "sos i really must be patient while wearing my makeup , you must finish the customer threedots eos ==> makeup , you must finish the customer threedots eos\n",
      "sos i really must be patient while wearing my === sos i really must be patient while wearing my makeup , you must finish the customer threedots eos\n",
      "0.20651066204432655 sos to live outside the law , you must be honest . eos\n",
      "sos jon and kate threedots so sad ! clearly , i have to wait on the phone again eos ==> , i have to wait on the phone again eos\n",
      "sos jon and kate threedots so sad ! clearly === sos jon and kate threedots so sad ! clearly , i have to wait on the phone again eos\n",
      "0.5716699660851173 sos ew , i have to mow the lawn bye ! eos\n",
      "sos why is a praying muslim like apple pie with ice cream ? nice smile . look for the one you love , and he is to the cheers that makes shoes seem people . eos ==> cream ? nice smile . look for the one you love , and he is to the cheers that makes shoes seem people . eos\n",
      "sos why is a praying muslim like apple pie with ice === sos why is a praying muslim like apple pie with ice cream ? nice smile . look for the one you love , and he is to the cheers that makes shoes seem people . eos\n",
      "0.44710899420164746 sos and if you can't be with the one you love , eat because you're bored . eos\n",
      "sos what's a pirate's favorite letter in the alphabet ? everyone is here on what i have once had done , i need it able to jump up the walls eos ==> is here on what i have once had done , i need it able to jump up the walls eos\n",
      "sos what's a pirate's favorite letter in the alphabet ? everyone === sos what's a pirate's favorite letter in the alphabet ? everyone is here on what i have once had done , i need it able to jump up the walls eos\n",
      "0.43022845308808944 sos i love change , i need it . eos\n",
      "sos well that was an unsuccessful walk threedots got to switch to experiences last night . and that's funny . eos ==> switch to experiences last night . and that's funny . eos\n",
      "sos well that was an unsuccessful walk threedots got to === sos well that was an unsuccessful walk threedots got to switch to experiences last night . and that's funny . eos\n",
      "0.46211715726000974 sos wow huge night last night . and huge headache . eos\n",
      "sos finally i have a week off from work threedots i'm coming round today threedots and i'm starting to miss jeff . eos ==> threedots i'm coming round today threedots and i'm starting to miss jeff . eos\n",
      "sos finally i have a week off from work === sos finally i have a week off from work threedots i'm coming round today threedots and i'm starting to miss jeff . eos\n",
      "0.4267846002900141 sos spanish homework again . for all day threedots and i'm starting to feel forgotten threedots not gonna lie . eos\n",
      "sos have you guys had the new lance armstrong trail ? as far as mine sits late again , i feel sick threedots eos ==> ? as far as mine sits late again , i feel sick threedots eos\n",
      "sos have you guys had the new lance armstrong trail === sos have you guys had the new lance armstrong trail ? as far as mine sits late again , i feel sick threedots eos\n",
      "0.7464600004158386 sos im hungry , im thirsty , i feel sick threedots eos\n",
      "sos sunday morning , and i'm awake and drinking red bull . i don't think i should get up early . eos ==> i don't think i should get up early . eos\n",
      "sos sunday morning , and i'm awake and drinking red bull . === sos sunday morning , and i'm awake and drinking red bull . i don't think i should get up early . eos\n",
      "0.7463685936680633 sos i hate when i have to get up early . eos\n",
      "sos i was at the track yesterday . i backed a horse at the eye council today . it was a flip toe . eos ==> the eye council today . it was a flip toe . eos\n",
      "sos i was at the track yesterday . i backed a horse at === sos i was at the track yesterday . i backed a horse at the eye council today . it was a flip toe . eos\n",
      "0.6496973265493158 sos saw monsters vs aliens today . it was a very good film . eos\n",
      "sos porn addiction is a serious thing threedots unfortunately i just have to work on my good site today again eos ==> threedots unfortunately i just have to work on my good site today again eos\n",
      "sos porn addiction is a serious thing === sos porn addiction is a serious thing threedots unfortunately i just have to work on my good site today again eos\n",
      "0.5653249579058751 sos ugh i have to work on my project after school eos\n",
      "sos talking from morning to night about sex has helped my skiing , because i have a pic for it searching me for the answer . eos ==> i have a pic for it searching me for the answer . eos\n",
      "sos talking from morning to night about sex has helped my skiing , because === sos talking from morning to night about sex has helped my skiing , because i have a pic for it searching me for the answer . eos\n",
      "0.5069533243726111 sos your mind will answer most questions if you learn to relax and wait for the answer . eos\n",
      "sos i need to be in a better frame earlier than i did by myself . i vending it in my plate . eos ==> earlier than i did by myself . i vending it in my plate . eos\n",
      "sos i need to be in a better frame === sos i need to be in a better frame earlier than i did by myself . i vending it in my plate . eos\n",
      "0.43133726987558463 sos movie night by myself . i just stepped on my cat eos\n",
      "sos i like my coffee like i used to bless to let them know threedots i look like a good best at it . eos ==> used to bless to let them know threedots i look like a good best at it . eos\n",
      "sos i like my coffee like i === sos i like my coffee like i used to bless to let them know threedots i look like a good best at it . eos\n",
      "0.5041049761167905 sos lesson learnt threedots i look like a rock star from kiss haha eos\n",
      "sos what did one saggy boob say to the other saggy boob ? if we don't have fruit eos ==> don't have fruit eos\n",
      "sos what did one saggy boob say to the other saggy boob ? if we === sos what did one saggy boob say to the other saggy boob ? if we don't have fruit eos\n",
      "0.13380810039119737 sos i don't have any style . eos\n",
      "sos a guy said threedots in india we need quantity not to think about school anyways . i don't want the fallen allergic bird eos ==> to think about school anyways . i don't want the fallen allergic bird eos\n",
      "sos a guy said threedots in india we need quantity not === sos a guy said threedots in india we need quantity not to think about school anyways . i don't want the fallen allergic bird eos\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4236843241877823 sos ugh . i don't want to go to work ! eos\n",
      "sos at home , watching nothing . i had so much fun with it all day again my poor little guy looks pressure . eos ==> i had so much fun with it all day again my poor little guy looks pressure . eos\n",
      "sos at home , watching nothing . === sos at home , watching nothing . i had so much fun with it all day again my poor little guy looks pressure . eos\n",
      "0.7039056039366212 sos i had so much fun with my friends today ! eos\n",
      "sos god has no phone , but i talk to him . he has no facebook , but he can see me not looking forward . eos ==> can see me not looking forward . eos\n",
      "sos god has no phone , but i talk to him . he has no facebook , but he === sos god has no phone , but i talk to him . he has no facebook , but he can see me not looking forward . eos\n",
      "0.3759622303493606 sos i'm up i have to go to graduation watch my friends leave me not looking forward to it eos\n",
      "sos what is a mattress ' but i can't draw it ! cat , dishes , and i don't wanna go walk door . eos ==> but i can't draw it ! cat , dishes , and i don't wanna go walk door . eos\n",
      "sos what is a mattress ' === sos what is a mattress ' but i can't draw it ! cat , dishes , and i don't wanna go walk door . eos\n",
      "0.7043941435269256 sos lots of missed calls , and i don't wanna go home eos\n",
      "sos autocorrect is horrible threedots i'm the one me i love these days eos ==> the one me i love these days eos\n",
      "sos autocorrect is horrible threedots i'm === sos autocorrect is horrible threedots i'm the one me i love these days eos\n",
      "0.3253888316994076 sos sooo , i spent most of my day sitting at home in front of the computer in my pajamas threedots i love these days . eos\n",
      "sos just my first professional make made it wonderful threedots haha , thanks too so i can continue with it mint a full day today eos ==> made it wonderful threedots haha , thanks too so i can continue with it mint a full day today eos\n",
      "sos just my first professional make === sos just my first professional make made it wonderful threedots haha , thanks too so i can continue with it mint a full day today eos\n",
      "0.32336787589333915 sos back from school , had a full day today so worn out haha eos\n",
      "sos you never can trust atoms threedots it's like a king of non goldfish ! now from threedots curiosity or genocide . eos ==> it's like a king of non goldfish ! now from threedots curiosity or genocide . eos\n",
      "sos you never can trust atoms threedots === sos you never can trust atoms threedots it's like a king of non goldfish ! now from threedots curiosity or genocide . eos\n",
      "0.1757865929497264 sos i'm sick of not training . it's like a piece of me is missing . eos\n",
      "sos my obsession with square roots has been split . i think i really really wish i could have it done . eos ==> been split . i think i really really wish i could have it done . eos\n",
      "sos my obsession with square roots has === sos my obsession with square roots has been split . i think i really really wish i could have it done . eos\n",
      "0.7321442233518527 sos i really really wish i could go to comic con eos\n",
      "sos feeling totally blah . but i'm not a successful person . i think i am going to cry . eos ==> i'm not a successful person . i think i am going to cry . eos\n",
      "sos feeling totally blah . but === sos feeling totally blah . but i'm not a successful person . i think i am going to cry . eos\n",
      "0.8232006455858831 sos i think i am going to cry happy tears tho , but sad too . eos\n",
      "sos wow im tweeting so much threedots sometimes i use my car to log anywhere . u wanted for it ! miss you eos ==> threedots sometimes i use my car to log anywhere . u wanted for it ! miss you eos\n",
      "sos wow im tweeting so much === sos wow im tweeting so much threedots sometimes i use my car to log anywhere . u wanted for it ! miss you eos\n",
      "0.35835739835078595 sos happy father's day . love you , papa ! miss you eos\n",
      "sos dreamt horrible dreams last night meaning i woke up earlier for a hug and all the rest of this occasion , i liked it eos ==> up earlier for a hug and all the rest of this occasion , i liked it eos\n",
      "sos dreamt horrible dreams last night meaning i woke === sos dreamt horrible dreams last night meaning i woke up earlier for a hug and all the rest of this occasion , i liked it eos\n",
      "0.5684035685874651 sos today wasnt as hot as i expected , i liked it eos\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.49642574464227296"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(np.array(score_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sos i love === sos i love the music that i love because i love it . i can't tell you how to live without it . eos\n",
      "0.668606699416691 sos i am always training because i love it . eos\n"
     ]
    }
   ],
   "source": [
    "joke = \"sos i love the music that i love because i love it . i can't tell you how to live without it . eos\"\n",
    "seed = \"sos i love\"\n",
    "#phraseOverlap(joke,sent)\n",
    "n = len(joke)\n",
    "score = 0\n",
    "print(seed,\"===\",joke)\n",
    "for sent in sentences:\n",
    "#         if sent.find(seed) != -1:\n",
    "#             continue\n",
    "    newscore = phraseOverlap(joke,sent,flist)\n",
    "    if newscore >= score:\n",
    "        score = newscore\n",
    "        match_joke = sent\n",
    "    \n",
    "    if newscore >= 0.9:\n",
    "        print(sent,phraseOverlap(joke,sent,flist))\n",
    "print(score,match_joke)\n",
    "score_list.append(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "def k_jaccard(sent1,sent2,k):\n",
    "    \n",
    "    w_1 = sent1.split()\n",
    "    w_2 = sent2.split()\n",
    "    \n",
    "    w_1_list = []\n",
    "    for i in range(len(w_1)-k+1):\n",
    "        w_1_list.append(' '.join(w_1[i:i+k]))\n",
    "        \n",
    "    w_2_list = []\n",
    "    for i in range(len(w_2)-k+1):\n",
    "        w_2_list.append(' '.join(w_2[i:i+k]))\n",
    "        \n",
    "    #print(w_1_list)\n",
    "    #print(w_2_list)\n",
    "    w_2_set = set(w_2_list)\n",
    "    w_1_set = set(w_1_list)\n",
    "    #print(len(w_2_set.union(w_1_set)),len(w_2_set.intersection(w_1_set)))\n",
    "    return len(w_1_set.intersection(w_2_set))*1.0/(len(w_2_set.union(w_1_set)))\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['sos what', 'what do', 'do you', 'you call', 'call a', 'a vegetarian', 'vegetarian ?', '? a', 'a rip', 'rip -', '- off', 'off .', '. eos']\n",
      "['sos what', 'what do', 'do you', 'you call', 'call a', 'a cheap', 'cheap circumcision', 'circumcision ?', '? a', 'a rip', 'rip -', '- off', 'off .', '. eos']\n",
      "16 11\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.6875"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k_jaccard('sos what do you call a vegetarian ? a rip - off . eos','sos what do you call a cheap circumcision ? a rip - off . eos', 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 7, 1234, 8, 63, 2969, 8, 9, 7, 386]\n",
      "[1, 31, 165, 166, 7981, 10, 427, 3197, 110]\n",
      "[1, 51, 172, 275, 38]\n",
      "[1, 5, 941, 7]\n",
      "[1, 250, 585, 39, 264]\n",
      "[1, 50, 38, 218, 66, 25, 61, 36, 1677, 13]\n",
      "[1, 36, 4, 343, 897, 8, 5, 352, 7, 1276, 204]\n",
      "[1, 5, 47, 461, 25, 20, 31, 92]\n",
      "[1, 68, 291, 17, 7, 67, 169, 647, 11, 33, 7, 1658, 145]\n",
      "[1, 27, 14, 35, 10, 2575]\n",
      "[1, 75, 6, 205]\n",
      "[1, 64, 6, 313, 6, 408, 229, 74, 4]\n",
      "[1, 14, 146, 101, 1667]\n",
      "[1, 46, 347, 51, 67]\n",
      "[1, 49, 2039, 376, 10597]\n",
      "[1, 49, 5, 171, 563, 7]\n",
      "[1, 1471, 50, 28, 1593, 31, 534, 56, 137, 133, 55]\n",
      "[1, 169, 4]\n",
      "[1, 146, 14, 111, 348, 132, 544, 49]\n",
      "[1, 31, 333, 3, 83, 361]\n",
      "[1, 631, 158, 9, 218]\n",
      "[1, 7, 1027, 15, 4910, 11, 360, 7, 2092]\n",
      "[1, 10, 434, 3693]\n",
      "[1, 10, 253, 1240, 22, 63, 1248, 1028, 71, 8, 1214]\n",
      "[1, 5, 589, 41, 10, 1472, 19, 7]\n",
      "[1, 49, 14, 1087, 37, 406, 17, 7]\n",
      "[1, 463, 67, 3, 3, 3]\n",
      "[1, 9680, 3709, 48, 15, 4, 418, 4660]\n",
      "[1, 18, 85, 120, 285]\n",
      "[1, 7, 61, 872]\n",
      "[1, 755, 6, 1094, 385, 25, 31, 17]\n",
      "[1, 49, 153, 33, 738, 6, 4105]\n",
      "[1, 66, 74, 340, 1151, 2257, 21, 158, 4729, 20]\n",
      "[1, 72, 14, 205, 56, 4, 1418, 8]\n",
      "[1, 51, 27, 364, 445]\n",
      "[1, 122, 41, 181]\n",
      "[1, 240, 11, 1989, 4, 1310, 6]\n",
      "[1, 11, 573, 55]\n",
      "[1, 25, 686, 4, 2894, 10099]\n",
      "[1, 4, 145, 56, 720]\n",
      "[1, 5, 292, 252]\n",
      "[1, 31, 70, 61, 17, 139]\n",
      "[1, 149, 40, 45]\n",
      "[1, 5, 363, 6, 237, 1617]\n",
      "[1, 1766, 22, 306, 8, 1130, 24, 14, 3, 1766]\n",
      "[1, 5, 111, 7, 163, 159]\n",
      "[1, 50, 27, 66, 21]\n",
      "[1, 985, 10, 392, 3, 141, 33]\n",
      "[1, 966, 24, 26, 5, 146]\n",
      "[1, 182, 5713, 721]\n"
     ]
    }
   ],
   "source": [
    "joke_list = []\n",
    "for i in random[:50]:\n",
    "    sent = sentences[i]\n",
    "    seed_list = sent.split()\n",
    "    seed = seed_list[:int(len(seed_list)/3)]\n",
    "    joke = generate_categorical(model, tokenizer,' '.join(seed), maxlen=40, tag=2, exploration_factor=0.1)\n",
    "    joke_list.append((' '.join(seed),joke))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "parser_sent = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 7, 1234, 8, 63, 2969, 8, 9, 7, 386, 8531, 307, 106, 4, 230]\n",
      "[1, 7, 1234, 8, 63, 2969, 8, 9, 7, 386, 8531, 307, 106, 4, 230]\n",
      "[1, 7, 1234, 8, 63, 2969, 8, 9, 7, 386, 8531, 307, 106, 4, 230]\n",
      "[1, 31, 165, 166, 7981, 10, 427, 3197, 110, 1512, 73, 404, 73, 4]\n",
      "[1, 31, 165, 166, 7981, 10, 427, 3197, 110, 1512, 73, 404, 73, 4]\n",
      "[1, 31, 165, 166, 7981, 10, 427, 3197, 110, 1512, 73, 404, 73, 4]\n",
      "[1, 51, 172, 275, 38, 81, 17, 2231]\n",
      "[1, 51, 172, 275, 38, 81, 17, 2231]\n",
      "[1, 51, 172, 275, 38, 81, 17, 2231]\n",
      "[1, 5, 941, 7, 502, 74, 4830]\n",
      "[1, 5, 941, 7, 502, 74, 4830]\n",
      "[1, 5, 941, 7, 502, 74, 4830]\n",
      "[1, 250, 585, 39, 264, 3, 208, 6]\n",
      "[1, 250, 585, 39, 264, 3, 208, 6]\n",
      "[1, 250, 585, 39, 264, 3, 208, 6]\n",
      "[1, 50, 38, 218, 66, 25, 61, 36, 1677, 13, 4, 4733, 4719, 15, 4]\n",
      "[1, 50, 38, 218, 66, 25, 61, 36, 1677, 13, 4, 4733, 4719, 15, 4]\n",
      "[1, 50, 38, 218, 66, 25, 61, 36, 1677, 13, 4, 4733, 4719, 15, 4]\n",
      "[1, 36, 4, 343, 897, 8, 5, 352, 7, 1276, 204, 586, 4, 147, 349, 3892]\n",
      "[1, 36, 4, 343, 897, 8, 5, 352, 7, 1276, 204, 586, 4, 147, 349, 3892]\n",
      "[1, 36, 4, 343, 897, 8, 5, 352, 7, 1276, 204, 586, 4, 147, 349, 3892]\n",
      "[1, 5, 47, 461, 25, 20, 31, 92, 183, 6, 30, 266]\n",
      "[1, 5, 47, 461, 25, 20, 31, 92, 183, 6, 30, 266]\n",
      "[1, 5, 47, 461, 25, 20, 31, 92, 183, 6, 30, 266]\n",
      "[1, 68, 291, 17, 7, 67, 169, 647, 11, 33, 7, 1658, 145, 8, 46, 92, 7, 457, 180, 43]\n",
      "[1, 68, 291, 17, 7, 67, 169, 647, 11, 33, 7, 1658, 145, 8, 46, 92, 7, 457, 180, 43]\n",
      "[1, 68, 291, 17, 7, 67, 169, 647, 11, 33, 7, 1658, 145, 8, 46, 92, 7, 457, 180, 43]\n",
      "[1, 27, 14, 35, 10, 2575, 13, 5, 78, 37]\n",
      "[1, 27, 14, 35, 10, 2575, 13, 5, 78, 37]\n",
      "[1, 27, 14, 35, 10, 2575, 13, 5, 78, 37]\n",
      "[1, 75, 6, 205, 7, 3225, 203]\n",
      "[1, 75, 6, 205, 7, 3225, 203]\n",
      "[1, 75, 6, 205, 7, 3225, 203]\n",
      "[1, 64, 6, 313, 6, 408, 229, 74, 4, 425, 199, 15, 4, 45]\n",
      "[1, 64, 6, 313, 6, 408, 229, 74, 4, 425, 199, 15, 4, 45]\n",
      "[1, 64, 6, 313, 6, 408, 229, 74, 4, 425, 199, 15, 4, 45]\n",
      "[1, 14, 146, 101, 1667, 7, 1615, 1359]\n",
      "[1, 14, 146, 101, 1667, 7, 1615, 1359]\n",
      "[1, 14, 146, 101, 1667, 7, 1615, 1359]\n",
      "[1, 46, 347, 51, 67, 1687, 44, 153]\n",
      "[1, 46, 347, 51, 67, 1687, 44, 153]\n",
      "[1, 46, 347, 51, 67, 1687, 44, 153]\n",
      "[1, 49, 2039, 376, 10597, 122, 6, 258]\n",
      "[1, 49, 2039, 376, 10597, 122, 6, 258]\n",
      "[1, 49, 2039, 376, 10597, 122, 6, 258]\n",
      "[1, 49, 5, 171, 563, 7, 2362, 8, 31]\n",
      "[1, 49, 5, 171, 563, 7, 2362, 8, 31]\n",
      "[1, 49, 5, 171, 563, 7, 2362, 8, 31]\n",
      "[1, 1471, 50, 28, 1593, 31, 534, 56, 137, 133, 55, 3, 350, 114, 4, 1091, 17]\n",
      "[1, 1471, 50, 28, 1593, 31, 534, 56, 137, 133, 55, 3, 350, 114, 4, 1091, 17]\n",
      "[1, 1471, 50, 28, 1593, 31, 534, 56, 137, 133, 55, 3, 350, 114, 4, 1091, 17]\n",
      "[1, 169, 4, 2167, 1353]\n",
      "[1, 169, 4, 2167, 1353]\n",
      "[1, 169, 4, 2167, 1353]\n",
      "[1, 146, 14, 111, 348, 132, 544, 49, 40, 15, 54, 2001]\n",
      "[1, 146, 14, 111, 348, 132, 544, 49, 40, 15, 54, 2001]\n",
      "[1, 146, 14, 111, 348, 132, 544, 49, 40, 15, 54, 2001]\n",
      "[1, 31, 333, 3, 83, 361, 6, 471, 17]\n",
      "[1, 31, 333, 3, 83, 361, 6, 471, 17]\n",
      "[1, 31, 333, 3, 83, 361, 6, 471, 17]\n",
      "[1, 631, 158, 9, 218, 9, 208, 6]\n",
      "[1, 631, 158, 9, 218, 9, 208, 6]\n",
      "[1, 631, 158, 9, 218, 9, 208, 6]\n",
      "[1, 7, 1027, 15, 4910, 11, 360, 7, 2092, 3, 7, 1027, 15, 66]\n",
      "[1, 7, 1027, 15, 4910, 11, 360, 7, 2092, 3, 7, 1027, 15, 66]\n",
      "[1, 7, 1027, 15, 4910, 11, 360, 7, 2092, 3, 7, 1027, 15, 66]\n",
      "[1, 10, 434, 3693, 11, 4383, 5]\n",
      "[1, 10, 434, 3693, 11, 4383, 5]\n",
      "[1, 10, 434, 3693, 11, 4383, 5]\n",
      "[1, 10, 253, 1240, 22, 63, 1248, 1028, 71, 8, 1214, 41, 8, 103, 1511, 22, 380]\n",
      "[1, 10, 253, 1240, 22, 63, 1248, 1028, 71, 8, 1214, 41, 8, 103, 1511, 22, 380]\n",
      "[1, 10, 253, 1240, 22, 63, 1248, 1028, 71, 8, 1214, 41, 8, 103, 1511, 22, 380]\n",
      "[1, 5, 589, 41, 10, 1472, 19, 7, 642, 102, 24, 4]\n",
      "[1, 5, 589, 41, 10, 1472, 19, 7, 642, 102, 24, 4]\n",
      "[1, 5, 589, 41, 10, 1472, 19, 7, 642, 102, 24, 4]\n",
      "[1, 49, 14, 1087, 37, 406, 17, 7, 1971, 2509, 26, 146]\n",
      "[1, 49, 14, 1087, 37, 406, 17, 7, 1971, 2509, 26, 146]\n",
      "[1, 49, 14, 1087, 37, 406, 17, 7, 1971, 2509, 26, 146]\n",
      "[1, 463, 67, 3, 3, 3, 350, 188, 42]\n",
      "[1, 463, 67, 3, 3, 3, 350, 188, 42]\n",
      "[1, 463, 67, 3, 3, 3, 350, 188, 42]\n",
      "[1, 9680, 3709, 48, 15, 4, 418, 4660, 2173, 62, 251, 87]\n",
      "[1, 9680, 3709, 48, 15, 4, 418, 4660, 2173, 62, 251, 87]\n",
      "[1, 9680, 3709, 48, 15, 4, 418, 4660, 2173, 62, 251, 87]\n",
      "[1, 18, 85, 120, 285, 2479, 722, 4]\n",
      "[1, 18, 85, 120, 285, 2479, 722, 4]\n",
      "[1, 18, 85, 120, 285, 2479, 722, 4]\n",
      "[1, 7, 61, 872, 15, 7070, 399]\n",
      "[1, 7, 61, 872, 15, 7070, 399]\n",
      "[1, 7, 61, 872, 15, 7070, 399]\n",
      "[1, 755, 6, 1094, 385, 25, 31, 17, 531, 2431, 640, 36]\n",
      "[1, 755, 6, 1094, 385, 25, 31, 17, 531, 2431, 640, 36]\n",
      "[1, 755, 6, 1094, 385, 25, 31, 17, 531, 2431, 640, 36]\n",
      "[1, 49, 153, 33, 738, 6, 4105, 2126, 5689, 113, 50]\n",
      "[1, 49, 153, 33, 738, 6, 4105, 2126, 5689, 113, 50]\n",
      "[1, 49, 153, 33, 738, 6, 4105, 2126, 5689, 113, 50]\n",
      "[1, 66, 74, 340, 1151, 2257, 21, 158, 4729, 20, 5387, 245, 59, 4778, 1938]\n",
      "[1, 66, 74, 340, 1151, 2257, 21, 158, 4729, 20, 5387, 245, 59, 4778, 1938]\n",
      "[1, 66, 74, 340, 1151, 2257, 21, 158, 4729, 20, 5387, 245, 59, 4778, 1938]\n",
      "[1, 72, 14, 205, 56, 4, 1418, 8, 2992, 8, 3215, 13]\n",
      "[1, 72, 14, 205, 56, 4, 1418, 8, 2992, 8, 3215, 13]\n",
      "[1, 72, 14, 205, 56, 4, 1418, 8, 2992, 8, 3215, 13]\n",
      "[1, 51, 27, 364, 445, 289, 680, 24]\n",
      "[1, 51, 27, 364, 445, 289, 680, 24]\n",
      "[1, 51, 27, 364, 445, 289, 680, 24]\n",
      "[1, 122, 41, 181, 13, 5]\n",
      "[1, 122, 41, 181, 13, 5]\n",
      "[1, 122, 41, 181, 13, 5]\n",
      "[1, 240, 11, 1989, 4, 1310, 6, 97, 3, 5]\n",
      "[1, 240, 11, 1989, 4, 1310, 6, 97, 3, 5]\n",
      "[1, 240, 11, 1989, 4, 1310, 6, 97, 3, 5]\n",
      "[1, 11, 573, 55, 16, 2343]\n",
      "[1, 11, 573, 55, 16, 2343]\n",
      "[1, 11, 573, 55, 16, 2343]\n",
      "[1, 25, 686, 4, 2894, 10099, 11, 510, 17]\n",
      "[1, 25, 686, 4, 2894, 10099, 11, 510, 17]\n",
      "[1, 25, 686, 4, 2894, 10099, 11, 510, 17]\n",
      "[1, 4, 145, 56, 720, 11, 20, 46]\n",
      "[1, 4, 145, 56, 720, 11, 20, 46]\n",
      "[1, 4, 145, 56, 720, 11, 20, 46]\n",
      "[1, 5, 292, 252, 51, 6197]\n",
      "[1, 5, 292, 252, 51, 6197]\n",
      "[1, 5, 292, 252, 51, 6197]\n",
      "[1, 31, 70, 61, 17, 139, 12, 5, 408, 24]\n",
      "[1, 31, 70, 61, 17, 139, 12, 5, 408, 24]\n",
      "[1, 31, 70, 61, 17, 139, 12, 5, 408, 24]\n",
      "[1, 149, 40, 45, 16, 5]\n",
      "[1, 149, 40, 45, 16, 5]\n",
      "[1, 149, 40, 45, 16, 5]\n",
      "[1, 5, 363, 6, 237, 1617, 6, 143, 264]\n",
      "[1, 5, 363, 6, 237, 1617, 6, 143, 264]\n",
      "[1, 5, 363, 6, 237, 1617, 6, 143, 264]\n",
      "[1, 1766, 22, 306, 8, 1130, 24, 14, 3, 1766, 22, 1103, 8, 1130, 24]\n",
      "[1, 1766, 22, 306, 8, 1130, 24, 14, 3, 1766, 22, 1103, 8, 1130, 24]\n",
      "[1, 1766, 22, 306, 8, 1130, 24, 14, 3, 1766, 22, 1103, 8, 1130, 24]\n",
      "[1, 5, 111, 7, 163, 159, 65, 3, 1265]\n",
      "[1, 5, 111, 7, 163, 159, 65, 3, 1265]\n",
      "[1, 5, 111, 7, 163, 159, 65, 3, 1265]\n",
      "[1, 50, 27, 66, 21, 6, 30, 25]\n",
      "[1, 50, 27, 66, 21, 6, 30, 25]\n",
      "[1, 50, 27, 66, 21, 6, 30, 25]\n",
      "[1, 985, 10, 392, 3, 141, 33, 17, 7, 441, 643]\n",
      "[1, 985, 10, 392, 3, 141, 33, 17, 7, 441, 643]\n",
      "[1, 985, 10, 392, 3, 141, 33, 17, 7, 441, 643]\n",
      "[1, 966, 24, 26, 5, 146, 27, 65, 13, 161]\n",
      "[1, 966, 24, 26, 5, 146, 27, 65, 13, 161]\n",
      "[1, 966, 24, 26, 5, 146, 27, 65, 13, 161]\n",
      "[1, 182, 5713, 721, 65, 12]\n",
      "[1, 182, 5713, 721, 65, 12]\n",
      "[1, 182, 5713, 721, 65, 12]\n"
     ]
    }
   ],
   "source": [
    "joke_list = []\n",
    "for i in random[:50]:\n",
    "    sent = sentences[i]\n",
    "    seed_list = sent.split()\n",
    "    seed = seed_list[:int(len(seed_list)/2)]\n",
    "    joke = generate_categorical(model, tokenizer,' '.join(seed), maxlen=40, tag=0, exploration_factor=0.5)\n",
    "    quote = generate_categorical(model, tokenizer,' '.join(seed), maxlen=40, tag=1, exploration_factor=0.5)\n",
    "    tweet = generate_categorical(model, tokenizer,' '.join(seed), maxlen=40, tag=2, exploration_factor=0.5)\n",
    "    joke_list.append((' '.join(seed),joke,quote,tweet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('sos a priest , an astronaut , and a hair dresser walk into the bar',\n",
       "  'sos a priest , an astronaut , and a hair dresser walk into the bar . the bartender says \" hey i don\\'t care . \" eos',\n",
       "  'sos a priest , an astronaut , and a hair dresser walk into the bar and walk with a blank sheet . eos',\n",
       "  \"sos a priest , an astronaut , and a hair dresser walk into the bar . i don't have to go to work . eos\"),\n",
       " (\"sos i'm sad after discovering my awesome headphones only reach as far as the\",\n",
       "  \"sos i'm sad after discovering my awesome headphones only reach as far as the determination was in its hands . if i didn't show up and got a good job , i wasn't too good . eos\",\n",
       "  \"sos i'm sad after discovering my awesome headphones only reach as far as the camera and the camera on my films . eos\",\n",
       "  \"sos i'm sad after discovering my awesome headphones only reach as far as the same as i are that no good will i have to wake up out of her soul but im not realizing threedots eos\"),\n",
       " ('sos how many days are there in canadian',\n",
       "  'sos how many days are there in canadian cities ? because it\\'d be called \" the explorer \" . eos',\n",
       "  'sos how many days are there in canadian affairs and in street groups and , in the civil war , there are no authorities on the streets . eos',\n",
       "  'sos how many days are there in canadian business ? i believe that i am not drunk until an hour and i have to go out . eos'),\n",
       " ('sos i needed a break from stupidity',\n",
       "  'sos i needed a break from stupidity today . it was pretty hard to perform before it was a good hour . eos',\n",
       "  'sos i needed a break from stupidity . i could not speak any way i could tell you . eos',\n",
       "  'sos i needed a break from stupidity . let me know if i could get you out of it on my way to everything . eos'),\n",
       " ('sos watching x - men . hard to',\n",
       "  \"sos watching x - men . hard to explain , but i don't know how to feel my own ! now i have just found out that i can never be a good person . eos\",\n",
       "  'sos watching x - men . hard to find joy in the face of the song . starve to this music . eos',\n",
       "  'sos watching x - men . hard to get up and get up and go to bed today . eos'),\n",
       " ('sos why are black people so good at basketball ? the core elements of the',\n",
       "  'sos why are black people so good at basketball ? the core elements of the road while what they hate . eos',\n",
       "  'sos why are black people so good at basketball ? the core elements of the world are not informed . i think that is useful . eos',\n",
       "  'sos why are black people so good at basketball ? the core elements of the world are not going to be a good day eos'),\n",
       " ('sos at the coffee shop , i saw a german guy reading the first few pages',\n",
       "  'sos at the coffee shop , i saw a german guy reading the first few pages of his tweets . it was a words of how to pick up the german emails . eos',\n",
       "  'sos at the coffee shop , i saw a german guy reading the first few pages of \\' how did you \" and months to see that you did not know ? eos',\n",
       "  'sos at the coffee shop , i saw a german guy reading the first few pages of some facilities . i think it was a good day eos'),\n",
       " (\"sos i don't drink so that i'm more fun to be around\",\n",
       "  \"sos i don't drink so that i'm more fun to be around . but i don't know how to move through the world outside eos\",\n",
       "  \"sos i don't drink so that i'm more fun to be around . i feel like i have been able to do some things . eos\",\n",
       "  \"sos i don't drink so that i'm more fun to be around . i don't want to go to school tomorrow . eos\"),\n",
       " (\"sos we live in a time where government is not a leadership thing , it's more a business that's out\",\n",
       "  \"sos we live in a time where government is not a leadership thing , it's more a business that's out of the way . eos\",\n",
       "  \"sos we live in a time where government is not a leadership thing , it's more a business that's out there . eos\",\n",
       "  \"sos we live in a time where government is not a leadership thing , it's more a business that's out of europe . eos\"),\n",
       " ('sos do you like my suit ? i think this',\n",
       "  'sos do you like my suit ? i think this is a fuck in the middle of your night . eos',\n",
       "  'sos do you like my suit ? i think this is a better way to get out of the house , and i make a lot of money . eos',\n",
       "  'sos do you like my suit ? i think this is a good day . i had to deal with it all day . eos'),\n",
       " ('sos want to hear a construction joke',\n",
       "  'sos want to hear a construction joke ? nevermind , it was a repost . eos',\n",
       "  'sos want to hear a construction joke . i apologize tell myself that i was as good as a movie . eos',\n",
       "  \"sos want to hear a construction joke but i don't want to get up early ! eos\"),\n",
       " ('sos going to try to stay away from the computer most of the day',\n",
       "  'sos going to try to stay away from the computer most of the day , but i make it wait for the rest of my life . eos',\n",
       "  \"sos going to try to stay away from the computer most of the day , i don't want any other ideas to believe in . eos\",\n",
       "  'sos going to try to stay away from the computer most of the day . i hate it that i have to go to work eos'),\n",
       " ('sos you should never marry a tennis player',\n",
       "  'sos you should never marry a tennis player . you never know what they say . period ! eos',\n",
       "  'sos you should never marry a tennis player . just wait for the next moment . eos',\n",
       "  'sos you should never marry a tennis player , good - que - u guys ! eos'),\n",
       " (\"sos it's funny how time flies when you're\",\n",
       "  \"sos it's funny how time flies when you're showers , i'm not sure if i'm going to get mad . eos\",\n",
       "  \"sos it's funny how time flies when you're born , you shouldn't be able to do it . they'll say , ' gee , we're going to die . ' and then you get to the point of being a dad\",\n",
       "  \"sos it's funny how time flies when you're trying to get up and get you a free window . eos\"),\n",
       " (\"sos if pot gets legalized what's to stop\",\n",
       "  \"sos if pot gets legalized what's to stop the rest of the world threedots it was obviously the only way to get the money to fires . eos\",\n",
       "  \"sos if pot gets legalized what's to stop the government from the government or if you don't have the money , you don't have a government . eos\",\n",
       "  \"sos if pot gets legalized what's to stop me from bicycles , checked me out , wanna go to the beach with a friend before i leave the house to do it . eos\"),\n",
       " (\"sos if i ever become a vet , i'm\",\n",
       "  \"sos if i ever become a vet , i'm for a tooth . but i don't think i'm going to get conscience . eos\",\n",
       "  \"sos if i ever become a vet , i'm not where i want to be , but i don't think i can take it . eos\",\n",
       "  \"sos if i ever become a vet , i'm just married my stomach . i hate it . eos\"),\n",
       " (\"sos idk why but gosh i'm crazy about him right now . maybe its the singing in\",\n",
       "  \"sos idk why but gosh i'm crazy about him right now . maybe its the singing in better luck . eos\",\n",
       "  \"sos idk why but gosh i'm crazy about him right now . maybe its the singing in the music industry . eos\",\n",
       "  \"sos idk why but gosh i'm crazy about him right now . maybe its the singing in the music . eos\"),\n",
       " (\"sos where the woman's neck\",\n",
       "  \"sos where the woman's neck is ? what the left side of the body is ? the female . eos\",\n",
       "  \"sos where the woman's neck is , she is always the right person . the second is the right person who wants to be the right person . eos\",\n",
       "  \"sos where the woman's neck is gone backwards ? i don't want to go out with my friends . eos\"),\n",
       " ('sos should you feel ok being broke if all of your bills',\n",
       "  'sos should you feel ok being broke if all of your bills were gone ? \" \" cuz i don\\'t know what to do anymore . \" eos',\n",
       "  'sos should you feel ok being broke if all of your bills have gone through an emotional impact on the part of your potential . eos',\n",
       "  'sos should you feel ok being broke if all of your bills taken out of you ? email will be better . eos'),\n",
       " (\"sos i'm bored . who wants to fall in\",\n",
       "  \"sos i'm bored . who wants to fall in love ? there's no way i can get it down . eos\",\n",
       "  \"sos i'm bored . who wants to fall in love with me ? i don't think you can do that . eos\",\n",
       "  \"sos i'm bored . who wants to fall in love with me right now ? i don't know what to do . eos\"),\n",
       " ('sos whats long and black and hard to',\n",
       "  'sos whats long and black and hard to kill ? a broken nut ! eos',\n",
       "  'sos whats long and black and hard to be in a relationship and not a bad thing to talk to . eos',\n",
       "  'sos whats long and black and hard to wake up early ? not . tired of being sick . eos'),\n",
       " ('sos a group of crows is called a murder . a group of people',\n",
       "  \"sos a group of crows is called a murder . a group of people don't know ! eos\",\n",
       "  'sos a group of crows is called a murder . a group of people who basis a label is very funny . eos',\n",
       "  'sos a group of crows is called a murder . a group of people no reason ! eos'),\n",
       " ('sos my favorite element is helium i',\n",
       "  \"sos my favorite element is helium i don't care about effects . if i ain't heard you have a good joke , i don't recommend it inside . eos\",\n",
       "  \"sos my favorite element is helium i don't know if i'll ever get to the top . eos\",\n",
       "  \"sos my favorite element is helium i am wearing a pair of boots sometimes . i don't think it should be shopping . eos\"),\n",
       " ('sos my mom calls me an older soul because , growing up , she taught me stuff',\n",
       "  'sos my mom calls me an older soul because , growing up , she taught me stuff called \" black friday \" . he won the bar . eos',\n",
       "  'sos my mom calls me an older soul because , growing up , she taught me stuff , and she taught me how to play time , and i decided to go outside that i could go to auditions out on',\n",
       "  'sos my mom calls me an older soul because , growing up , she taught me stuff . i love her dad . eos'),\n",
       " ('sos i gave up my seat for a blind man on the',\n",
       "  'sos i gave up my seat for a blind man on the road threedots threedots he was an outstanding owner . eos',\n",
       "  \"sos i gave up my seat for a blind man on the playground , every day , and he just would've gone to the movies . i know i was a good photographer . eos\",\n",
       "  'sos i gave up my seat for a blind man on the way home . i just want to see him again apparently . eos'),\n",
       " ('sos if you drop this book in a pig pen what should',\n",
       "  'sos if you drop this book in a pig pen what should you do ? a : chew chew . eos',\n",
       "  \"sos if you drop this book in a pig pen what should you do ? we don't serve to know that . eos\",\n",
       "  'sos if you drop this book in a pig pen what should i do for you ? eos'),\n",
       " (\"sos class time . . . maybe i'll get\",\n",
       "  \"sos class time . . . maybe i'll get the earthquake in the middle of the ocean . walt threedots eos\",\n",
       "  \"sos class time . . . maybe i'll get through some of the work that i do . but i don't like the idea of opening label - whatever it is , i don't care what's wrong with me . eos\",\n",
       "  \"sos class time . . . maybe i'll get some sleep . i hate this . i don't like it . eos\"),\n",
       " ('sos barry hoped one of the almost dozen puns he told his',\n",
       "  'sos barry hoped one of the almost dozen puns he told his father they had to spend the rest of his life building . eos',\n",
       "  'sos barry hoped one of the almost dozen puns he told his boss was a little bit less by putting his hands in the pleasure of the family . eos',\n",
       "  \"sos barry hoped one of the almost dozen puns he told his head was not left . i was throwing out the window . i can't sleep . eos\"),\n",
       " ('sos it has been pretty decent seeing the',\n",
       "  \"sos it has been pretty decent seeing the world about me . i don't know what to say threedots in the field . eos\",\n",
       "  'sos it has been pretty decent seeing the world . i was heavily discouraged from doing it . eos',\n",
       "  'sos it has been pretty decent seeing the new moon trailer . and in the end , got to be won . eos'),\n",
       " ('sos a good deal of tyranny goes',\n",
       "  'sos a good deal of tyranny goes out , q : what do you call a black man flying a plane ? a pilot you racist bastards . eos',\n",
       "  'sos a good deal of tyranny goes making a difference . the only thing that can be avoided is that they can be killed than the people who are in india raising the debt of the armed conflicts with the united',\n",
       "  \"sos a good deal of tyranny goes to the airport to get my phone off . idk what to do tonight it's going to happen . eos\"),\n",
       " (\"sos lazy to cook dinner so i'm in drive thru line at\",\n",
       "  \"sos lazy to cook dinner so i'm in drive thru line at a restaurant when i get your car at the shed eos\",\n",
       "  \"sos lazy to cook dinner so i'm in drive thru line at a very young age . eos\",\n",
       "  \"sos lazy to cook dinner so i'm in drive thru line at home in the morning lol eos\"),\n",
       " (\"sos if you're not supposed to abuse cough syrup then why\",\n",
       "  \"sos if you're not supposed to abuse cough syrup then why do you feel like i'm not watching heroin ? eos\",\n",
       "  \"sos if you're not supposed to abuse cough syrup then why do you have to play it sir ? spoil it . eos\",\n",
       "  \"sos if you're not supposed to abuse cough syrup then why did you have to go to sleep ? eos\"),\n",
       " ('sos people from both political parties have long recognized that welfare without work creates negative',\n",
       "  'sos people from both political parties have long recognized that welfare without work creates negative incentives is for my kids . eos',\n",
       "  'sos people from both political parties have long recognized that welfare without work creates negative incentives that begin to make decisions based on science . eos',\n",
       "  'sos people from both political parties have long recognized that welfare without work creates negative incentives . eos'),\n",
       " ('sos did you hear about the dyslexic , agnostic , insomniac ?',\n",
       "  'sos did you hear about the dyslexic , agnostic , insomniac ? he stayed up all night wondering if there was a dog . eos',\n",
       "  'sos did you hear about the dyslexic , agnostic , insomniac ? he stayed up all night wondering if there was a dog . eos',\n",
       "  \"sos did you hear about the dyslexic , agnostic , insomniac ? i don't just want to go to sister's well . eos\"),\n",
       " ('sos how do three gay guys sit on',\n",
       "  'sos how do three gay guys sit on the couch ? throw a cigarette overboard . eos',\n",
       "  'sos how do three gay guys sit on the couch every day ? no one knows . eos',\n",
       "  'sos how do three gay guys sit on the couch ? my mom is know i have headache . eos'),\n",
       " (\"sos what's up everyone ? i\",\n",
       "  \"sos what's up everyone ? i hope you are a good dad . go to the doctors . eos\",\n",
       "  \"sos what's up everyone ? i don't stuff . i don't want to play the game . i don't want to play football , i don't want to practice . eos\",\n",
       "  \"sos what's up everyone ? i don't know what to do with her . i appreciate it . eos\"),\n",
       " ('sos family is honestly the key to life . i',\n",
       "  'sos family is honestly the key to life . i would wacky if they had a good time . eos',\n",
       "  'sos family is honestly the key to life . i love the characters . i love being in the movies . eos',\n",
       "  \"sos family is honestly the key to life . i don't want to go and get my son in . eos\"),\n",
       " ('sos is writing now ! setting',\n",
       "  'sos is writing now ! setting a new book on sex because the first one is going to ask what she is doing . eos',\n",
       "  'sos is writing now ! setting goals and athletics , and i think we need to be able to do that . eos',\n",
       "  'sos is writing now ! setting it up for a few hours gotta go into the office tho . eos'),\n",
       " ('sos so apparently the financial district is busy in',\n",
       "  'sos so apparently the financial district is busy in the market . i can barely afford to get it we can have grapes . eos',\n",
       "  \"sos so apparently the financial district is busy in the market - that's just a genetic thing . eos\",\n",
       "  \"sos so apparently the financial district is busy in the house . i don't want to seem to go to school tomorrow eos\"),\n",
       " (\"sos the thing about science is that it's\",\n",
       "  'sos the thing about science is that it\\'s not the same in the same way threedots threedots so it wouldn\\'t be called \" the same way for me . \" eos',\n",
       "  \"sos the thing about science is that it's often the policy of the state killing something , but to be able to do it . eos\",\n",
       "  \"sos the thing about science is that it's not long enough to write it . but i don't know what to do eos\"),\n",
       " ('sos i cant believe how worthless',\n",
       "  \"sos i cant believe how worthless my boss is haha ! i or i don't know what's wrong with me threedots eos\",\n",
       "  \"sos i cant believe how worthless my favorite questions come from being . i don't know what i mean . eos\",\n",
       "  \"sos i cant believe how worthless life is . i guess all i'll have to watch is calm in the middle of my night . eos\"),\n",
       " (\"sos i'm really good in bed threedots i stay on\",\n",
       "  \"sos i'm really good in bed threedots i stay on the couch and i'm not doing though . eos\",\n",
       "  \"sos i'm really good in bed threedots i stay on the road and i just want to be the mom . eos\",\n",
       "  \"sos i'm really good in bed threedots i stay on twitter and go back to sleep , but i cant sleep eos\"),\n",
       " ('sos school all day ! i',\n",
       "  'sos school all day ! i was reminded that i had to stop the mexican searches from the gas station threedots eos',\n",
       "  'sos school all day ! i still have a best at the tire of burning magazines . eos',\n",
       "  'sos school all day ! i have so much to do all day ! eos'),\n",
       " ('sos i used to give anal to other men',\n",
       "  'sos i used to give anal to other men threedots but yesterday , i finally got a new bag of chips . eos',\n",
       "  'sos i used to give anal to other men , but i was a very shy butterfly . for me , i was trying to do the same thing for the creed . eos',\n",
       "  'sos i used to give anal to other men . you know what they say about me ? eos'),\n",
       " ('sos fool me once , shame on you . fool me twice , shame on',\n",
       "  'sos fool me once , shame on you . fool me twice , shame on you . fool me twice , fool me twice . eos',\n",
       "  'sos fool me once , shame on you . fool me twice , shame on you . remember , i am not sure . eos',\n",
       "  'sos fool me once , shame on you . fool me twice , shame on me . eos'),\n",
       " ('sos i feel a little better today . hmm',\n",
       "  \"sos i feel a little better today . hmm threedots i don't know what he laced them with but i've been tripping all day eos\",\n",
       "  \"sos i feel a little better today . hmm , i'm not going to wear clothes . eos\",\n",
       "  'sos i feel a little better today . hmm , and i want to go to church eos'),\n",
       " ('sos why do people have to be so',\n",
       "  \"sos why do people have to be so ugly when we die ? because they can't even get into the street for the rest of the year . eos\",\n",
       "  \"sos why do people have to be so rich ? because they don't think they think it has . eos\",\n",
       "  \"sos why do people have to be so tired ? i don't think i can get up early to cook a sandwich and i still have no idea what * i * should do . eos\"),\n",
       " ('sos changed my mind . im not in a water mood',\n",
       "  'sos changed my mind . im not in a water mood ! you now have to do bit of work ! eos',\n",
       "  \"sos changed my mind . im not in a water mood , but it's very hard for me to get up to do it . eos\",\n",
       "  'sos changed my mind . im not in a water mood . i thought it was a good rest ! eos'),\n",
       " ('sos ideas on what i should do today ? come',\n",
       "  \"sos ideas on what i should do today ? come on , i'll be getting a new job . eos\",\n",
       "  \"sos ideas on what i should do today ? come back to me , i don't know . it will be a good job . eos\",\n",
       "  \"sos ideas on what i should do today ? come on , i don't care . wine is not coming . eos\"),\n",
       " ('sos having difficulty walking today threedots',\n",
       "  \"sos having difficulty walking today threedots with some of these chickens . i think that's the one babysitter she did . eos\",\n",
       "  'sos having difficulty walking today threedots is not a success , but a patient is not a failure . eos',\n",
       "  'sos having difficulty walking today threedots have an bad day threedots i dont like laundry threedots eos')]"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joke_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('evaluate_jokes.txt','w') as fop:\n",
    "    for joke in joke_list:\n",
    "        fop.write(joke[0]+\"\\t\"+joke[1]+\"\\t\"+joke[2]+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sos a priest , an astronaut , and a hair . i don't know what to do . i don't want to go to work eos\n",
      "sent:  A priest , an astronaut , and a hair.\n",
      "sent:  I don't know what to do.\n",
      "sent:  I don't want to go to work.\n",
      "sos i'm sad after discovering my awesome headphones only it's not working . i don't want to go to school tomorrow eos\n",
      "sent:  I'm sad after discovering my awesome headphones only it's not working.\n",
      "sent:  I don't want to go to school tomorrow.\n",
      "sos how many days are you ? i don't know what to do with my life . eos\n",
      "sent:  How many days are you ? I don't know what to do with my life.\n",
      "sos i needed a new phone . i don't want to go threedots i hate it . i hate it . eos\n",
      "sent:  I needed a new phone.\n",
      "sent:  I don't want to go threedots I hate it.\n",
      "sent:  I hate it.\n",
      "sos watching x - men origins : wolverine , and the movie awards . i love the movie awards ! eos\n",
      "sent:  Watching x - men origins : wolverine , and the movie awards.\n",
      "sent:  I love the movie awards !.\n",
      "sos why are black people so good at basketball ? they are not good at all . eos\n",
      "sent:  Why are black people so good at basketball ? they are not good at all.\n",
      "sos at the coffee shop , i saw a german guy in my house and i was so tired . eos\n",
      "sent:  At the coffee shop , I saw a german guy in my house and I was so tired.\n",
      "sos i don't drink so that i'm more than anything else . i don't want to go anywhere . i don't want to go to school eos\n",
      "sent:  I don't drink so that i'm more than anything else.\n",
      "sent:  I don't want to go anywhere.\n",
      "sent:  I don't want to go to school.\n",
      "sos we live in a time where government is not a leadership thing , but i don't want to go to work . eos\n",
      "sent:  We live in a time where government is not a leadership thing , but I don't want to go to work.\n",
      "sos do you like my suit ? i don't know what to do with my life . eos\n",
      "sent:  Do you like my suit ? I don't know what to do with my life.\n",
      "sos want to hear about the new iphone internet but i don't know what to do . eos\n",
      "sent:  Want to hear about the new iphone internet but I don't know what to do.\n",
      "sos going to try to stay away from the sun and see if i can go to sleep . its not a good way to start the week eos\n",
      "sent:  Going to try to stay away from the sun and see if I can go to sleep.\n",
      "sent:  Its not a good way to start the week.\n",
      "sos you should never marry me . i don't know what to do . i don't want to go to school . eos\n",
      "sent:  You should never marry me.\n",
      "sent:  I don't know what to do.\n",
      "sent:  I don't want to go to school.\n",
      "sos it's funny how time is it ? i don't know what to do with my life . eos\n",
      "sent:  It's funny how time is it ? I don't know what to do with my life.\n",
      "sos if pot gets legalized , i don't want to go to school tomorrow tickle me . eos\n",
      "sent:  If pot gets legalized , I don't want to go to school tomorrow tickle me.\n",
      "sos if i ever become a person , i don't know what to do . i don't want to go to sleep . eos\n",
      "sent:  If I ever become a person , I don't know what to do.\n",
      "sent:  I don't want to go to sleep.\n",
      "sos idk why but gosh i'm crazy about him right now . i hate him . i hate him . eos\n",
      "sent:  Idk why but gosh i'm crazy about him right now.\n",
      "sent:  I hate him.\n",
      "sent:  I hate him.\n",
      "sos where the hell is the sun ? i don't know what to do today . eos\n",
      "sent:  Where the hell is the sun ? I don't know what to do today.\n",
      "sos should you feel ok being broke if you fall asleep ? i don't know what to do . eos\n",
      "sent:  Should you feel ok being broke if you fall asleep ? I don't know what to do.\n",
      "sos i'm bored . who wants to go to bed ? where is the sun ? eos\n",
      "sent:  I'm bored.\n",
      "sent:  Who wants to go to bed ? where is the sun ?\n",
      "sos whats long and black and white and black and white and black and white and black and white and black and white and black and white and black and white and black and white and black and white and black\n",
      "sent:  Whats long and black and white and black and white and black and white and black and white and black and white and black and white and black and white and black and white and black and white and black.\n",
      "sos a group of crows is called a murder . i don't want to go to school tomorrow . eos\n",
      "sent:  A group of crows is called a murder.\n",
      "sent:  I don't want to go to school tomorrow.\n",
      "sos my favorite element is not a good idea . i don't want to go to school tomorrow . eos\n",
      "sent:  My favorite element is not a good idea.\n",
      "sent:  I don't want to go to school tomorrow.\n",
      "sos my mom calls me an older soul because , growing up my heart is not what i want to do . eos\n",
      "sent:  My mom calls me an older soul because , growing up my heart is not what I want to do.\n",
      "sos i gave up my seat for a few hours . i don't want to go to school tomorrow . eos\n",
      "sent:  I gave up my seat for a few hours.\n",
      "sent:  I don't want to go to school tomorrow.\n",
      "sos if you drop this book in a few days , i guess i will have to go to work tomorrow eos\n",
      "sent:  If you drop this book in a few days , I guess I will have to go to work tomorrow.\n",
      "sos class time . . . i hate being sick . i hate being sick . finally . i hate being sick . eos\n",
      "sent:  Class time.\n",
      "sent:  I hate being sick.\n",
      "sent:  I hate being sick.\n",
      "sent:  Finally.\n",
      "sent:  I hate being sick.\n",
      "sos barry hoped one of the almost dozen days of school is over . i don't even know what to do . eos\n",
      "sent:  Barry hoped one of the almost dozen days of school is over.\n",
      "sent:  I don't even know what to do.\n",
      "sos it has been pretty hard to get my hair done my hair , and i don't want to go to work eos\n",
      "sent:  It has been pretty hard to get my hair done my hair , and I don't want to go to work.\n",
      "sos a good deal for me . i am not going to free my house . i hate it . eos\n",
      "sent:  A good deal for me.\n",
      "sent:  I am not going to free my house.\n",
      "sent:  I hate it.\n",
      "sos lazy to cook dinner so i'm in a good mood . i hate it when i get home . eos\n",
      "sent:  Lazy to cook dinner so i'm in a good mood.\n",
      "sent:  I hate it when I get home.\n",
      "sos if you're not supposed to abuse anything , you can never be able to get it . eos\n",
      "sent:  If you're not supposed to abuse anything , you can never be able to get it.\n",
      "sos people from both political parties have long recognized that i don't want to go to school . every time i get to see my friends . eos\n",
      "sent:  People from both political parties have long recognized that I don't want to go to school.\n",
      "sent:  Every time I get to see my friends.\n",
      "sos did you hear about the dyslexic , agnostic , insomniac ? he was a good friend . but she was a good friend . eos\n",
      "sent:  Did you hear about the dyslexic , agnostic , insomniac ? he was a good friend.\n",
      "sent:  But she was a good friend.\n",
      "sos how do three gay people get to work ? they are not going to be a good day . eos\n",
      "sent:  How do three gay people get to work ? they are not going to be a good day.\n",
      "sos what's up everyone ? wouldn't you be pretentious ? i think i am going to be able to follow you . eos\n",
      "sent:  What's up everyone ? wouldn't you be pretentious ? I think I am going to be able to follow you.\n",
      "sos family is honestly the key to the music . i don't want to go to school tomorrow . eos\n",
      "sent:  Family is honestly the key to the music.\n",
      "sent:  I don't want to go to school tomorrow.\n",
      "sos is writing now . i have to go to work tomorrow . i hate being sick . eos\n",
      "sent:  Is writing now.\n",
      "sent:  I have to go to work tomorrow.\n",
      "sent:  I hate being sick.\n",
      "sos so apparently the financial district is going to be a long day . i think i have a headache . eos\n",
      "sent:  So apparently the financial district is going to be a long day.\n",
      "sent:  I think I have a headache.\n",
      "sos the thing about science is that i don't know how to use twitter . i don't want to go to school . eos\n",
      "sent:  The thing about science is that I don't know how to use twitter.\n",
      "sent:  I don't want to go to school.\n",
      "sos i cant believe it's not going to be a good day . i hate it when i get to work . eos\n",
      "sent:  I cant believe it's not going to be a good day.\n",
      "sent:  I hate it when I get to work.\n",
      "sos i'm really good in bed . i hate being sick . i hate being sick . eos\n",
      "sent:  I'm really good in bed.\n",
      "sent:  I hate being sick.\n",
      "sent:  I hate being sick.\n",
      "sos school all day fun ! i have to go to school tomorrow . i hate being sick . eos\n",
      "sent:  School all day fun ! I have to go to school tomorrow.\n",
      "sent:  I hate being sick.\n",
      "sos i used to give anal to my dad . i hate him . i hate sleeping so much . eos\n",
      "sent:  I used to give anal to my dad.\n",
      "sent:  I hate him.\n",
      "sent:  I hate sleeping so much.\n",
      "sos fool me once , shame on you . fool me . i am better than you . eos\n",
      "sent:  Fool me once , shame on you.\n",
      "sent:  Fool me.\n",
      "sent:  I am better than you.\n",
      "sos i feel a little better now . man , i have to go to work tomorrow . eos\n",
      "sent:  I feel a little better now.\n",
      "sent:  Man , I have to go to work tomorrow.\n",
      "sos why do people have to go to school ? i don't know what to do with my life ? eos\n",
      "sent:  Why do people have to go to school ? I don't know what to do with my life ?\n",
      "sos changed my mind . im not going to be able to sleep in the morning . eos\n",
      "sent:  Changed my mind.\n",
      "sent:  Im not going to be able to sleep in the morning.\n",
      "sos ideas on what i should be doing today threedots i don't want to go to school tomorrow threedots eos\n",
      "sent:  Ideas on what I should be doing today threedots I don't want to go to school tomorrow threedots.\n",
      "sos having difficulty walking on the road and i don't want to go to school tomorrow . i hate being sick . eos\n",
      "sent:  Having difficulty walking on the road and I don't want to go to school tomorrow.\n",
      "sent:  I hate being sick.\n"
     ]
    }
   ],
   "source": [
    "for joke in joke_list:\n",
    "    print(joke[1])\n",
    "    temp = joke[1]\n",
    "    temp = temp.replace(\"sos \",\"\").replace(\" eos\",\"\")`\n",
    "    temp = re.split(\"[.]+\",temp)\n",
    "    for t in temp:\n",
    "        sent = (((t.strip().capitalize()+\".\").replace(\"?.\",\"?\")).replace(\"\\\"\",\"\")).replace(\" i \",\" I \")\n",
    "        if len(sent) >= 3:\n",
    "            print(\"sent: \",sent)\n",
    "            parser_sent.append(sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "345\n",
      "A priest , an astronaut , and a hair walk into a bar.\n",
      "The bartender says ,  what is this , some kind of joke ? .\n",
      "I'm sad after discovering my awesome headphones only to lead my life to the middle of the night.\n",
      "How many days are you ?   no , i'm not going to get a job.\n",
      "I needed a new job threedots I didn't know what to do with my hands.\n",
      "Watching x - men origins :  I don't know , but I don't know why.\n",
      "Why are black people so good at basketball ? because they always get a kick out of the crowd.\n",
      "At the coffee shop , I saw a german guy in the restaurant last night.\n",
      "He was a good guy.\n",
      "I don't drink so that i'm more likely to be a man than I am.\n",
      "We live in a time where government is not a leadership thing , but it is a shame to be a crime.\n",
      "Do you like my suit ?   I dunno , but I don't know how to get it.\n",
      "Want to hear a joke about lentil ? nevermind , I don't know , I just think it's funny.\n",
      "Going to try to stay away from the internet , but I don't want to be a celebrity.\n",
      "You should never marry a man.\n",
      "You never know what you mean.\n",
      "It's funny how time flies were found in the ocean a man would have to be a good friend.\n",
      "If pot gets legalized , I will be a man.\n",
      "I don't know what he laced them with but i've been tripping all day.\n",
      "If I ever become a zombie , I will be criminal.\n",
      "I don't know what he laced them with but i've been tripping all day.\n",
      "Idk why but gosh i'm crazy about him right now.\n",
      "I don't know what he laced them with but i've been tripping all day.\n",
      "Where the explosive diarrhea is ? the polar bear gets the same.\n",
      "Should you feel ok being broke if you don't have a job ? you have to go to the cemetery.\n",
      "I'm bored.\n",
      "Who wants to be a good person ? none , because I don't know what to do.\n",
      "Whats long and black and comes in and goes ,  I don't know , I don't know , but I don't know what he laced them with , but i've been tripping all day.\n",
      "A group of crows is called a murder.\n",
      "The police are looking for a new job.\n",
      "My favorite element is  I don't know  , but I don't know how to tell you how to get it.\n",
      "My mom calls me an older soul because , growing up , I just want to be a better person.\n",
      "I gave up my seat for a couple of hours.\n",
      "I was going to get a job for the job.\n",
      "If you drop this book in a bag of rice , you can only have one more hour of sleep.\n",
      "Class time.\n",
      "What do you call a mexican with a rubber toe ? roberto.\n",
      "Barry hoped one of the almost dozen people in the world was a great few days old , but he was a little bit of a shit.\n",
      "It has been pretty bad for my wife.\n",
      "Though , she was a real friend.\n",
      "A good deal is a lot like a box of chocolates threedots threedots but it doesn't work.\n",
      "Lazy to cook dinner so i'm in a wheelchair.\n",
      "I don't know how to drive this thing.\n",
      "All I want is a lot more fun.\n",
      "If you're not supposed to abuse me , I do it.\n",
      "This is my first time.\n",
      "People from both political parties have long recognized that yes , everyone is telling me to get out of it.\n",
      "Did you hear about the dyslexic , agnostic insomniac ? he stayed up all night wondering if there really is a dog.\n",
      "How do three gay guys walk into a bar ? one slip of the tongue and you're in deep shit.\n",
      "What's up everyone ? a : a : a lot of prank numbers are not going to be the same.\n",
      "Family is honestly the key to a good joke.\n",
      "I don't know what they say about me.\n",
      "Is writing now that I have to go to the gym and to get out of the house.\n",
      "So apparently the financial district is going to be a long term.\n",
      "I don't know what he laced them with but i've been tripping all day.\n",
      "The thing about science is that you can never be a good person.\n",
      "You have to be careful about your own skin.\n",
      "I cant believe a lot of people are like a fart.\n",
      "They don't like it that they don't like it.\n",
      "I'm really good in bed that I have no idea what my life is.\n",
      "I don't know what I have to do.\n",
      "School all day , I was going to be a kid , but I was a little bit of a kid.\n",
      "I used to give anal sex threedots but I got a lot of shit to do with it.\n",
      "Fool me once , shame on you.\n",
      "Fool me twice , but I can't tell you how much I love you.\n",
      "I feel a little better than I thought , but I was just going to have to go outside.\n",
      "Why do people have to get up in the morning ? because they don't want to be a dirty person.\n",
      "Changed my mind.\n",
      "Im not sure if I can get it.\n",
      "I could have been a lot more fun.\n",
      "Ideas on what I should do for those who have a tattoo of a seashell on their thigh.\n",
      "If you put them in a glove , you can put a ring on it.\n",
      "Having difficulty walking in the woods is usually a bit of a shock.\n",
      "A priest , an astronaut , and a hair , a piece of paper , and a piece of paper.\n",
      "I'm sad after discovering my awesome headphones only in general.\n",
      "I don't know how many times I have to go to the movies.\n",
      "How many days are engaged in a relationship , and that is not to be able to talk about it.\n",
      "I needed a new job , I would work on a show that would be terrible if you had to do it.\n",
      "Watching x - men is the most important thing in life.\n",
      "I don't know if I can do it.\n",
      "Why are black people so good at basketball ? because they don't want to be a star.\n",
      "At the coffee shop , I saw a german guy in the car , and I was like , ' oh , this is a great city.\n",
      "I don't drink so that i'm more likely to be a good person than I am.\n",
      "We live in a time where government is not a leadership thing , but it is not a weakness that is not a problem.\n",
      "Do you like my suit ? I don't think I can do that.\n",
      "I don't think I can do what I want.\n",
      "It's not something I don't want to do.\n",
      "Want to hear the truth , and the truth is that it always gets better.\n",
      "Going to try to stay away from the world is to you.\n",
      "I don't know what to do.\n",
      "You should never marry a man.\n",
      "You have to be careful about what you want to do.\n",
      "It's funny how time is spent in the car , it's just a matter of being a good driver.\n",
      "If pot gets legalized , it is not a weakness that is not a miracle.\n",
      "If I ever become a footballer to be successful , you have to be competitive.\n",
      "Interviews are not posterity.\n",
      "Idk why but gosh i'm crazy about him right now.\n",
      "I don't know why.\n",
      "I don't think anything about it.\n",
      "Where the money is , the more money you have , the more you know , the more you know , the more you know , the more you know , the more you know , the better you know.\n",
      "Should you feel ok being broke if you don't have a job , you have to be willing to accept that you don't have to work with a fear of failure.\n",
      "I'm bored.\n",
      "Who wants to be a musician or artist or a writer or a writer or a writer or a writer or a writer or a writer or a writer or a writer or a writer or a.\n",
      "Whats long and black and white and white and white and white and white and black and white and black and white and black and white and white and white and white and white and white and white and black.\n",
      "A group of crows is called a murder of the united states.\n",
      "And to be left alone is to be a man.\n",
      "My favorite element is that I don't have variety of issues that are not going to be a good thing.\n",
      "My mom calls me an older soul because , growing up , I don't want to be a celebrity.\n",
      "I gave up my seat for a couple of years , and I was a very shy kid.\n",
      "If you drop this book in a book , you may not have to go to the store and do it.\n",
      "Class time.\n",
      "I think it's a very subjective experience.\n",
      "I think it's very important to me.\n",
      "I don't know why.\n",
      "I don't know why.\n",
      "Jogging is a very important part of my life.\n",
      "Barry hoped one of the almost dozen hours of the day , and I was not a happy person.\n",
      "It has been pretty good to me , but I don't think I can do it.\n",
      "I don't want to be a celebrity.\n",
      "A good deal of make up for the rest of your life , you can still be a part of your life.\n",
      "Lazy to cook dinner so i'm in a restaurant and I don't want to eat.\n",
      "I don't want to eat.\n",
      "If you're not supposed to abuse , you can do it.\n",
      "You can do it.\n",
      "People from both political parties have long recognized that society is a free society.\n",
      "Did you hear about the dyslexic , agnostic , insomniac , agnostic , insomniac do not know who you are ? what is the most important thing ? what is the most important thing ?\n",
      "How do three gay men do not have many rod stewart in their life ?\n",
      "What's up everyone is doing , and we're all going to be together.\n",
      "I think it's a very subjective experience.\n",
      "Family is honestly the key to happiness.\n",
      "It in the end is that you have to have a lot of fun.\n",
      "Is writing now , and I don't think I can do it.\n",
      "I don't want to be a celebrity.\n",
      "I don't want to be a celebrity.\n",
      "So apparently the financial district aspect of the world is a very important part of my life.\n",
      "The thing about science is that it is not god's gift to you.\n",
      "I cant believe that I am a failure.\n",
      "I don't think I can be a failure.\n",
      "I'm really good in bed , and I won't break my heart.\n",
      "I don't want to be a star.\n",
      "School all day is a great day , and I don't want to be a successful person.\n",
      "I don't want to be a celebrity.\n",
      "I used to give anal to my father , but I was a little shy of being a kid.\n",
      "Fool me once , shame on you.\n",
      "Fool yourself , and you are not alone.\n",
      "I feel a little better about myself and I think I am a very good character.\n",
      "I think it's a very strong character.\n",
      "Why do people have to be so smart ? they are always going to be the best.\n",
      "Changed my mind.\n",
      "Im not a mother.\n",
      "I don't want to be a father.\n",
      "I don't want to be a father.\n",
      "Ideas on what I should do for myself.\n",
      "I don't care if I have a good job.\n",
      "I don't care if I have a good job.\n",
      "Having difficulty walking is a very intimate relationship.\n",
      "I think it's a very subjective experience.\n",
      "A priest , an astronaut , and a hair.\n",
      "I don't know what to do.\n",
      "I don't want to go to work.\n",
      "I'm sad after discovering my awesome headphones only it's not working.\n",
      "I don't want to go to school tomorrow.\n",
      "How many days are you ? I don't know what to do with my life.\n",
      "I needed a new phone.\n",
      "I don't want to go threedots I hate it.\n",
      "I hate it.\n",
      "Watching x - men origins : wolverine , and the movie awards.\n",
      "I love the movie awards !.\n",
      "Why are black people so good at basketball ? they are not good at all.\n",
      "At the coffee shop , I saw a german guy in my house and I was so tired.\n",
      "I don't drink so that i'm more than anything else.\n",
      "I don't want to go anywhere.\n",
      "I don't want to go to school.\n",
      "We live in a time where government is not a leadership thing , but I don't want to go to work.\n",
      "Do you like my suit ? I don't know what to do with my life.\n",
      "Want to hear about the new iphone internet but I don't know what to do.\n",
      "Going to try to stay away from the sun and see if I can go to sleep.\n",
      "Its not a good way to start the week.\n",
      "You should never marry me.\n",
      "I don't know what to do.\n",
      "I don't want to go to school.\n",
      "It's funny how time is it ? I don't know what to do with my life.\n",
      "If pot gets legalized , I don't want to go to school tomorrow tickle me.\n",
      "If I ever become a person , I don't know what to do.\n",
      "I don't want to go to sleep.\n",
      "Idk why but gosh i'm crazy about him right now.\n",
      "I hate him.\n",
      "I hate him.\n",
      "Where the hell is the sun ? I don't know what to do today.\n",
      "Should you feel ok being broke if you fall asleep ? I don't know what to do.\n",
      "I'm bored.\n",
      "Who wants to go to bed ? where is the sun ?\n",
      "Whats long and black and white and black and white and black and white and black and white and black and white and black and white and black and white and black and white and black and white and black.\n",
      "A group of crows is called a murder.\n",
      "I don't want to go to school tomorrow.\n",
      "My favorite element is not a good idea.\n",
      "I don't want to go to school tomorrow.\n",
      "My mom calls me an older soul because , growing up my heart is not what I want to do.\n",
      "I gave up my seat for a few hours.\n",
      "I don't want to go to school tomorrow.\n",
      "If you drop this book in a few days , I guess I will have to go to work tomorrow.\n",
      "Class time.\n",
      "I hate being sick.\n",
      "I hate being sick.\n",
      "Finally.\n",
      "I hate being sick.\n",
      "Barry hoped one of the almost dozen days of school is over.\n",
      "I don't even know what to do.\n",
      "It has been pretty hard to get my hair done my hair , and I don't want to go to work.\n",
      "A good deal for me.\n",
      "I am not going to free my house.\n",
      "I hate it.\n",
      "Lazy to cook dinner so i'm in a good mood.\n",
      "I hate it when I get home.\n",
      "If you're not supposed to abuse anything , you can never be able to get it.\n",
      "People from both political parties have long recognized that I don't want to go to school.\n",
      "Every time I get to see my friends.\n",
      "Did you hear about the dyslexic , agnostic , insomniac ? he was a good friend.\n",
      "But she was a good friend.\n",
      "How do three gay people get to work ? they are not going to be a good day.\n",
      "What's up everyone ? wouldn't you be pretentious ? I think I am going to be able to follow you.\n",
      "Family is honestly the key to the music.\n",
      "I don't want to go to school tomorrow.\n",
      "Is writing now.\n",
      "I have to go to work tomorrow.\n",
      "I hate being sick.\n",
      "So apparently the financial district is going to be a long day.\n",
      "I think I have a headache.\n",
      "The thing about science is that I don't know how to use twitter.\n",
      "I don't want to go to school.\n",
      "I cant believe it's not going to be a good day.\n",
      "I hate it when I get to work.\n",
      "I'm really good in bed.\n",
      "I hate being sick.\n",
      "I hate being sick.\n",
      "School all day fun ! I have to go to school tomorrow.\n",
      "I hate being sick.\n",
      "I used to give anal to my dad.\n",
      "I hate him.\n",
      "I hate sleeping so much.\n",
      "Fool me once , shame on you.\n",
      "Fool me.\n",
      "I am better than you.\n",
      "I feel a little better now.\n",
      "Man , I have to go to work tomorrow.\n",
      "Why do people have to go to school ? I don't know what to do with my life ?\n",
      "Changed my mind.\n",
      "Im not going to be able to sleep in the morning.\n",
      "Ideas on what I should be doing today threedots I don't want to go to school tomorrow threedots.\n",
      "Having difficulty walking on the road and I don't want to go to school tomorrow.\n",
      "I hate being sick.\n",
      "A priest , an astronaut , and a hair.\n",
      "I don't know what to do.\n",
      "I don't want to go to work.\n",
      "I'm sad after discovering my awesome headphones only it's not working.\n",
      "I don't want to go to school tomorrow.\n",
      "How many days are you ? I don't know what to do with my life.\n",
      "I needed a new phone.\n",
      "I don't want to go threedots I hate it.\n",
      "I hate it.\n",
      "Watching x - men origins : wolverine , and the movie awards.\n",
      "I love the movie awards !.\n",
      "Why are black people so good at basketball ? they are not good at all.\n",
      "At the coffee shop , I saw a german guy in my house and I was so tired.\n",
      "I don't drink so that i'm more than anything else.\n",
      "I don't want to go anywhere.\n",
      "I don't want to go to school.\n",
      "We live in a time where government is not a leadership thing , but I don't want to go to work.\n",
      "Do you like my suit ? I don't know what to do with my life.\n",
      "Want to hear about the new iphone internet but I don't know what to do.\n",
      "Going to try to stay away from the sun and see if I can go to sleep.\n",
      "Its not a good way to start the week.\n",
      "You should never marry me.\n",
      "I don't know what to do.\n",
      "I don't want to go to school.\n",
      "It's funny how time is it ? I don't know what to do with my life.\n",
      "If pot gets legalized , I don't want to go to school tomorrow tickle me.\n",
      "If I ever become a person , I don't know what to do.\n",
      "I don't want to go to sleep.\n",
      "Idk why but gosh i'm crazy about him right now.\n",
      "I hate him.\n",
      "I hate him.\n",
      "Where the hell is the sun ? I don't know what to do today.\n",
      "Should you feel ok being broke if you fall asleep ? I don't know what to do.\n",
      "I'm bored.\n",
      "Who wants to go to bed ? where is the sun ?\n",
      "Whats long and black and white and black and white and black and white and black and white and black and white and black and white and black and white and black and white and black and white and black.\n",
      "A group of crows is called a murder.\n",
      "I don't want to go to school tomorrow.\n",
      "My favorite element is not a good idea.\n",
      "I don't want to go to school tomorrow.\n",
      "My mom calls me an older soul because , growing up my heart is not what I want to do.\n",
      "I gave up my seat for a few hours.\n",
      "I don't want to go to school tomorrow.\n",
      "If you drop this book in a few days , I guess I will have to go to work tomorrow.\n",
      "Class time.\n",
      "I hate being sick.\n",
      "I hate being sick.\n",
      "Finally.\n",
      "I hate being sick.\n",
      "Barry hoped one of the almost dozen days of school is over.\n",
      "I don't even know what to do.\n",
      "It has been pretty hard to get my hair done my hair , and I don't want to go to work.\n",
      "A good deal for me.\n",
      "I am not going to free my house.\n",
      "I hate it.\n",
      "Lazy to cook dinner so i'm in a good mood.\n",
      "I hate it when I get home.\n",
      "If you're not supposed to abuse anything , you can never be able to get it.\n",
      "People from both political parties have long recognized that I don't want to go to school.\n",
      "Every time I get to see my friends.\n",
      "Did you hear about the dyslexic , agnostic , insomniac ? he was a good friend.\n",
      "But she was a good friend.\n",
      "How do three gay people get to work ? they are not going to be a good day.\n",
      "What's up everyone ? wouldn't you be pretentious ? I think I am going to be able to follow you.\n",
      "Family is honestly the key to the music.\n",
      "I don't want to go to school tomorrow.\n",
      "Is writing now.\n",
      "I have to go to work tomorrow.\n",
      "I hate being sick.\n",
      "So apparently the financial district is going to be a long day.\n",
      "I think I have a headache.\n",
      "The thing about science is that I don't know how to use twitter.\n",
      "I don't want to go to school.\n",
      "I cant believe it's not going to be a good day.\n",
      "I hate it when I get to work.\n",
      "I'm really good in bed.\n",
      "I hate being sick.\n",
      "I hate being sick.\n",
      "School all day fun ! I have to go to school tomorrow.\n",
      "I hate being sick.\n",
      "I used to give anal to my dad.\n",
      "I hate him.\n",
      "I hate sleeping so much.\n",
      "Fool me once , shame on you.\n",
      "Fool me.\n",
      "I am better than you.\n",
      "I feel a little better now.\n",
      "Man , I have to go to work tomorrow.\n",
      "Why do people have to go to school ? I don't know what to do with my life ?\n",
      "Changed my mind.\n",
      "Im not going to be able to sleep in the morning.\n",
      "Ideas on what I should be doing today threedots I don't want to go to school tomorrow threedots.\n",
      "Having difficulty walking on the road and I don't want to go to school tomorrow.\n",
      "I hate being sick.\n"
     ]
    }
   ],
   "source": [
    "print(len(parser_sent))\n",
    "for p in parser_sent:\n",
    "    print(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#process joke list from parser"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sos what do you do when you lose your voice ? i don't know if you can get it . eos ==> voice ? i don't know if you can get it . eos\n",
      "sos what do you do when you lose your === sos what do you do when you lose your voice ? i don't know if you can get it . eos\n",
      "0.7401539091275828 sos i don't know if you can change things , but it's a drop in the ocean . eos\n",
      "sos i'm what you might call ' internet pretty ' threedots i don't know what to do threedots eos ==> ' threedots i don't know what to do threedots eos\n",
      "sos i'm what you might call ' internet pretty === sos i'm what you might call ' internet pretty ' threedots i don't know what to do threedots eos\n",
      "0.914532364949822 sos damn i miss him and i don't know what to do threedots eos\n",
      "sos how many feminists does it take to change a light bulb ? i don't know what to do . eos ==> a light bulb ? i don't know what to do . eos\n",
      "sos how many feminists does it take to change === sos how many feminists does it take to change a light bulb ? i don't know what to do . eos\n",
      "0.919266079461024 sos i don't know what to do . i am boring eos\n",
      "sos not ready for another week to begin threedots just got back from the gym threedots eos ==> got back from the gym threedots eos\n",
      "sos not ready for another week to begin threedots just === sos not ready for another week to begin threedots just got back from the gym threedots eos\n",
      "0.7355335880127966 sos got back from the gym threedots had an awesome workout ! time for some tech eos\n",
      "sos it's sad going to mcdonald's and finding out how to get my hair done . i hate it . eos ==> how to get my hair done . i hate it . eos\n",
      "sos it's sad going to mcdonald's and finding out === sos it's sad going to mcdonald's and finding out how to get my hair done . i hate it . eos\n",
      "0.9127029929766353 sos going to get my hair done . i need blonde ! eos\n",
      "sos i really must be patient while wearing my makeup . i don't want to go to school tomorrow eos ==> makeup . i don't want to go to school tomorrow eos\n",
      "sos i really must be patient while wearing my === sos i really must be patient while wearing my makeup . i don't want to go to school tomorrow eos\n",
      "0.9589569617435278 sos getting ready for bed . i don't want to go to school tomorrow eos\n",
      "sos jon and kate threedots so sad ! clearly , i have to go to school tomorrow eos ==> , i have to go to school tomorrow eos\n",
      "sos jon and kate threedots so sad ! clearly === sos jon and kate threedots so sad ! clearly , i have to go to school tomorrow eos\n",
      "0.9035504623727352 sos time for bed . ugh i feel weird . and i have to go to school tomorrow eos\n",
      "sos why is a praying muslim like apple pie with ice cream ? i don't want to go to work . eos ==> cream ? i don't want to go to work . eos\n",
      "sos why is a praying muslim like apple pie with ice === sos why is a praying muslim like apple pie with ice cream ? i don't want to go to work . eos\n",
      "0.9566432734786126 sos i'm just in a bad mood . and i don't want to go to work . eos\n",
      "sos what's a pirate's favorite letter in the alphabet ? everyone is talking about it . eos ==> is talking about it . eos\n",
      "sos what's a pirate's favorite letter in the alphabet ? everyone === sos what's a pirate's favorite letter in the alphabet ? everyone is talking about it . eos\n",
      "0.6262935568682804 sos black holes suck . think about it . eos\n",
      "sos well that was an unsuccessful walk threedots got to go to bed threedots i hate being sick ! eos ==> go to bed threedots i hate being sick ! eos\n",
      "sos well that was an unsuccessful walk threedots got to === sos well that was an unsuccessful walk threedots got to go to bed threedots i hate being sick ! eos\n",
      "0.8442980057527317 sos i feel like crap ! i hate being sick ! eos\n",
      "sos finally i have a week off from work threedots i hate being sick threedots eos ==> threedots i hate being sick threedots eos\n",
      "sos finally i have a week off from work === sos finally i have a week off from work threedots i hate being sick threedots eos\n",
      "0.7034086880708935 sos i'm sick and can't sleep threedots i hate being sick threedots my throat hurts eos\n",
      "sos have you guys had the new lance armstrong trail ? i don't know what to do . eos ==> ? i don't know what to do . eos\n",
      "sos have you guys had the new lance armstrong trail === sos have you guys had the new lance armstrong trail ? i don't know what to do . eos\n",
      "0.9522391643216558 sos i don't know what to do . i am boring eos\n",
      "sos sunday morning , and i'm awake and drinking red bull . i hate it when i get up and i have the hours of sleep . eos ==> i hate it when i get up and i have the hours of sleep . eos\n",
      "sos sunday morning , and i'm awake and drinking red bull . === sos sunday morning , and i'm awake and drinking red bull . i hate it when i get up and i have the hours of sleep . eos\n",
      "0.6882695268465256 sos i hate it when i have to stay back ! eos\n",
      "sos i was at the track yesterday . i backed a horse at home beside me . eos ==> home beside me . eos\n",
      "sos i was at the track yesterday . i backed a horse at === sos i was at the track yesterday . i backed a horse at home beside me . eos\n",
      "0.3941185679308146 sos i have a joke . me . eos\n",
      "sos porn addiction is a serious thing . i don't want to go to school tomorrow . eos ==> . i don't want to go to school tomorrow . eos\n",
      "sos porn addiction is a serious thing === sos porn addiction is a serious thing . i don't want to go to school tomorrow . eos\n",
      "0.9445617501775281 sos just woke up threedots don't want to go to school tomorrow . eos\n",
      "sos talking from morning to night about sex has helped my skiing , because i only have one more day of school eos ==> i only have one more day of school eos\n",
      "sos talking from morning to night about sex has helped my skiing , because === sos talking from morning to night about sex has helped my skiing , because i only have one more day of school eos\n",
      "0.9615086065096015 sos glad i only have one more day of school ! eos\n",
      "sos i need to be in a better frame of my life . i don't want to go to work . eos ==> of my life . i don't want to go to work . eos\n",
      "sos i need to be in a better frame === sos i need to be in a better frame of my life . i don't want to go to work . eos\n",
      "0.9623669127426725 sos ugh . i don't want to go to work ! eos\n",
      "sos i like my coffee like i like my coffee threedots i hate those people . eos ==> like my coffee threedots i hate those people . eos\n",
      "sos i like my coffee like i === sos i like my coffee like i like my coffee threedots i hate those people . eos\n",
      "0.6559743341284773 sos i know that there are people who don't love their fellow man i hate those people . eos\n",
      "sos what did one saggy boob say to the other saggy boob ? if we don't get some support soon , we will be able to do it again eos ==> don't get some support soon , we will be able to do it again eos\n",
      "sos what did one saggy boob say to the other saggy boob ? if we === sos what did one saggy boob say to the other saggy boob ? if we don't get some support soon , we will be able to do it again eos\n",
      "0.5310619086733336 sos what did the left tit say to the right tit ? if we don't get some support soon , people are going to think we're nuts ! eos\n",
      "sos a guy said threedots in india we need quantity not to be able to do it . i love him so much . eos ==> to be able to do it . i love him so much . eos\n",
      "sos a guy said threedots in india we need quantity not === sos a guy said threedots in india we need quantity not to be able to do it . i love him so much . eos\n",
      "0.8550731867775825 sos i love him so much . but he is gone . eos\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.8080256920461416"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#score = []#sos i love my job as a road worker . i don't know what he laced them with but i've been tripping all day . eos\n",
    "score_list = []\n",
    "for seed,joke in joke_list[:30]:\n",
    "    n = len(joke)\n",
    "    joke_rep = ' '.join(joke.split()[len(seed.split()):])\n",
    "    print(joke+\" ==> \"+joke_rep)\n",
    "    print(seed,\"===\",joke)\n",
    "    score = 0\n",
    "    for sent in sentences:\n",
    "        newscore = phraseOverlap(joke_rep,sent,flist)\n",
    "        if newscore >= score:\n",
    "            score = newscore\n",
    "            match_joke = sent\n",
    "    if newscore >= 0.9:\n",
    "        print(sent,phraseOverlap(joke_rep,sent,flist))\n",
    "    print(score,match_joke)\n",
    "    \n",
    "    #print(score)\n",
    "    score_list.append(score)\n",
    "np.mean(np.array(score_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sos what do you do when you lose your voice ? i don't know if you can get it . eos ==> voice ? i don't know if you can get it . eos\n",
      "sos what do you do when you lose your === sos what do you do when you lose your voice ? i don't know if you can get it . eos\n",
      "0.13636363636363635 sos i don't know if you can change things , but it's a drop in the ocean . eos\n",
      "sos i'm what you might call ' internet pretty ' threedots i don't know what to do threedots eos ==> ' threedots i don't know what to do threedots eos\n",
      "sos i'm what you might call ' internet pretty === sos i'm what you might call ' internet pretty ' threedots i don't know what to do threedots eos\n",
      "0.38461538461538464 sos damn i miss him and i don't know what to do threedots eos\n",
      "sos how many feminists does it take to change a light bulb ? i don't know what to do . eos ==> a light bulb ? i don't know what to do . eos\n",
      "sos how many feminists does it take to change === sos how many feminists does it take to change a light bulb ? i don't know what to do . eos\n",
      "0.2857142857142857 sos feel so lonely and don't know what to do . eos\n",
      "sos not ready for another week to begin threedots just got back from the gym threedots eos ==> got back from the gym threedots eos\n",
      "sos not ready for another week to begin threedots just === sos not ready for another week to begin threedots just got back from the gym threedots eos\n",
      "0.2 sos got back from the gym threedots had an awesome workout ! time for some tech eos\n",
      "sos it's sad going to mcdonald's and finding out how to get my hair done . i hate it . eos ==> how to get my hair done . i hate it . eos\n",
      "sos it's sad going to mcdonald's and finding out === sos it's sad going to mcdonald's and finding out how to get my hair done . i hate it . eos\n",
      "0.26666666666666666 sos going to get my hair done . i need blonde ! eos\n",
      "sos i really must be patient while wearing my makeup . i don't want to go to school tomorrow eos ==> makeup . i don't want to go to school tomorrow eos\n",
      "sos i really must be patient while wearing my === sos i really must be patient while wearing my makeup . i don't want to go to school tomorrow eos\n",
      "0.5384615384615384 sos getting ready for bed . i don't want to go to school tomorrow eos\n",
      "sos jon and kate threedots so sad ! clearly , i have to go to school tomorrow eos ==> , i have to go to school tomorrow eos\n",
      "sos jon and kate threedots so sad ! clearly === sos jon and kate threedots so sad ! clearly , i have to go to school tomorrow eos\n",
      "0.3125 sos the only bad thing is , i have to go to school tomorrow in gorgeous weather eos\n",
      "sos why is a praying muslim like apple pie with ice cream ? i don't want to go to work . eos ==> cream ? i don't want to go to work . eos\n",
      "sos why is a praying muslim like apple pie with ice === sos why is a praying muslim like apple pie with ice cream ? i don't want to go to work . eos\n",
      "0.35294117647058826 sos i'm just in a bad mood . and i don't want to go to work . eos\n",
      "sos what's a pirate's favorite letter in the alphabet ? everyone is talking about it . eos ==> is talking about it . eos\n",
      "sos what's a pirate's favorite letter in the alphabet ? everyone === sos what's a pirate's favorite letter in the alphabet ? everyone is talking about it . eos\n",
      "0.1111111111111111 sos writers write . dreamers talk about it . eos\n",
      "sos well that was an unsuccessful walk threedots got to go to bed threedots i hate being sick ! eos ==> go to bed threedots i hate being sick ! eos\n",
      "sos well that was an unsuccessful walk threedots got to === sos well that was an unsuccessful walk threedots got to go to bed threedots i hate being sick ! eos\n",
      "0.23529411764705882 sos my head hurts sooo bad right now ! ugh threedots i hate being sick ! eos\n",
      "sos finally i have a week off from work threedots i hate being sick threedots eos ==> threedots i hate being sick threedots eos\n",
      "sos finally i have a week off from work === sos finally i have a week off from work threedots i hate being sick threedots eos\n",
      "0.21428571428571427 sos i'm sick and can't sleep threedots i hate being sick threedots my throat hurts eos\n",
      "sos have you guys had the new lance armstrong trail ? i don't know what to do . eos ==> ? i don't know what to do . eos\n",
      "sos have you guys had the new lance armstrong trail === sos have you guys had the new lance armstrong trail ? i don't know what to do . eos\n",
      "0.36363636363636365 sos feel so lonely and don't know what to do . eos\n",
      "sos sunday morning , and i'm awake and drinking red bull . i hate it when i get up and i have the hours of sleep . eos ==> i hate it when i get up and i have the hours of sleep . eos\n",
      "sos sunday morning , and i'm awake and drinking red bull . === sos sunday morning , and i'm awake and drinking red bull . i hate it when i get up and i have the hours of sleep . eos\n",
      "0.10714285714285714 sos i hate it when i get new stuff and can't even touch it til the end of the day eos\n",
      "sos i was at the track yesterday . i backed a horse at home beside me . eos ==> home beside me . eos\n",
      "sos i was at the track yesterday . i backed a horse at === sos i was at the track yesterday . i backed a horse at home beside me . eos\n",
      "0.0 sos end of the weekend i just want it to be summer and no classes or work threedots ha ! like that will happen ! eos\n",
      "sos porn addiction is a serious thing . i don't want to go to school tomorrow . eos ==> . i don't want to go to school tomorrow . eos\n",
      "sos porn addiction is a serious thing === sos porn addiction is a serious thing . i don't want to go to school tomorrow . eos\n",
      "0.46153846153846156 sos just woke up threedots don't want to go to school tomorrow . eos\n",
      "sos talking from morning to night about sex has helped my skiing , because i only have one more day of school eos ==> i only have one more day of school eos\n",
      "sos talking from morning to night about sex has helped my skiing , because === sos talking from morning to night about sex has helped my skiing , because i only have one more day of school eos\n",
      "0.5 sos glad i only have one more day of school ! eos\n",
      "sos i need to be in a better frame of my life . i don't want to go to work . eos ==> of my life . i don't want to go to work . eos\n",
      "sos i need to be in a better frame === sos i need to be in a better frame of my life . i don't want to go to work . eos\n",
      "0.35714285714285715 sos ugh . i don't want to go to work ! eos\n",
      "sos i like my coffee like i like my coffee threedots i hate those people . eos ==> like my coffee threedots i hate those people . eos\n",
      "sos i like my coffee like i === sos i like my coffee like i like my coffee threedots i hate those people . eos\n",
      "0.15 sos i know that there are people who don't love their fellow man i hate those people . eos\n",
      "sos what did one saggy boob say to the other saggy boob ? if we don't get some support soon , we will be able to do it again eos ==> don't get some support soon , we will be able to do it again eos\n",
      "sos what did one saggy boob say to the other saggy boob ? if we === sos what did one saggy boob say to the other saggy boob ? if we don't get some support soon , we will be able to do it again eos\n",
      "0.09090909090909091 sos well heading home from an amazing weekend cant wait to do it again eos\n",
      "sos a guy said threedots in india we need quantity not to be able to do it . i love him so much . eos ==> to be able to do it . i love him so much . eos\n",
      "sos a guy said threedots in india we need quantity not === sos a guy said threedots in india we need quantity not to be able to do it . i love him so much . eos\n",
      "0.16666666666666666 sos i miss my boyfriend tremendously , every time he leaves my house i love him so much . eos\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.2617494964186141"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#score = []#sos i love my job as a road worker . i don't know what he laced them with but i've been tripping all day . eos\n",
    "score_list = []\n",
    "for seed,joke in joke_list[:20]:\n",
    "    n = len(joke)\n",
    "    joke_rep = ' '.join(joke.split()[len(seed.split()):])\n",
    "    print(joke+\" ==> \"+joke_rep)\n",
    "    print(seed,\"===\",joke)\n",
    "    score = 0\n",
    "    for sent in sentences:\n",
    "        newscore = k_jaccard(joke_rep,sent,4)\n",
    "        if newscore >= score:\n",
    "            score = newscore\n",
    "            match_joke = sent\n",
    "    if newscore >= 0.9:\n",
    "        print(sent,k_jaccard(joke_rep,sent,4))\n",
    "    print(score,match_joke)\n",
    "    \n",
    "    #print(score)\n",
    "    score_list.append(score)\n",
    "np.mean(np.array(score_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}