LSTM with Word Embeddings for Combined data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Following https://www.kaggle.com/qqgeogor/keras-lstm-attention-glove840b-lb-0-043"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "import codecs\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle \n",
    "\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import SnowballStemmer\n",
    "from string import punctuation\n",
    "\n",
    "from gensim.models import KeyedVectors\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation,Bidirectional\n",
    "from keras.layers.merge import concatenate\n",
    "from keras.models import Model\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.callbacks import EarlyStopping, ModelCheckpoint\n",
    "from keras.metrics import categorical_accuracy\n",
    "\n",
    "from keras import backend as K\n",
    "from keras.engine.topology import Layer\n",
    "#from keras import initializations\n",
    "from keras import initializers, regularizers, constraints"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Attention(Layer):\n",
    "    def __init__(self, step_dim,\n",
    "                 W_regularizer=None, b_regularizer=None,\n",
    "                 W_constraint=None, b_constraint=None,\n",
    "                 bias=True, **kwargs):\n",
    "        \"\"\"\n",
    "        Keras Layer that implements an Attention mechanism for temporal data.\n",
    "        Supports Masking.\n",
    "        Follows the work of Raffel et al. [https://arxiv.org/abs/1512.08756]\n",
    "        # Input shape\n",
    "            3D tensor with shape: `(samples, steps, features)`.\n",
    "        # Output shape\n",
    "            2D tensor with shape: `(samples, features)`.\n",
    "        :param kwargs:\n",
    "        Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n",
    "        The dimensions are inferred based on the output shape of the RNN.\n",
    "        Example:\n",
    "            model.add(LSTM(64, return_sequences=True))\n",
    "            model.add(Attention())\n",
    "        \"\"\"\n",
    "        self.supports_masking = True\n",
    "        #self.init = initializations.get('glorot_uniform')\n",
    "        self.init = initializers.get('glorot_uniform')\n",
    "\n",
    "        self.W_regularizer = regularizers.get(W_regularizer)\n",
    "        self.b_regularizer = regularizers.get(b_regularizer)\n",
    "\n",
    "        self.W_constraint = constraints.get(W_constraint)\n",
    "        self.b_constraint = constraints.get(b_constraint)\n",
    "\n",
    "        self.bias = bias\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert len(input_shape) == 3\n",
    "\n",
    "        self.W = self.add_weight((input_shape[-1],),\n",
    "                                 initializer=self.init,\n",
    "                                 name='{}_W'.format(self.name),\n",
    "                                 regularizer=self.W_regularizer,\n",
    "                                 constraint=self.W_constraint)\n",
    "        self.features_dim = input_shape[-1]\n",
    "\n",
    "        if self.bias:\n",
    "            self.b = self.add_weight((input_shape[1],),\n",
    "                                     initializer='zero',\n",
    "                                     name='{}_b'.format(self.name),\n",
    "                                     regularizer=self.b_regularizer,\n",
    "                                     constraint=self.b_constraint)\n",
    "        else:\n",
    "            self.b = None\n",
    "\n",
    "        self.built = True\n",
    "\n",
    "    def compute_mask(self, input, input_mask=None):\n",
    "        # do not pass the mask to the next layers\n",
    "        return None\n",
    "\n",
    "    def call(self, x, mask=None):\n",
    "        # eij = K.dot(x, self.W) TF backend doesn't support it\n",
    "\n",
    "        # features_dim = self.W.shape[0]\n",
    "        # step_dim = x._keras_shape[1]\n",
    "\n",
    "        features_dim = self.features_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n",
    "\n",
    "        if self.bias:\n",
    "            eij += self.b\n",
    "\n",
    "        eij = K.tanh(eij)\n",
    "\n",
    "        a = K.exp(eij)\n",
    "\n",
    "        # apply mask after the exp. will be re-normalized next\n",
    "        if mask is not None:\n",
    "            # Cast the mask to floatX to avoid float64 upcasting in theano\n",
    "            a *= K.cast(mask, K.floatx())\n",
    "\n",
    "        # in some cases especially in the early stages of training the sum may be almost zero\n",
    "        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n",
    "\n",
    "        a = K.expand_dims(a)\n",
    "        weighted_input = x * a\n",
    "    #print weigthted_input.shape\n",
    "        return K.sum(weighted_input, axis=1)\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        #return input_shape[0], input_shape[-1]\n",
    "        return input_shape[0],  self.features_dim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## set directories and parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "EMBEDDING_FILE = \"./data/glove/glove.6B.100d.txt\"\n",
    "TRAIN_DATA_FILE = \"./datasets/combined.pickle\"\n",
    "VOCABULARY_FILE = \"./datasets/combined_vocabulary.pickle\"\n",
    "MAX_SEQUENCE_LENGTH = 10\n",
    "MAX_NB_WORDS = 200000\n",
    "EMBEDDING_DIM = 100\n",
    "SEQUENCE_STEP = 1\n",
    "#VALIDATION_SPLIT = 0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_lstm = 300\n",
    "num_dense = 256\n",
    "rate_drop_lstm = 0.25\n",
    "rate_drop_dense = 0.25\n",
    "\n",
    "act = 'relu'\n",
    "#re_weight = True # whether to re-weight classes to fit the 17.5% share in test set\n",
    "\n",
    "STAMP = 'lstm_%d_%d_%.2f_%.2f'%(num_lstm, num_dense, rate_drop_lstm, \\\n",
    "        rate_drop_dense)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Glove vectors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Indexing word vectors\n"
     ]
    }
   ],
   "source": [
    "print('Indexing word vectors')\n",
    "\n",
    "#Glove Vectors\n",
    "embeddings_index = {}\n",
    "f = open(EMBEDDING_FILE)\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "print('Total %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "print('Total %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#training jokes/quotes:  270807\n",
      "#Vocabulary  12644\n"
     ]
    }
   ],
   "source": [
    "data = pickle.load(open(TRAIN_DATA_FILE,\"rb\"))\n",
    "vocabulary = pickle.load(open(VOCABULARY_FILE,\"rb\"))\n",
    "print(\"#training jokes/quotes: \",len(data))\n",
    "print(\"#Vocabulary \",len(vocabulary))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check OOV words that are not present in glove vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#non vocabulary words:  300\n",
      "[\"someone's\", \"daughter's\", \"mother's\", 'crossfitter', \"i'd\", \"joke's\", \"parkinson's\", \"aren't\", 'motherfucking', \"husband's\", 'titties', \"body's\", \"don't\", 'redditor', 'brexit', \"couldn't\", \"why'd\", 'badum', \"how'd\", \"everybody's\", 'pornstars', 'dumbass', 'rubbit', \"it'll\", \"dad's\", \"son's\", 'tifu', 'selfie', 'roamin', \"we're\", \"he'd\", \"horse's\", \"she'll\", \"why's\", 'cums', \"cosby's\", \"must've\", \"year's\", \"who've\", \"can't\", \"life's\", 'unfollow', \"kardashian's\", \"rubik's\", \"cat's\", \"hasn't\", \"everyone's\", 'reposts', \"women's\", 'hodor', \"a's\", \"what'll\", 'upvotes', \"china's\", \"we've\", \"mama's\", \"dentist's\", \"wonder's\", 'idk', \"mexican's\", \"could've\", 'pussies', \"we'll\", \"neighbor's\", 'sleevies', \"boy's\", \"cow's\", 'cking', \"apple's\", \"o'furniture\", \"should've\", \"america's\", 'moaner', \"they'd\", \"guy's\", \"people's\", \"korea's\", \"tonight's\", \"wife's\", \"didn't\", \"wasn't\", \"dracula's\", \"you're\", \"neckbeard's\", \"schrodinger's\", \"dog's\", 'e.u', \"i've\", \"i'm\", \"kid's\", \"wouldn't\", \"god's\", 'emoji', \"y'all\", \"arthur's\", \"mario's\", 'upvote', \"state's\", \"somebody's\", \"music's\", \"writer's\", \"father's\", \"another's\", \"beethoven's\", \"who'd\", 'downvote', \"night's\", \"they're\", \"pirate's\", \"character's\", \"lee's\", 'beiber', 'threedots', 'oeuf', \"jackson's\", \"victoria's\", \"baby's\", \"nothing's\", 'shits', \"society's\", 'blowjobs', \"won't\", \"psychiatrist's\", \"devil's\", 'farted', 'snackbar', \"mine's\", 'turds', \"where'd\", \"man's\", \"would've\", 'friendzone', \"grandma's\", \"president's\", \"public's\", \"it's\", \"poet's\", \"yesterday's\", \"girlfriend's\", \"let's\", \"haven't\", 'unfollowed', \"child's\", 'redditors', \"parent's\", \"heart's\", \"mcdonald's\", \"something's\", \"today's\", \"anyone's\", \"government's\", \"fly's\", \"momma's\", 'updog', 'hahahaha', \"wendy's\", 'blowjob', 'impasta', 'damnit', 'hahaha', 'twerking', \"children's\", 'tweetment', \"isn't\", \"i'll\", \"family's\", \"pedophile's\", \"that'll\", 'shitting', \"hillary's\", 'arrr', \"world's\", \"nature's\", \"film's\", \"love's\", \"king's\", \"palin's\", \"other's\", \"money's\", \"weren't\", \"else's\", \"bush's\", \"soul's\", \"person's\", \"car's\", \"actor's\", \"day's\", \"reddit's\", \"here's\", \"anybody's\", 'tsss', 'tauntaun', \"tourette's\", \"elephant's\", 'cumference', \"frankenstein's\", 'lmao', 'labracadabrador', 'harambe', \"ghost's\", \"nation's\", 'walmarts', \"santa's\", \"he'll\", \"britain's\", \"individual's\", 'pornhub', \"woman's\", 'retweet', \"farmer's\", 'shitzu', \"they've\", \"chicken's\", 'clickbait', 'tennish', \"mind's\", \"adam's\", \"where's\", \"what'd\", \"dick's\", \"country's\", \"men's\", \"blonde's\", \"mum's\", \"friend's\", \"she's\", \"jehovah's\", 'hurty', 'neckbeard', \"artist's\", \"it'd\", \"clinton's\", \"keller's\", \"you'd\", 'irrelephant', 'tearable', 'douchebag', \"audience's\", \"how's\", \"ex's\", \"girl's\", \"ain't\", \"mustn't\", \"you've\", \"earth's\", \"they'll\", \"nazi's\", \"vampire's\", \"grandfather's\", \"name's\", \"trump's\", \"boyfriend's\", \"kermit's\", \"one's\", \"mom's\", 'pubes', \"who's\", \"patrick's\", 'necrophiliacs', \"obama's\", 'subreddit', \"he's\", 'kylo', 'selfies', \"that's\", \"driver's\", 'wanking', 'snowwoman', \"she'd\", \"alzheimer's\", \"batman's\", 'beastiality', 'handjob', \"you'll\", \"teacher's\", \"hitler's\", \"there's\", \"shouldn't\", \"art's\", 'neverlands', \"cannibal's\", \"everything's\", \"we'd\", \"doctor's\", \"brother's\", \"israel's\", \"sister's\", \"what's\", 'nsfw', \"doesn't\", 'spaghetto', 'sinko', \"hadn't\", \"valentine's\", \"director's\", \"houston's\", \"there'll\", \"frog's\", 'teethbrush', \"nobody's\"]\n"
     ]
    }
   ],
   "source": [
    "oov = []\n",
    "for word in vocabulary:\n",
    "    if word not in embeddings_index:\n",
    "        oov.append(word)\n",
    "print(\"#non vocabulary words: \",len(oov))\n",
    "print(oov[:max(len(oov),100)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reducing data for faster iterations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "percentage = 0.001\n",
    "data_reduced = data[:int(percentage*len(data))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenizing Sentences to create sequence of inetgers and building word to id mappings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda/envs/py35/lib/python3.5/site-packages/keras/preprocessing/text.py:157: UserWarning: The `nb_words` argument in `Tokenizer` has been renamed `num_words`.\n",
      "  warnings.warn('The `nb_words` argument in `Tokenizer` '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 7150 unique tokens.\n",
      "9419 9419\n"
     ]
    }
   ],
   "source": [
    "tokenizer = Tokenizer(nb_words=MAX_NB_WORDS)\n",
    "tokenizer.fit_on_texts(data_reduced)\n",
    "sequences = tokenizer.texts_to_sequences(data_reduced)\n",
    "#word to index\n",
    "word_index = tokenizer.word_index\n",
    "#index to word\n",
    "index_word = {}\n",
    "for k,v in word_index.items():\n",
    "    index_word[v] = k\n",
    "print('Found %s unique tokens.' % len(word_index))\n",
    "print(len(sequences),len(data_reduced))\n",
    "assert len(data_reduced) == len(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#Total Sequence Length: 173427\n"
     ]
    }
   ],
   "source": [
    "nb_words = min(MAX_NB_WORDS, len(word_index))+1 #Vocabulary size\n",
    "word_index_list = []\n",
    "for sentence in sequences:\n",
    "    word_index_list += sentence\n",
    "print(\"#Total Sequence Length:\", len(word_index_list))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare training data sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 10\n",
      "nb sequences: 173417\n",
      "(173417, 10) (173417, 7151)\n"
     ]
    }
   ],
   "source": [
    "#create sequences\n",
    "X_train = []\n",
    "y_train = np.zeros((len(word_index_list)-MAX_SEQUENCE_LENGTH,nb_words),dtype=np.bool)\n",
    "print(SEQUENCE_STEP,MAX_SEQUENCE_LENGTH)\n",
    "for i in range(0, len(word_index_list) - MAX_SEQUENCE_LENGTH, SEQUENCE_STEP):\n",
    "    X_train.append(word_index_list[i: i + MAX_SEQUENCE_LENGTH])\n",
    "    y_train[i][word_index_list[i + MAX_SEQUENCE_LENGTH]] = 1\n",
    "X_train = np.array(X_train)\n",
    "print('nb sequences:', len(X_train))\n",
    "print(X_train.shape,y_train.shape)\n",
    "assert X_train.shape[0] == y_train.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## prepare embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Preparing embedding matrix\n",
      "Null word embeddings: 91\n"
     ]
    }
   ],
   "source": [
    "print('Preparing embedding matrix')\n",
    "embedding_matrix = np.zeros((nb_words, EMBEDDING_DIM))\n",
    "for word,i in word_index.items():\n",
    "    if i >= MAX_NB_WORDS:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector\n",
    "\n",
    "print('Null word embeddings: %d' % np.sum(np.sum(embedding_matrix, axis=1) == 0))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define the model structure"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_9 (InputLayer)         (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "embedding_9 (Embedding)      (None, 10, 100)           715100    \n",
      "_________________________________________________________________\n",
      "bidirectional_9 (Bidirection (None, 10, 600)           962400    \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (None, 10, 600)           0         \n",
      "_________________________________________________________________\n",
      "attention_4 (Attention)      (None, 600)               610       \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 7151)              4297751   \n",
      "=================================================================\n",
      "Total params: 5,975,861\n",
      "Trainable params: 5,260,761\n",
      "Non-trainable params: 715,100\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "embedding_layer = Embedding(nb_words,\n",
    "        EMBEDDING_DIM,\n",
    "        weights=[embedding_matrix],\n",
    "        input_length=MAX_SEQUENCE_LENGTH,\n",
    "        trainable=False)\n",
    "lstm_layer = Bidirectional(LSTM(num_lstm, activation=\"relu\",return_sequences=True,dropout=rate_drop_lstm, recurrent_dropout=rate_drop_lstm),input_shape=(MAX_SEQUENCE_LENGTH,EMBEDDING_DIM))\n",
    "data_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n",
    "embedded_sequences= embedding_layer(data_input)\n",
    "x = lstm_layer(embedded_sequences)\n",
    "x = Dropout(rate_drop_dense)(x)\n",
    "#Add attention\n",
    "x = Attention(MAX_SEQUENCE_LENGTH)(x)\n",
    "preds = Dense(nb_words, activation='softmax')(x)\n",
    "model = Model(inputs=[data_input], \\\n",
    "        outputs=preds)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "        optimizer='rmsprop',\n",
    "        metrics=[categorical_accuracy])\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "simple_lstm_glove_vectors_0.25_0.25\n"
     ]
    }
   ],
   "source": [
    "STAMP = 'simple_lstm_glove_vectors_%.2f_%.2f'%(rate_drop_lstm,rate_drop_dense)\n",
    "print(STAMP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "early_stopping =EarlyStopping(monitor='val_loss', patience=2)\n",
    "bst_model_path = STAMP + '.h5'\n",
    "model_checkpoint = ModelCheckpoint(bst_model_path, save_best_only=True, save_weights_only=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 156075 samples, validate on 17342 samples\n",
      "Epoch 1/5\n"
     ]
    },
    {
     "ename": "ResourceExhaustedError",
     "evalue": "OOM when allocating tensor with shape[256,300] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[Node: bidirectional_9/while_1/mul_7 = Mul[T=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](bidirectional_9/while_1/Switch_2:1, bidirectional_9/while_1/mul_7/Enter)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\t [[Node: metrics_1/categorical_accuracy/Mean/_241 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device_incarnation=1, tensor_name=\"edge_4054_metrics_1/categorical_accuracy/Mean\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"]()]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\nCaused by op 'bidirectional_9/while_1/mul_7', defined at:\n  File \"/anaconda/envs/py35/lib/python3.5/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/anaconda/envs/py35/lib/python3.5/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/kernelapp.py\", line 478, in start\n    self.io_loop.start()\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tornado/ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2728, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2850, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2910, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-39-7835c0cf51e2>\", line 9, in <module>\n    x = lstm_layer(embedded_sequences)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/wrappers.py\", line 324, in __call__\n    return super(Bidirectional, self).__call__(inputs, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/engine/topology.py\", line 617, in __call__\n    output = self.call(inputs, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/wrappers.py\", line 385, in call\n    y_rev = self.backward_layer.call(inputs, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/recurrent.py\", line 2063, in call\n    initial_state=initial_state)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/recurrent.py\", line 608, in call\n    input_length=timesteps)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\", line 2764, in rnn\n    swap_memory=True)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 3096, in while_loop\n    result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2874, in BuildLoop\n    pred, body, original_loop_vars, loop_vars, shape_invariants)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2814, in _BuildLoop\n    body_result = body(*packed_vars_for_body)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\", line 2750, in _step\n    tuple(constants))\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/recurrent.py\", line 599, in step\n    return self.cell.call(inputs, states, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/recurrent.py\", line 1846, in call\n    h_tm1_o = h_tm1 * rec_dp_mask[3]\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/ops/math_ops.py\", line 934, in binary_op_wrapper\n    return func(x, y, name=name)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/ops/math_ops.py\", line 1161, in _mul_dispatch\n    return gen_math_ops._mul(x, y, name=name)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/ops/gen_math_ops.py\", line 2789, in _mul\n    \"Mul\", x=x, y=y, name=name)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 3271, in create_op\n    op_def=op_def)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 1650, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[256,300] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[Node: bidirectional_9/while_1/mul_7 = Mul[T=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](bidirectional_9/while_1/Switch_2:1, bidirectional_9/while_1/mul_7/Enter)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\t [[Node: metrics_1/categorical_accuracy/Mean/_241 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device_incarnation=1, tensor_name=\"edge_4054_metrics_1/categorical_accuracy/Mean\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"]()]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1360\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1361\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1362\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1339\u001b[0m           return tf_session.TF_Run(session, options, feed_dict, fetch_list,\n\u001b[0;32m-> 1340\u001b[0;31m                                    target_list, status, run_metadata)\n\u001b[0m\u001b[1;32m   1341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/framework/errors_impl.py\u001b[0m in \u001b[0;36m__exit__\u001b[0;34m(self, type_arg, value_arg, traceback_arg)\u001b[0m\n\u001b[1;32m    515\u001b[0m             \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mc_api\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_Message\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 516\u001b[0;31m             c_api.TF_GetCode(self.status.status))\n\u001b[0m\u001b[1;32m    517\u001b[0m     \u001b[0;31m# Delete the underlying status object from memory otherwise it stays alive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[256,300] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[Node: bidirectional_9/while_1/mul_7 = Mul[T=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](bidirectional_9/while_1/Switch_2:1, bidirectional_9/while_1/mul_7/Enter)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\t [[Node: metrics_1/categorical_accuracy/Mean/_241 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device_incarnation=1, tensor_name=\"edge_4054_metrics_1/categorical_accuracy/Mean\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"]()]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m                    Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-43-6534c4791254>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m hist = model.fit(X_train,y_train,         epochs=5, batch_size=256, shuffle=True,          callbacks=[early_stopping, model_checkpoint],\n\u001b[0;32m----> 2\u001b[0;31m                 validation_split=0.1)\n\u001b[0m",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)\u001b[0m\n\u001b[1;32m   1710\u001b[0m                               \u001b[0minitial_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minitial_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1711\u001b[0m                               \u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0msteps_per_epoch\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1712\u001b[0;31m                               validation_steps=validation_steps)\n\u001b[0m\u001b[1;32m   1713\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1714\u001b[0m     def evaluate(self, x=None, y=None,\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/keras/engine/training.py\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)\u001b[0m\n\u001b[1;32m   1233\u001b[0m                         \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mins_batch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1234\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1235\u001b[0;31m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1236\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1237\u001b[0m                         \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   2473\u001b[0m         \u001b[0msession\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2474\u001b[0m         updated = session.run(fetches=fetches, feed_dict=feed_dict,\n\u001b[0;32m-> 2475\u001b[0;31m                               **self.session_kwargs)\n\u001b[0m\u001b[1;32m   2476\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mupdated\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2477\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    903\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 905\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    906\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1135\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1136\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1137\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1138\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1139\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1353\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1354\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1355\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1356\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1357\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1372\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1373\u001b[0m           \u001b[0;32mpass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1374\u001b[0;31m       \u001b[0;32mraise\u001b[0m \u001b[0mtype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_def\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1375\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1376\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mResourceExhaustedError\u001b[0m: OOM when allocating tensor with shape[256,300] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[Node: bidirectional_9/while_1/mul_7 = Mul[T=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](bidirectional_9/while_1/Switch_2:1, bidirectional_9/while_1/mul_7/Enter)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\t [[Node: metrics_1/categorical_accuracy/Mean/_241 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device_incarnation=1, tensor_name=\"edge_4054_metrics_1/categorical_accuracy/Mean\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"]()]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\nCaused by op 'bidirectional_9/while_1/mul_7', defined at:\n  File \"/anaconda/envs/py35/lib/python3.5/runpy.py\", line 193, in _run_module_as_main\n    \"__main__\", mod_spec)\n  File \"/anaconda/envs/py35/lib/python3.5/runpy.py\", line 85, in _run_code\n    exec(code, run_globals)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/__main__.py\", line 3, in <module>\n    app.launch_new_instance()\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/traitlets/config/application.py\", line 658, in launch_instance\n    app.start()\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/kernelapp.py\", line 478, in start\n    self.io_loop.start()\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/zmq/eventloop/ioloop.py\", line 177, in start\n    super(ZMQIOLoop, self).start()\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tornado/ioloop.py\", line 888, in start\n    handler_func(fd_obj, events)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 440, in _handle_events\n    self._handle_recv()\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 472, in _handle_recv\n    self._run_callback(callback, msg)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/zmq/eventloop/zmqstream.py\", line 414, in _run_callback\n    callback(*args, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tornado/stack_context.py\", line 277, in null_wrapper\n    return fn(*args, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 283, in dispatcher\n    return self.dispatch_shell(stream, msg)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 233, in dispatch_shell\n    handler(stream, idents, msg)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/kernelbase.py\", line 399, in execute_request\n    user_expressions, allow_stdin)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/ipkernel.py\", line 208, in do_execute\n    res = shell.run_cell(code, store_history=store_history, silent=silent)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/ipykernel/zmqshell.py\", line 537, in run_cell\n    return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2728, in run_cell\n    interactivity=interactivity, compiler=compiler, result=result)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2850, in run_ast_nodes\n    if self.run_code(code, result):\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/IPython/core/interactiveshell.py\", line 2910, in run_code\n    exec(code_obj, self.user_global_ns, self.user_ns)\n  File \"<ipython-input-39-7835c0cf51e2>\", line 9, in <module>\n    x = lstm_layer(embedded_sequences)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/wrappers.py\", line 324, in __call__\n    return super(Bidirectional, self).__call__(inputs, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/engine/topology.py\", line 617, in __call__\n    output = self.call(inputs, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/wrappers.py\", line 385, in call\n    y_rev = self.backward_layer.call(inputs, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/recurrent.py\", line 2063, in call\n    initial_state=initial_state)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/recurrent.py\", line 608, in call\n    input_length=timesteps)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\", line 2764, in rnn\n    swap_memory=True)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 3096, in while_loop\n    result = loop_context.BuildLoop(cond, body, loop_vars, shape_invariants)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2874, in BuildLoop\n    pred, body, original_loop_vars, loop_vars, shape_invariants)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/ops/control_flow_ops.py\", line 2814, in _BuildLoop\n    body_result = body(*packed_vars_for_body)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/backend/tensorflow_backend.py\", line 2750, in _step\n    tuple(constants))\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/recurrent.py\", line 599, in step\n    return self.cell.call(inputs, states, **kwargs)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/keras/layers/recurrent.py\", line 1846, in call\n    h_tm1_o = h_tm1 * rec_dp_mask[3]\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/ops/math_ops.py\", line 934, in binary_op_wrapper\n    return func(x, y, name=name)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/ops/math_ops.py\", line 1161, in _mul_dispatch\n    return gen_math_ops._mul(x, y, name=name)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/ops/gen_math_ops.py\", line 2789, in _mul\n    \"Mul\", x=x, y=y, name=name)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py\", line 787, in _apply_op_helper\n    op_def=op_def)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 3271, in create_op\n    op_def=op_def)\n  File \"/anaconda/envs/py35/lib/python3.5/site-packages/tensorflow/python/framework/ops.py\", line 1650, in __init__\n    self._traceback = self._graph._extract_stack()  # pylint: disable=protected-access\n\nResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[256,300] and type float on /job:localhost/replica:0/task:0/device:GPU:0 by allocator GPU_0_bfc\n\t [[Node: bidirectional_9/while_1/mul_7 = Mul[T=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:GPU:0\"](bidirectional_9/while_1/Switch_2:1, bidirectional_9/while_1/mul_7/Enter)]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n\t [[Node: metrics_1/categorical_accuracy/Mean/_241 = _Recv[client_terminated=false, recv_device=\"/job:localhost/replica:0/task:0/device:CPU:0\", send_device=\"/job:localhost/replica:0/task:0/device:GPU:0\", send_device_incarnation=1, tensor_name=\"edge_4054_metrics_1/categorical_accuracy/Mean\", tensor_type=DT_FLOAT, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"]()]]\nHint: If you want to see a list of allocated tensors when OOM happens, add report_tensor_allocations_upon_oom to RunOptions for current allocation info.\n\n"
     ]
    }
   ],
   "source": [
    "hist = model.fit(X_train,y_train, \\\n",
    "        epochs=5, batch_size=256, shuffle=True, \\\n",
    "         callbacks=[early_stopping, model_checkpoint],\n",
    "                validation_split=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights(bst_model_path)\n",
    "bst_val_score = min(hist.history['val_loss'])\n",
    "print(bst_val_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Test Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict = model.predict([X_train[0:10]])\n",
    "idx = np.argmax(predict,axis=1)\n",
    "for id in idx:\n",
    "    print(index_word[id])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py35]",
   "language": "python",
   "name": "conda-env-py35-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}