text-classification/44.capsule-birrn-seq2seq-lstm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from utils import *\n",
    "import tensorflow as tf\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['negative', 'positive']\n",
      "10662\n",
      "10662\n"
     ]
    }
   ],
   "source": [
    "trainset = sklearn.datasets.load_files(container_path = 'data', encoding = 'UTF-8')\n",
    "trainset.data, trainset.target = separate_dataset(trainset,1.0)\n",
    "print (trainset.target_names)\n",
    "print (len(trainset.data))\n",
    "print (len(trainset.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "ONEHOT = np.zeros((len(trainset.data),len(trainset.target_names)))\n",
    "ONEHOT[np.arange(len(trainset.data)),trainset.target] = 1.0\n",
    "train_X, test_X, train_Y, test_Y, train_onehot, test_onehot = train_test_split(trainset.data, \n",
    "                                                                               trainset.target, \n",
    "                                                                               ONEHOT, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 20465\n",
      "Most common words [('the', 10129), ('a', 7312), ('and', 6199), ('of', 6063), ('to', 4233), ('is', 3378)]\n",
      "Sample data [4, 660, 9, 2553, 8, 22, 4, 3326, 17188, 98] ['the', 'rock', 'is', 'destined', 'to', 'be', 'the', '21st', 'centurys', 'new']\n"
     ]
    }
   ],
   "source": [
    "concat = ' '.join(trainset.data).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary['GO']\n",
    "PAD = dictionary['PAD']\n",
    "EOS = dictionary['EOS']\n",
    "UNK = dictionary['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def squash(X, epsilon = 1e-9):\n",
    "    vec_squared_norm = tf.reduce_sum(tf.square(X), -2, keep_dims=True)\n",
    "    scalar_factor = vec_squared_norm / (1 + vec_squared_norm) / tf.sqrt(vec_squared_norm + epsilon)\n",
    "    return scalar_factor * X\n",
    "\n",
    "def conv_layer(X, num_output, num_vector, kernel=None, stride=None):\n",
    "    global batch_size\n",
    "    capsules = tf.layers.conv1d(X, num_output * num_vector,\n",
    "                                kernel, stride, padding=\"VALID\", activation=tf.nn.relu)\n",
    "    capsules = tf.reshape(capsules, (batch_size, -1, num_vector, 1))\n",
    "    return squash(capsules)\n",
    "\n",
    "def routing(X, b_IJ, seq_len, dimension_out, routing_times = 2):\n",
    "    global batch_size\n",
    "    shape_X = X.shape[1].value\n",
    "    w = tf.Variable(tf.truncated_normal([1, shape_X, seq_len, 8, dimension_out//2], stddev=1e-1))\n",
    "    X = tf.tile(X, [1, 1, seq_len, 1, dimension_out])\n",
    "    w = tf.tile(w, [batch_size, 1, 1, 1, routing_times])\n",
    "    print('X shape: %s, w shape: %s'%(str(X.shape), str(w.shape)))\n",
    "    u_hat = tf.matmul(w, X, transpose_a=True)\n",
    "    u_hat_stopped = tf.stop_gradient(u_hat)\n",
    "    for i in range(routing_times):\n",
    "        c_IJ = tf.nn.softmax(b_IJ, dim=2)\n",
    "        if i == routing_times - 1:\n",
    "            s_J = tf.multiply(c_IJ, u_hat)\n",
    "            s_J = tf.reduce_sum(s_J, axis=1, keep_dims=True)\n",
    "            v_J = squash(s_J)\n",
    "        else:\n",
    "            s_J = tf.multiply(c_IJ, u_hat_stopped)\n",
    "            s_J = tf.reduce_sum(s_J, axis=1, keep_dims=True)\n",
    "            v_J = squash(s_J)\n",
    "            v_J_tiled = tf.tile(v_J, [1, shape_X, 1, 1, 1])\n",
    "            u_produce_v = tf.matmul(u_hat_stopped, v_J_tiled, transpose_a=True)\n",
    "            b_IJ += u_produce_v\n",
    "    return v_J\n",
    "\n",
    "def fully_conn_layer(X, num_output, dimension_out):\n",
    "    global batch_size\n",
    "    X_ = tf.reshape(X, shape=(batch_size, -1, 1, X.shape[-2].value, 1))\n",
    "    b_IJ = tf.constant(np.zeros([batch_size, X.shape[1].value, num_output, 1, 1], dtype=np.float32))\n",
    "    capsules = routing(X_, b_IJ, num_output, dimension_out, routing_times = 2)\n",
    "    capsules = tf.squeeze(capsules, axis=1)\n",
    "    return capsules\n",
    "\n",
    "class CapsuleNetwork:\n",
    "    def __init__(self, batch_size, learning_rate, seq_len,\n",
    "                 size_layer, num_layers, maxlen,\n",
    "                 dict_size, embedded_size, dimension_output,\n",
    "                 kernels=[6,3,2],strides=[3,2,1],epsilon=1e-8,\n",
    "                skip=5):\n",
    "        \n",
    "        def cells(size,reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [batch_size, maxlen])\n",
    "        self.Y = tf.placeholder(tf.float32, [None, dimension_output])\n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        encoder_embedded = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        \n",
    "        results = []\n",
    "        for i in range(len(kernels)):\n",
    "            conv = tf.layers.conv1d(encoder_embedded, filters=32,\n",
    "                                    kernel_size=kernels[i], strides=strides[i],\n",
    "                                    padding='VALID')\n",
    "            caps1 = conv_layer(conv, 8, 8, kernels[i], strides[i])\n",
    "            caps2 = fully_conn_layer(caps1,seq_len,32)\n",
    "            v_length = tf.sqrt(tf.reduce_sum(tf.square(caps2),axis=2, keep_dims=True) + epsilon)[:,:,0,:]\n",
    "            print('output shape: %s'%(str(v_length.shape)))\n",
    "            results.append(v_length)\n",
    "        results = tf.concat(results,1)\n",
    "        decoder_embedded = results[:,skip:,:]\n",
    "        \n",
    "        for n in range(num_layers):\n",
    "            (out_fw, out_bw), (state_fw, state_bw) = tf.nn.bidirectional_dynamic_rnn(\n",
    "                cell_fw = cells(size_layer // 2),\n",
    "                cell_bw = cells(size_layer // 2),\n",
    "                inputs = results,\n",
    "                dtype = tf.float32,\n",
    "                scope = 'bidirectional_rnn_%d'%(n))\n",
    "            results = tf.concat((out_fw, out_bw), 2)\n",
    "        bi_state_c = tf.concat((state_fw.c, state_bw.c), -1)\n",
    "        bi_state_h = tf.concat((state_fw.h, state_bw.h), -1)\n",
    "        bi_lstm_state = tf.nn.rnn_cell.LSTMStateTuple(c=bi_state_c, h=bi_state_h)\n",
    "        last_state = tuple([bi_lstm_state] * num_layers)\n",
    "        \n",
    "        with tf.variable_scope(\"decoder\"):\n",
    "            rnn_cells_dec = tf.nn.rnn_cell.MultiRNNCell([cells(size_layer) for _ in range(num_layers)])\n",
    "            outputs, _ = tf.nn.dynamic_rnn(rnn_cells_dec, decoder_embedded, \n",
    "                                           initial_state = last_state,\n",
    "                                           dtype = tf.float32)\n",
    "            \n",
    "        W = tf.get_variable('w',shape=(size_layer, dimension_output),initializer=tf.orthogonal_initializer())\n",
    "        b = tf.get_variable('b',shape=(dimension_output),initializer=tf.zeros_initializer())\n",
    "        self.logits = tf.matmul(outputs[:, -1], W) + b\n",
    "        self.cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.logits, labels = self.Y))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        correct_pred = tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "maxlen = 50\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "dimension_output = len(trainset.target_names)\n",
    "learning_rate = 1e-4\n",
    "batch_size = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X shape: (64, 32, 5, 8, 32), w shape: (64, 32, 5, 8, 32)\n",
      "output shape: (64, 5, 32)\n",
      "X shape: (64, 88, 5, 8, 32), w shape: (64, 88, 5, 8, 32)\n",
      "output shape: (64, 5, 32)\n",
      "X shape: (64, 384, 5, 8, 32), w shape: (64, 384, 5, 8, 32)\n",
      "output shape: (64, 5, 32)\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = CapsuleNetwork(batch_size, learning_rate, 5, size_layer, \n",
    "                       num_layers, maxlen, vocabulary_size+4, embedded_size, dimension_output)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.500000\n",
      "time taken: 130.76251459121704\n",
      "epoch: 0, training loss: 0.693607, training acc: 0.495301, valid loss: 0.693596, valid acc: 0.500000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 5, 0, 0, 0\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n'%(EPOCH))\n",
    "        break\n",
    "        \n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    for i in range(0, (len(train_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(train_X[i:i+batch_size],dictionary,maxlen)\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : train_onehot[i:i+batch_size]})\n",
    "        train_loss += loss\n",
    "        train_acc += acc\n",
    "    \n",
    "    for i in range(0, (len(test_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(test_X[i:i+batch_size],dictionary,maxlen)\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : test_onehot[i:i+batch_size]})\n",
    "        test_loss += loss\n",
    "        test_acc += acc\n",
    "    \n",
    "    train_loss /= (len(train_X) // batch_size)\n",
    "    train_acc /= (len(train_X) // batch_size)\n",
    "    test_loss /= (len(test_X) // batch_size)\n",
    "    test_acc /= (len(test_X) // batch_size)\n",
    "    \n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print('epoch: %d, pass acc: %f, current acc: %f'%(EPOCH,CURRENT_ACC, test_acc))\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "        \n",
    "    print('time taken:', time.time()-lasttime)\n",
    "    print('epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'%(EPOCH,train_loss,\n",
    "                                                                                          train_acc,test_loss,\n",
    "                                                                                          test_acc))\n",
    "    EPOCH += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}