text-similarity/5.transformer-crossentropy.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget http://qim.fs.quoracdn.net/quora_duplicate_questions.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jupyter/.local/lib/python3.6/site-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import collections\n",
    "from unidecode import unidecode\n",
    "from sklearn.cross_validation import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_dataset(words, n_words):\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3], ['SEPARATOR', 4]]\n",
    "    count.extend(collections.Counter(words).most_common(n_words - 1))\n",
    "    dictionary = dict()\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    unk_count = 0\n",
    "    for word in words:\n",
    "        index = dictionary.get(word, 0)\n",
    "        if index == 0:\n",
    "            unk_count += 1\n",
    "        data.append(index)\n",
    "    count[0][1] = unk_count\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, count, dictionary, reversed_dictionary\n",
    "\n",
    "def str_idx(corpus, dic, maxlen, UNK=3):\n",
    "    X = np.zeros((len(corpus),maxlen))\n",
    "    for i in range(len(corpus)):\n",
    "        for no, k in enumerate(corpus[i][:maxlen][::-1]):\n",
    "            val = dic[k] if k in dic else UNK\n",
    "            X[i,-1 - no]= val\n",
    "    return X\n",
    "\n",
    "def cleaning(string):\n",
    "    string = unidecode(string).replace('.', ' . ').replace(',', ' , ')\n",
    "    string = re.sub('[^A-Za-z\\- ]+', ' ', string)\n",
    "    string = re.sub(r'[ ]+', ' ', string).strip()\n",
    "    return string.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>qid1</th>\n",
       "      <th>qid2</th>\n",
       "      <th>question1</th>\n",
       "      <th>question2</th>\n",
       "      <th>is_duplicate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>What is the step by step guide to invest in sh...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>What is the story of Kohinoor (Koh-i-Noor) Dia...</td>\n",
       "      <td>What would happen if the Indian government sto...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>6</td>\n",
       "      <td>How can I increase the speed of my internet co...</td>\n",
       "      <td>How can Internet speed be increased by hacking...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>Why am I mentally very lonely? How can I solve...</td>\n",
       "      <td>Find the remainder when [math]23^{24}[/math] i...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>10</td>\n",
       "      <td>Which one dissolve in water quikly sugar, salt...</td>\n",
       "      <td>Which fish would survive in salt water?</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  qid1  qid2                                          question1  \\\n",
       "0   0     1     2  What is the step by step guide to invest in sh...   \n",
       "1   1     3     4  What is the story of Kohinoor (Koh-i-Noor) Dia...   \n",
       "2   2     5     6  How can I increase the speed of my internet co...   \n",
       "3   3     7     8  Why am I mentally very lonely? How can I solve...   \n",
       "4   4     9    10  Which one dissolve in water quikly sugar, salt...   \n",
       "\n",
       "                                           question2  is_duplicate  \n",
       "0  What is the step by step guide to invest in sh...             0  \n",
       "1  What would happen if the Indian government sto...             0  \n",
       "2  How can Internet speed be increased by hacking...             0  \n",
       "3  Find the remainder when [math]23^{24}[/math] i...             0  \n",
       "4            Which fish would survive in salt water?             0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('quora_duplicate_questions.tsv', delimiter='\\t').dropna()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "left, right, label = df['question1'].tolist(), df['question2'].tolist(), df['is_duplicate'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 1]), array([255024, 149263]))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(label, return_counts = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 404287/404287 [00:07<00:00, 52786.23it/s]\n"
     ]
    }
   ],
   "source": [
    "for i in tqdm(range(len(left))):\n",
    "    left[i] = cleaning(left[i])\n",
    "    right[i] = cleaning(right[i])\n",
    "    left[i] = left[i] + ' SEPARATOR ' + right[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 87662\n",
      "Most common words [['SEPARATOR', 4], ('SEPARATOR', 404287), ('the', 377593), ('what', 324635), ('is', 269934), ('i', 223893)]\n",
      "Sample data [6, 7, 5, 1286, 63, 1286, 2502, 11, 565, 12] ['what', 'is', 'the', 'step', 'by', 'step', 'guide', 'to', 'invest', 'in']\n"
     ]
    }
   ],
   "source": [
    "concat = ' '.join(left).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def position_encoding(inputs):\n",
    "    T = tf.shape(inputs)[1]\n",
    "    repr_dim = inputs.get_shape()[-1].value\n",
    "    pos = tf.reshape(tf.range(0.0, tf.to_float(T), dtype=tf.float32), [-1, 1])\n",
    "    i = np.arange(0, repr_dim, 2, np.float32)\n",
    "    denom = np.reshape(np.power(10000.0, i / repr_dim), [1, -1])\n",
    "    enc = tf.expand_dims(tf.concat([tf.sin(pos / denom), tf.cos(pos / denom)], 1), 0)\n",
    "    return tf.tile(enc, [tf.shape(inputs)[0], 1, 1])\n",
    "\n",
    "def layer_norm(inputs, epsilon=1e-8):\n",
    "    mean, variance = tf.nn.moments(inputs, [-1], keep_dims=True)\n",
    "    normalized = (inputs - mean) / (tf.sqrt(variance + epsilon))\n",
    "    params_shape = inputs.get_shape()[-1:]\n",
    "    gamma = tf.get_variable('gamma', params_shape, tf.float32, tf.ones_initializer())\n",
    "    beta = tf.get_variable('beta', params_shape, tf.float32, tf.zeros_initializer())\n",
    "    return gamma * normalized + beta\n",
    "\n",
    "def self_attention(inputs, is_training, num_units, num_heads = 8, activation=None):\n",
    "    T_q = T_k = tf.shape(inputs)[1]\n",
    "    Q_K_V = tf.layers.dense(inputs, 3*num_units, activation)\n",
    "    Q, K, V = tf.split(Q_K_V, 3, -1)\n",
    "    Q_ = tf.concat(tf.split(Q, num_heads, axis=2), 0)\n",
    "    K_ = tf.concat(tf.split(K, num_heads, axis=2), 0)\n",
    "    V_ = tf.concat(tf.split(V, num_heads, axis=2), 0)\n",
    "    align = tf.matmul(Q_, K_, transpose_b=True)\n",
    "    align *= tf.rsqrt(tf.to_float(K_.get_shape()[-1].value))\n",
    "    paddings = tf.fill(tf.shape(align), float('-inf'))\n",
    "    lower_tri = tf.ones([T_q, T_k])\n",
    "    lower_tri = tf.linalg.LinearOperatorLowerTriangular(lower_tri).to_dense()\n",
    "    masks = tf.tile(tf.expand_dims(lower_tri,0), [tf.shape(align)[0],1,1])\n",
    "    align = tf.where(tf.equal(masks, 0), paddings, align)\n",
    "    align = tf.nn.softmax(align)\n",
    "    align = tf.layers.dropout(align, 0.1, training=is_training) \n",
    "    x = tf.matmul(align, V_)\n",
    "    x = tf.concat(tf.split(x, num_heads, axis=0), 2)\n",
    "    x += inputs\n",
    "    x = layer_norm(x)\n",
    "    return x\n",
    "\n",
    "def ffn(inputs, hidden_dim, activation=tf.nn.relu):\n",
    "    x = tf.layers.conv1d(inputs, 4* hidden_dim, 1, activation=activation) \n",
    "    x = tf.layers.conv1d(x, hidden_dim, 1, activation=None)\n",
    "    x += inputs\n",
    "    x = layer_norm(x)\n",
    "    return x\n",
    "\n",
    "class Model:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size,\n",
    "                 dict_size, learning_rate, dropout, kernel_size = 5):\n",
    "        \n",
    "        def cnn(x, scope):\n",
    "            x += position_encoding(x)\n",
    "            with tf.variable_scope(scope, reuse = tf.AUTO_REUSE):\n",
    "                for n in range(num_layers):\n",
    "                    with tf.variable_scope('attn_%d'%i,reuse=tf.AUTO_REUSE):\n",
    "                        x = self_attention(x, True, size_layer)\n",
    "                    with tf.variable_scope('ffn_%d'%i, reuse=tf.AUTO_REUSE):\n",
    "                        x = ffn(x, size_layer)\n",
    "                \n",
    "                with tf.variable_scope('logits', reuse=tf.AUTO_REUSE):\n",
    "                    return tf.layers.dense(x, 2)[:, -1]\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None])\n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        embedded_left = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        \n",
    "        self.logits = cnn(embedded_left, 'left')\n",
    "        self.cost = tf.reduce_mean(\n",
    "            tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                logits = self.logits, labels = self.Y\n",
    "            )\n",
    "        )\n",
    "        \n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        correct_pred = tf.equal(\n",
    "            tf.argmax(self.logits, 1, output_type = tf.int32), self.Y\n",
    "        )\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "num_layers = 4\n",
    "embedded_size = 128\n",
    "learning_rate = 1e-4\n",
    "maxlen = 50\n",
    "batch_size = 128\n",
    "dropout = 0.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cross_validation import train_test_split\n",
    "\n",
    "vectors = str_idx(left, dictionary, maxlen)\n",
    "train_X, test_X, train_Y, test_Y = train_test_split(vectors, label, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From <ipython-input-9-3c97a48061fd>:4: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "WARNING:tensorflow:From <ipython-input-9-3c97a48061fd>:20: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "WARNING:tensorflow:From <ipython-input-9-3c97a48061fd>:33: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dropout instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From <ipython-input-9-3c97a48061fd>:41: conv1d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.conv1d instead.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(size_layer,num_layers,embedded_size,len(dictionary),learning_rate,dropout)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:54<00:00, 46.20it/s, accuracy=0.663, cost=0.652]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 110.07it/s, accuracy=0.644, cost=0.674]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:54, 46.61it/s, accuracy=0.648, cost=0.617]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.654326\n",
      "time taken: 60.44020199775696\n",
      "epoch: 0, training loss: 0.639404, training acc: 0.640978, valid loss: 0.628099, valid acc: 0.654326\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:54<00:00, 46.62it/s, accuracy=0.663, cost=0.619]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 112.44it/s, accuracy=0.622, cost=0.669]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:53, 47.00it/s, accuracy=0.68, cost=0.62]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.654326, current acc: 0.667128\n",
      "time taken: 59.827545404434204\n",
      "epoch: 0, training loss: 0.621935, training acc: 0.659585, valid loss: 0.614735, valid acc: 0.667128\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:54<00:00, 46.69it/s, accuracy=0.683, cost=0.577]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 112.01it/s, accuracy=0.6, cost=0.683]  \n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:54, 46.61it/s, accuracy=0.68, cost=0.621] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.667128, current acc: 0.672164\n",
      "time taken: 59.77066659927368\n",
      "epoch: 0, training loss: 0.610259, training acc: 0.670584, valid loss: 0.608394, valid acc: 0.672164\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:54<00:00, 46.65it/s, accuracy=0.713, cost=0.564]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 111.70it/s, accuracy=0.656, cost=0.666]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:53, 46.84it/s, accuracy=0.711, cost=0.604]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.672164, current acc: 0.679227\n",
      "time taken: 59.83059549331665\n",
      "epoch: 0, training loss: 0.601291, training acc: 0.679090, valid loss: 0.602495, valid acc: 0.679227\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:54<00:00, 46.56it/s, accuracy=0.703, cost=0.556]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 112.42it/s, accuracy=0.6, cost=0.659]  \n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:53, 46.75it/s, accuracy=0.695, cost=0.601]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.679227, current acc: 0.685867\n",
      "time taken: 59.903602838516235\n",
      "epoch: 0, training loss: 0.592938, training acc: 0.687245, valid loss: 0.597082, valid acc: 0.685867\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 46.87it/s, accuracy=0.743, cost=0.548]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 111.96it/s, accuracy=0.633, cost=0.672]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:53, 46.98it/s, accuracy=0.695, cost=0.585]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.685867, current acc: 0.688751\n",
      "time taken: 59.562599897384644\n",
      "epoch: 0, training loss: 0.585165, training acc: 0.693349, valid loss: 0.592944, valid acc: 0.688751\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 46.82it/s, accuracy=0.752, cost=0.529]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 112.44it/s, accuracy=0.622, cost=0.704]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:53, 46.92it/s, accuracy=0.719, cost=0.585]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.688751, current acc: 0.692926\n",
      "time taken: 59.60137748718262\n",
      "epoch: 0, training loss: 0.577756, training acc: 0.700359, valid loss: 0.590633, valid acc: 0.692926\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:54<00:00, 46.72it/s, accuracy=0.733, cost=0.524]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 112.66it/s, accuracy=0.622, cost=0.695]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:53, 46.71it/s, accuracy=0.719, cost=0.597]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.692926, current acc: 0.694126\n",
      "time taken: 59.701225996017456\n",
      "epoch: 0, training loss: 0.570621, training acc: 0.705953, valid loss: 0.587987, valid acc: 0.694126\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 47.07it/s, accuracy=0.743, cost=0.517]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 112.60it/s, accuracy=0.667, cost=0.664]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:53, 47.16it/s, accuracy=0.75, cost=0.59]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.694126, current acc: 0.697845\n",
      "time taken: 59.29985284805298\n",
      "epoch: 0, training loss: 0.563849, training acc: 0.711581, valid loss: 0.585073, valid acc: 0.697845\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 46.92it/s, accuracy=0.752, cost=0.49] \n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 112.64it/s, accuracy=0.689, cost=0.684]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:53, 47.25it/s, accuracy=0.734, cost=0.591]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.697845, current acc: 0.699698\n",
      "time taken: 59.466017723083496\n",
      "epoch: 0, training loss: 0.557104, training acc: 0.716393, valid loss: 0.583814, valid acc: 0.699698\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 46.91it/s, accuracy=0.733, cost=0.527]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 113.03it/s, accuracy=0.644, cost=0.68] \n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:54, 46.28it/s, accuracy=0.75, cost=0.56]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.699698, current acc: 0.700679\n",
      "time taken: 59.46453809738159\n",
      "epoch: 0, training loss: 0.551015, training acc: 0.721082, valid loss: 0.580544, valid acc: 0.700679\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 47.04it/s, accuracy=0.762, cost=0.522]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 113.48it/s, accuracy=0.678, cost=0.651]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:53, 47.44it/s, accuracy=0.758, cost=0.556]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.700679, current acc: 0.702092\n",
      "time taken: 59.29327607154846\n",
      "epoch: 0, training loss: 0.545043, training acc: 0.725462, valid loss: 0.581033, valid acc: 0.702092\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 47.21it/s, accuracy=0.762, cost=0.516]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 113.11it/s, accuracy=0.7, cost=0.654]  \n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:54, 46.67it/s, accuracy=0.727, cost=0.55] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.702092, current acc: 0.702943\n",
      "time taken: 59.11387062072754\n",
      "epoch: 0, training loss: 0.539628, training acc: 0.729723, valid loss: 0.581183, valid acc: 0.702943\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 47.15it/s, accuracy=0.762, cost=0.502]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 112.97it/s, accuracy=0.633, cost=0.693]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:52, 47.68it/s, accuracy=0.758, cost=0.545]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.702943, current acc: 0.705497\n",
      "time taken: 59.19653916358948\n",
      "epoch: 0, training loss: 0.533567, training acc: 0.734188, valid loss: 0.578577, valid acc: 0.705497\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 47.06it/s, accuracy=0.743, cost=0.483]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 112.83it/s, accuracy=0.644, cost=0.721]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:53, 47.13it/s, accuracy=0.727, cost=0.544]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.705497, current acc: 0.709658\n",
      "time taken: 59.30323553085327\n",
      "epoch: 0, training loss: 0.528961, training acc: 0.737278, valid loss: 0.575870, valid acc: 0.709658\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 47.01it/s, accuracy=0.782, cost=0.481]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 113.54it/s, accuracy=0.7, cost=0.699]  \n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:52, 47.92it/s, accuracy=0.805, cost=0.487]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 59.32865643501282\n",
      "epoch: 0, training loss: 0.522808, training acc: 0.741622, valid loss: 0.579368, valid acc: 0.706827\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 47.29it/s, accuracy=0.733, cost=0.481]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 113.10it/s, accuracy=0.622, cost=0.675]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:53, 47.33it/s, accuracy=0.789, cost=0.505]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 59.023605823516846\n",
      "epoch: 0, training loss: 0.517364, training acc: 0.744728, valid loss: 0.578737, valid acc: 0.709103\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 47.16it/s, accuracy=0.792, cost=0.454]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 113.06it/s, accuracy=0.567, cost=0.64] \n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:52, 47.79it/s, accuracy=0.789, cost=0.486]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.709658, current acc: 0.711080\n",
      "time taken: 59.17823839187622\n",
      "epoch: 0, training loss: 0.512706, training acc: 0.748938, valid loss: 0.575415, valid acc: 0.711080\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 47.10it/s, accuracy=0.782, cost=0.43] \n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 112.75it/s, accuracy=0.656, cost=0.655]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:54, 46.70it/s, accuracy=0.766, cost=0.531]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 59.26551961898804\n",
      "epoch: 0, training loss: 0.507218, training acc: 0.751649, valid loss: 0.579230, valid acc: 0.709997\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 47.01it/s, accuracy=0.832, cost=0.41] \n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 113.18it/s, accuracy=0.622, cost=0.669]\n",
      "train minibatch loop:   0%|          | 5/2527 [00:00<00:53, 47.25it/s, accuracy=0.734, cost=0.526]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 59.346855878829956\n",
      "epoch: 0, training loss: 0.502882, training acc: 0.755138, valid loss: 0.583503, valid acc: 0.707928\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|██████████| 2527/2527 [00:53<00:00, 47.27it/s, accuracy=0.802, cost=0.441]\n",
      "test minibatch loop: 100%|██████████| 632/632 [00:05<00:00, 113.33it/s, accuracy=0.622, cost=0.659]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time taken: 59.0352988243103\n",
      "epoch: 0, training loss: 0.498010, training acc: 0.757788, valid loss: 0.579649, valid acc: 0.709758\n",
      "\n",
      "break epoch:0\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 3, 0, 0, 0\n",
    "\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n' % (EPOCH))\n",
    "        break\n",
    "\n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    pbar = tqdm(range(0, len(train_X), batch_size), desc='train minibatch loop')\n",
    "    for i in pbar:\n",
    "        batch_x = train_X[i:min(i+batch_size,train_X.shape[0])]\n",
    "        batch_y = train_Y[i:min(i+batch_size,train_X.shape[0])]\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                           feed_dict = {model.X : batch_x,\n",
    "                                        model.Y : batch_y})\n",
    "        assert not np.isnan(loss)\n",
    "        train_loss += loss\n",
    "        train_acc += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "    \n",
    "    pbar = tqdm(range(0, len(test_X), batch_size), desc='test minibatch loop')\n",
    "    for i in pbar:\n",
    "        batch_x = test_X[i:min(i+batch_size,test_X.shape[0])]\n",
    "        batch_y = test_Y[i:min(i+batch_size,test_X.shape[0])]\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                           feed_dict = {model.X : batch_x,\n",
    "                                        model.Y : batch_y})\n",
    "        test_loss += loss\n",
    "        test_acc += acc\n",
    "        pbar.set_postfix(cost=loss, accuracy = acc)\n",
    "    \n",
    "    train_loss /= (len(train_X) / batch_size)\n",
    "    train_acc /= (len(train_X) / batch_size)\n",
    "    test_loss /= (len(test_X) / batch_size)\n",
    "    test_acc /= (len(test_X) / batch_size)\n",
    "    \n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print(\n",
    "            'epoch: %d, pass acc: %f, current acc: %f'\n",
    "            % (EPOCH, CURRENT_ACC, test_acc)\n",
    "        )\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "    \n",
    "    print('time taken:', time.time()-lasttime)\n",
    "    print('epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'%(EPOCH,train_loss,\n",
    "                                                                                          train_acc,test_loss,\n",
    "                                                                                          test_acc))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}