speech-to-text/1.tacotron.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "dimension = 400\n",
    "vocab = \"ES abcdefghijklmnopqrstuvwxyz'\"\n",
    "char2idx = {char: idx for idx, char in enumerate(vocab)}\n",
    "idx2char = {idx: char for idx, char in enumerate(vocab)}\n",
    "\n",
    "def text2idx(text):\n",
    "    text = re.sub(r'[^a-z ]', '', text.lower()).strip() + 'S'\n",
    "    converted = [char2idx[char] for char in text]\n",
    "    return text, converted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/husein/.local/lib/python3.6/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "\n",
    "train_X, train_Y = [], []\n",
    "text_files = [f for f in os.listdir('spectrogram-train') if f.endswith('.npy')]\n",
    "for fpath in text_files:\n",
    "    try:\n",
    "        splitted = fpath.split('-')\n",
    "        if len(splitted) == 2:\n",
    "            splitted[1] = splitted[1].split('.')[1]\n",
    "            fpath = splitted[0] + '.' + splitted[1]\n",
    "        with open('data/' + fpath.replace('npy', 'txt')) as fopen:\n",
    "            text, converted = text2idx(fopen.read())\n",
    "        w = np.load('spectrogram-train/' + fpath)\n",
    "        if w.shape[1] != dimension:\n",
    "            continue\n",
    "        train_X.append(w)\n",
    "        train_Y.append(converted)\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_X, test_Y = [], []\n",
    "text_files = [f for f in os.listdir('spectrogram-test') if f.endswith('.npy')]\n",
    "for fpath in text_files:\n",
    "    with open('data/' + fpath.replace('npy', 'txt')) as fopen:\n",
    "        text, converted = text2idx(fopen.read())\n",
    "    w = np.load('spectrogram-test/' + fpath)\n",
    "    if w.shape[1] != dimension:\n",
    "        continue\n",
    "    test_X.append(w)\n",
    "    test_Y.append(converted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13128, 560)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_X), len(test_X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed_size = 256\n",
    "encoder_num_banks = 16\n",
    "decoder_num_banks = 8\n",
    "num_highway_blocks = 4\n",
    "learning_rate = 1e-4\n",
    "\n",
    "def embed(inputs, vocab_size, dimension, scope = 'embedding', reuse = None):\n",
    "    with tf.variable_scope(scope, reuse = reuse):\n",
    "        lookup_table = tf.get_variable(\n",
    "            'lookup_table',\n",
    "            dtype = tf.float32,\n",
    "            shape = [vocab_size, dimension],\n",
    "            initializer = tf.truncated_normal_initializer(\n",
    "                mean = 0.0, stddev = 0.01\n",
    "            ),\n",
    "        )\n",
    "        lookup_table = tf.concat(\n",
    "            (tf.zeros(shape = [1, dimension]), lookup_table[1:, :]), 0\n",
    "        )\n",
    "    return tf.nn.embedding_lookup(lookup_table, inputs)\n",
    "\n",
    "\n",
    "def normalize_in(inputs, activation_fn = None, scope = 'normalize_in'):\n",
    "    with tf.variable_scope(scope):\n",
    "        batch, steps, channels = inputs.get_shape().as_list()\n",
    "        var_shape = [channels]\n",
    "        mu, sigma_sq = tf.nn.moments(inputs, [1], keep_dims = True)\n",
    "        shift = tf.Variable(tf.zeros(var_shape))\n",
    "        scale = tf.Variable(tf.ones(var_shape))\n",
    "        epsilon = 1e-8\n",
    "        normalized = (inputs - mu) / (sigma_sq + epsilon) ** (0.5)\n",
    "        outputs = scale * normalized + shift\n",
    "        if activation_fn:\n",
    "            outputs = activation_fn(outputs)\n",
    "    return outputs\n",
    "\n",
    "\n",
    "def conv1d(\n",
    "    inputs,\n",
    "    filters = None,\n",
    "    size = 1,\n",
    "    rate = 1,\n",
    "    padding = 'SAME',\n",
    "    use_bias = False,\n",
    "    activation_fn = None,\n",
    "    scope = 'conv1d',\n",
    "    reuse = None,\n",
    "):\n",
    "    with tf.variable_scope(scope):\n",
    "        if padding.lower() == 'causal':\n",
    "            pad_len = (size - 1) * rate\n",
    "            inputs = tf.pad(inputs, [[0, 0], [pad_len, 0], [0, 0]])\n",
    "            padding = 'valid'\n",
    "        if filters is None:\n",
    "            filters = inputs.get_shape().as_list()[-1]\n",
    "        params = {\n",
    "            'inputs': inputs,\n",
    "            'filters': filters,\n",
    "            'kernel_size': size,\n",
    "            'dilation_rate': rate,\n",
    "            'padding': padding,\n",
    "            'activation': activation_fn,\n",
    "            'use_bias': use_bias,\n",
    "            'reuse': reuse,\n",
    "        }\n",
    "        outputs = tf.layers.conv1d(**params)\n",
    "    return outputs\n",
    "\n",
    "\n",
    "def conv1d_banks(\n",
    "    inputs, K = 16, is_training = True, scope = 'conv1d_banks', reuse = None\n",
    "):\n",
    "    with tf.variable_scope(scope, reuse = reuse):\n",
    "        outputs = conv1d(inputs, embed_size // 2, 1)\n",
    "        outputs = normalize_in(outputs, tf.nn.relu)\n",
    "        for k in range(2, K + 1):\n",
    "            with tf.variable_scope('num_%d' % (k)):\n",
    "                output = conv1d(inputs, embed_size // 2, k)\n",
    "                output = normalize_in(output, tf.nn.relu)\n",
    "                outputs = tf.concat((outputs, output), -1)\n",
    "    return outputs\n",
    "\n",
    "\n",
    "def gru(inputs, units = None, bidirection = False, scope = 'gru', reuse = None):\n",
    "    with tf.variable_scope(scope, reuse = reuse):\n",
    "        if units is None:\n",
    "            units = inputs.get_shape().as_list()[-1]\n",
    "        cell = tf.contrib.rnn.GRUCell(units)\n",
    "        if bidirection:\n",
    "            cell_bw = tf.contrib.rnn.GRUCell(units)\n",
    "            outputs, _ = tf.nn.bidirectional_dynamic_rnn(\n",
    "                cell, cell_bw, inputs, dtype = tf.float32\n",
    "            )\n",
    "            return tf.concat(outputs, 2)\n",
    "        else:\n",
    "            outputs, _ = tf.nn.dynamic_rnn(cell, inputs, dtype = tf.float32)\n",
    "            return outputs\n",
    "\n",
    "\n",
    "def attention_decoder(\n",
    "    inputs, memory, units = None, scope = 'attention_decoder', reuse = None\n",
    "):\n",
    "    with tf.variable_scope(scope, reuse = reuse):\n",
    "        if units is None:\n",
    "            units = inputs.get_shape().as_list()[-1]\n",
    "        attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(\n",
    "            units, memory\n",
    "        )\n",
    "        decoder_cell = tf.contrib.rnn.GRUCell(units)\n",
    "        cell_with_attention = tf.contrib.seq2seq.AttentionWrapper(\n",
    "            decoder_cell, attention_mechanism, units\n",
    "        )\n",
    "        outputs, _ = tf.nn.dynamic_rnn(\n",
    "            cell_with_attention, inputs, dtype = tf.float32\n",
    "        )\n",
    "    return outputs\n",
    "\n",
    "\n",
    "def prenet(inputs, is_training = True, scope = 'prenet', reuse = None):\n",
    "    with tf.variable_scope(scope, reuse = reuse):\n",
    "        outputs = tf.layers.dense(\n",
    "            inputs, units = embed_size, activation = tf.nn.relu, name = 'dense1'\n",
    "        )\n",
    "        outputs = tf.nn.dropout(\n",
    "            outputs,\n",
    "            keep_prob = 0.5 if is_training == True else 1.0,\n",
    "            name = 'dropout1',\n",
    "        )\n",
    "        outputs = tf.layers.dense(\n",
    "            outputs,\n",
    "            units = embed_size // 2,\n",
    "            activation = tf.nn.relu,\n",
    "            name = 'dense2',\n",
    "        )\n",
    "        outputs = tf.nn.dropout(\n",
    "            outputs,\n",
    "            keep_prob = 0.5 if is_training == True else 1.0,\n",
    "            name = 'dropout2',\n",
    "        )\n",
    "    return outputs\n",
    "\n",
    "\n",
    "def highwaynet(inputs, units = None, scope = 'highwaynet', reuse = None):\n",
    "    with tf.variable_scope(scope, reuse = reuse):\n",
    "        if units is None:\n",
    "            units = inputs.get_shape().as_list()[-1]\n",
    "        H = tf.layers.dense(\n",
    "            inputs, units = units, activation = tf.nn.relu, name = 'dense1'\n",
    "        )\n",
    "        T = tf.layers.dense(\n",
    "            inputs, units = units, activation = tf.nn.sigmoid, name = 'dense2'\n",
    "        )\n",
    "        C = 1.0 - T\n",
    "        return H * T + inputs * C\n",
    "\n",
    "\n",
    "def shift_by_one(inputs):\n",
    "    return tf.concat((tf.zeros_like(inputs[:, :1]), inputs[:, :-1]), 1)\n",
    "\n",
    "def encode(inputs, is_training = True, scope = 'encoder', reuse = None):\n",
    "    with tf.variable_scope(scope, reuse = reuse):\n",
    "        prenet_out = prenet(inputs, scope = 'prenet', is_training = is_training)\n",
    "        enc = conv1d_banks(\n",
    "            prenet_out, K = encoder_num_banks, is_training = is_training\n",
    "        )\n",
    "        enc = tf.layers.max_pooling1d(enc, 2, 1, padding = 'same')\n",
    "        enc = conv1d(enc, embed_size // 2, 3, scope = 'conv1d_1')\n",
    "        enc = normalize_in(enc, activation_fn = tf.nn.relu)\n",
    "        enc = conv1d(enc, embed_size // 2, 3, scope = 'conv1d_2')\n",
    "        enc = normalize_in(enc, activation_fn = tf.nn.relu)\n",
    "        enc += prenet_out\n",
    "        for i in range(num_highway_blocks):\n",
    "            enc = highwaynet(\n",
    "                enc, units = embed_size // 2, scope = 'highwaynet_%d' % (i)\n",
    "            )\n",
    "        memory = gru(enc, embed_size // 2, True)\n",
    "    return memory\n",
    "\n",
    "\n",
    "def decode(\n",
    "    inputs, memory, is_training = True, scope = 'decoder_layers', reuse = None\n",
    "):\n",
    "    with tf.variable_scope(scope, reuse = reuse):\n",
    "        dec = prenet(inputs, is_training = is_training)\n",
    "        dec = attention_decoder(dec, memory, embed_size)\n",
    "        dec += gru(dec, embed_size, False, scope = 'gru1')\n",
    "        dec += gru(dec, embed_size, False, scope = 'gru2')\n",
    "        return tf.layers.dense(dec, len(char2idx))\n",
    "\n",
    "\n",
    "class Model:\n",
    "    def __init__(self, is_training = True):\n",
    "        self.X = tf.placeholder(\n",
    "            tf.float32, shape = (None, None, dimension)\n",
    "        )\n",
    "        self.Y = tf.placeholder(tf.int32, shape = (None, None))\n",
    "        self.Y_seq_len = tf.count_nonzero(self.Y, 1, dtype=tf.int32)\n",
    "        self.decoder_inputs = embed(\n",
    "            shift_by_one(self.Y), len(char2idx), embed_size\n",
    "        )\n",
    "        with tf.variable_scope('net'):\n",
    "            self.memory = encode(self.X, is_training = is_training)\n",
    "            self.outputs = decode(\n",
    "                self.decoder_inputs, self.memory, is_training = is_training\n",
    "            )\n",
    "            self.logprobs = tf.log(tf.nn.softmax(self.outputs) + 1e-10)\n",
    "            self.preds = tf.argmax(self.outputs, axis = -1)\n",
    "            correct_pred = tf.equal(tf.cast(self.preds, tf.int32), self.Y)\n",
    "            self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "            \n",
    "        masks = tf.sequence_mask(\n",
    "            self.Y_seq_len,\n",
    "            tf.reduce_max(self.Y_seq_len),\n",
    "            dtype = tf.float32,\n",
    "        )\n",
    "        self.cost = tf.contrib.seq2seq.sequence_loss(\n",
    "            logits = self.outputs, targets = self.Y, weights = masks\n",
    "        )\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate).minimize(\n",
    "            self.cost\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: Logging before flag parsing goes to stderr.\n",
      "W0830 13:12:56.280405 140415593760576 deprecation.py:506] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py:507: calling count_nonzero (from tensorflow.python.ops.math_ops) with axis is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "reduction_indices is deprecated, use axis instead\n",
      "W0830 13:12:56.310420 140415593760576 deprecation.py:323] From <ipython-input-6-2afb84048410>:122: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dense instead.\n",
      "W0830 13:12:56.314230 140415593760576 deprecation.py:506] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0830 13:12:56.590281 140415593760576 deprecation.py:506] From <ipython-input-6-2afb84048410>:127: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "W0830 13:12:56.644309 140415593760576 deprecation.py:323] From <ipython-input-6-2afb84048410>:66: conv1d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.keras.layers.Conv1D` instead.\n",
      "W0830 13:12:57.293771 140415593760576 deprecation.py:323] From <ipython-input-6-2afb84048410>:166: max_pooling1d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.MaxPooling1D instead.\n",
      "W0830 13:12:58.725348 140415593760576 lazy_loader.py:50] \n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "W0830 13:12:58.727220 140415593760576 deprecation.py:323] From <ipython-input-6-2afb84048410>:88: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n",
      "W0830 13:12:58.728699 140415593760576 deprecation.py:323] From <ipython-input-6-2afb84048410>:92: bidirectional_dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.Bidirectional(keras.layers.RNN(cell))`, which is equivalent to this API\n",
      "W0830 13:12:58.729964 140415593760576 deprecation.py:323] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/rnn.py:464: dynamic_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell)`, which is equivalent to this API\n",
      "W0830 13:12:58.806982 140415593760576 deprecation.py:506] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py:564: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0830 13:12:58.821044 140415593760576 deprecation.py:506] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py:574: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0830 13:13:02.284158 140415593760576 deprecation.py:323] From /home/husein/.local/lib/python3.6/site-packages/tensorflow/python/ops/array_grad.py:199: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model()\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13128, 48, 400)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_X = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "    train_X, dtype = 'float32', padding = 'post'\n",
    ")\n",
    "train_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(560, 48, 400)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_X = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "    test_X, dtype = 'float32', padding = 'post'\n",
    ")\n",
    "test_X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pad_sentence_batch(sentence_batch, pad_int):\n",
    "    padded_seqs = []\n",
    "    seq_lens = []\n",
    "    max_sentence_len = max([len(sentence) for sentence in sentence_batch])\n",
    "    for sentence in sentence_batch:\n",
    "        padded_seqs.append(sentence + [pad_int] * (max_sentence_len - len(sentence)))\n",
    "        seq_lens.append(len(sentence))\n",
    "    return padded_seqs, seq_lens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch = 20\n",
    "batch_size = 64"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:41<00:00,  4.25it/s, accuracy=0.783, cost=0.659]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:01<00:00,  6.04it/s, accuracy=0.7, cost=0.684]  \n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:36,  5.64it/s, accuracy=0.73, cost=0.683]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1, training avg loss 1.272585, training avg acc 0.584240\n",
      "epoch 1, testing avg loss 0.683997, testing avg acc 0.721852\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.56it/s, accuracy=0.803, cost=0.525]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 15.72it/s, accuracy=0.706, cost=0.553]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:36,  5.62it/s, accuracy=0.741, cost=0.552]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 2, training avg loss 0.612984, training avg acc 0.723206\n",
      "epoch 2, testing avg loss 0.565084, testing avg acc 0.731944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.40it/s, accuracy=0.803, cost=0.515]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 14.57it/s, accuracy=0.716, cost=0.517]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:38,  5.30it/s, accuracy=0.753, cost=0.524]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 3, training avg loss 0.542716, training avg acc 0.735273\n",
      "epoch 3, testing avg loss 0.527480, testing avg acc 0.740622\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:39<00:00,  5.35it/s, accuracy=0.829, cost=0.478]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 15.71it/s, accuracy=0.723, cost=0.497]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:38,  5.36it/s, accuracy=0.761, cost=0.494]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 4, training avg loss 0.510864, training avg acc 0.743509\n",
      "epoch 4, testing avg loss 0.505699, testing avg acc 0.745512\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.43it/s, accuracy=0.803, cost=0.478]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 14.91it/s, accuracy=0.729, cost=0.47] \n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:37,  5.53it/s, accuracy=0.767, cost=0.467]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 5, training avg loss 0.485896, training avg acc 0.750599\n",
      "epoch 5, testing avg loss 0.482513, testing avg acc 0.752478\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.35it/s, accuracy=0.829, cost=0.447]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 14.37it/s, accuracy=0.742, cost=0.449]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:36,  5.54it/s, accuracy=0.768, cost=0.447]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 6, training avg loss 0.462700, training avg acc 0.755660\n",
      "epoch 6, testing avg loss 0.461965, testing avg acc 0.756211\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.29it/s, accuracy=0.822, cost=0.408]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 14.07it/s, accuracy=0.737, cost=0.44] \n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:39,  5.18it/s, accuracy=0.776, cost=0.422]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 7, training avg loss 0.441943, training avg acc 0.761259\n",
      "epoch 7, testing avg loss 0.445603, testing avg acc 0.761199\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.43it/s, accuracy=0.849, cost=0.379]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 14.29it/s, accuracy=0.748, cost=0.41] \n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:38,  5.28it/s, accuracy=0.787, cost=0.4]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 8, training avg loss 0.419698, training avg acc 0.767303\n",
      "epoch 8, testing avg loss 0.426653, testing avg acc 0.765520\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.64it/s, accuracy=0.842, cost=0.365]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 14.72it/s, accuracy=0.75, cost=0.394] \n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:40,  5.07it/s, accuracy=0.796, cost=0.368]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 9, training avg loss 0.398379, training avg acc 0.773561\n",
      "epoch 9, testing avg loss 0.413055, testing avg acc 0.768295\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.42it/s, accuracy=0.862, cost=0.336]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 14.25it/s, accuracy=0.746, cost=0.408]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:39,  5.19it/s, accuracy=0.807, cost=0.354]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10, training avg loss 0.378367, training avg acc 0.779997\n",
      "epoch 10, testing avg loss 0.411091, testing avg acc 0.771466\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.30it/s, accuracy=0.849, cost=0.336]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 14.29it/s, accuracy=0.745, cost=0.402]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:38,  5.32it/s, accuracy=0.801, cost=0.337]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 11, training avg loss 0.358436, training avg acc 0.785431\n",
      "epoch 11, testing avg loss 0.402924, testing avg acc 0.772684\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.22it/s, accuracy=0.849, cost=0.308]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 15.25it/s, accuracy=0.752, cost=0.395]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:37,  5.45it/s, accuracy=0.811, cost=0.32]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 12, training avg loss 0.338979, training avg acc 0.791071\n",
      "epoch 12, testing avg loss 0.415512, testing avg acc 0.772348\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.45it/s, accuracy=0.855, cost=0.281]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 14.20it/s, accuracy=0.74, cost=0.42]  \n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:38,  5.30it/s, accuracy=0.818, cost=0.29]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 13, training avg loss 0.319973, training avg acc 0.796918\n",
      "epoch 13, testing avg loss 0.421690, testing avg acc 0.771532\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.32it/s, accuracy=0.875, cost=0.254]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 14.27it/s, accuracy=0.744, cost=0.432]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:38,  5.35it/s, accuracy=0.818, cost=0.28]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 14, training avg loss 0.299559, training avg acc 0.802903\n",
      "epoch 14, testing avg loss 0.439885, testing avg acc 0.772849\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.77it/s, accuracy=0.882, cost=0.257]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 14.40it/s, accuracy=0.743, cost=0.453]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:38,  5.32it/s, accuracy=0.823, cost=0.265]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 15, training avg loss 0.279262, training avg acc 0.807981\n",
      "epoch 15, testing avg loss 0.464145, testing avg acc 0.772904\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.52it/s, accuracy=0.914, cost=0.206]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 14.88it/s, accuracy=0.743, cost=0.476]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:38,  5.27it/s, accuracy=0.831, cost=0.241]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 16, training avg loss 0.262802, training avg acc 0.813376\n",
      "epoch 16, testing avg loss 0.478891, testing avg acc 0.773563\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.38it/s, accuracy=0.895, cost=0.209]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 15.17it/s, accuracy=0.744, cost=0.499]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:38,  5.36it/s, accuracy=0.837, cost=0.233]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 17, training avg loss 0.244635, training avg acc 0.818782\n",
      "epoch 17, testing avg loss 0.519443, testing avg acc 0.772614\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:39<00:00,  5.48it/s, accuracy=0.882, cost=0.22] \n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 15.45it/s, accuracy=0.744, cost=0.538]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:36,  5.56it/s, accuracy=0.83, cost=0.217]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 18, training avg loss 0.232706, training avg acc 0.822359\n",
      "epoch 18, testing avg loss 0.540628, testing avg acc 0.771803\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.41it/s, accuracy=0.914, cost=0.169]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 15.63it/s, accuracy=0.743, cost=0.552]\n",
      "minibatch loop:   0%|          | 1/206 [00:00<00:37,  5.46it/s, accuracy=0.833, cost=0.219]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 19, training avg loss 0.219639, training avg acc 0.827012\n",
      "epoch 19, testing avg loss 0.558029, testing avg acc 0.771475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "minibatch loop: 100%|██████████| 206/206 [00:38<00:00,  5.42it/s, accuracy=0.901, cost=0.177]\n",
      "minibatch loop: 100%|██████████| 9/9 [00:00<00:00, 15.38it/s, accuracy=0.74, cost=0.579] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 20, training avg loss 0.209063, training avg acc 0.829762\n",
      "epoch 20, testing avg loss 0.570628, testing avg acc 0.770907\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "for e in range(epoch):\n",
    "    pbar = tqdm(\n",
    "        range(0, len(train_X), batch_size), desc = 'minibatch loop')\n",
    "    train_loss, train_acc, test_loss, test_acc = [], [], [], []\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(train_X))\n",
    "        batch_x = train_X[i : index]\n",
    "        y = train_Y[i : index]\n",
    "        batch_y, _ = pad_sentence_batch(y, 0)\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y}\n",
    "        accuracy, loss, _ = sess.run([model.accuracy,model.cost,model.optimizer],\n",
    "                                    feed_dict = feed)\n",
    "        train_loss.append(loss)\n",
    "        train_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    \n",
    "    pbar = tqdm(\n",
    "        range(0, len(test_X), batch_size), desc = 'minibatch loop')\n",
    "    for i in pbar:\n",
    "        index = min(i + batch_size, len(test_X))\n",
    "        batch_x = train_X[i : index]\n",
    "        y = test_Y[i : index]\n",
    "        batch_y, _ = pad_sentence_batch(y, 0)\n",
    "        feed = {model.X: batch_x,\n",
    "                model.Y: batch_y,}\n",
    "        accuracy, loss = sess.run([model.accuracy,model.cost],\n",
    "                                    feed_dict = feed)\n",
    "\n",
    "        test_loss.append(loss)\n",
    "        test_acc.append(accuracy)\n",
    "        pbar.set_postfix(cost = loss, accuracy = accuracy)\n",
    "    \n",
    "    print('epoch %d, training avg loss %f, training avg acc %f'%(e+1,\n",
    "                                                                 np.mean(train_loss),np.mean(train_acc)))\n",
    "    print('epoch %d, testing avg loss %f, testing avg acc %f'%(e+1,\n",
    "                                                              np.mean(test_loss),np.mean(test_acc)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "empty_y = np.zeros((1, len(batch_y[0])))\n",
    "predicted = ''.join(\n",
    "    [\n",
    "        idx2char[c]\n",
    "        for c in sess.run(\n",
    "            model.preds, feed_dict = {model.X: batch_x[:1], model.Y: empty_y}\n",
    "        )[0]\n",
    "        if idx2char[c] not in ['S', 'E']\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "predicted: syytteeword jooe, ground truth: say the word tool\n"
     ]
    }
   ],
   "source": [
    "ground_truth = ''.join(\n",
    "    [idx2char[c] for c in batch_y[0] if idx2char[c] not in ['S', 'E']]\n",
    ")\n",
    "print('predicted: %s, ground truth: %s' % (predicted, ground_truth))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}