visualization/1.attention-visualization-bahdanau.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from utils import *\n",
    "import tensorflow as tf\n",
    "from sklearn.cross_validation import train_test_split\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['negative', 'positive']\n",
      "10662\n",
      "10662\n"
     ]
    }
   ],
   "source": [
    "trainset = sklearn.datasets.load_files(container_path = 'data', encoding = 'UTF-8')\n",
    "trainset.data, trainset.target = separate_dataset(trainset,1.0)\n",
    "print (trainset.target_names)\n",
    "print (len(trainset.data))\n",
    "print (len(trainset.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_X, test_X, train_Y, test_Y = train_test_split(trainset.data, trainset.target, test_size = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab from size: 20332\n",
      "Most common words [('film', 1453), ('movie', 1270), ('one', 727), ('like', 721), ('story', 477), ('much', 386)]\n",
      "Sample data [562, 2576, 3242, 16592, 36, 8849, 218, 150, 19, 4349] ['rock', 'destined', '21st', 'centurys', 'new', 'conan', 'hes', 'going', 'make', 'splash']\n"
     ]
    }
   ],
   "source": [
    "concat = ' '.join(trainset.data).split()\n",
    "vocabulary_size = len(list(set(concat)))\n",
    "data, count, dictionary, rev_dictionary = build_dataset(concat, vocabulary_size)\n",
    "print('vocab from size: %d'%(vocabulary_size))\n",
    "print('Most common words', count[4:10])\n",
    "print('Sample data', data[:10], [rev_dictionary[i] for i in data[:10]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "GO = dictionary['GO']\n",
    "PAD = dictionary['PAD']\n",
    "EOS = dictionary['EOS']\n",
    "UNK = dictionary['UNK']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_layer = 128\n",
    "num_layers = 2\n",
    "embedded_size = 128\n",
    "dimension_output = len(trainset.target_names)\n",
    "learning_rate = 1e-3\n",
    "maxlen = 50\n",
    "batch_size = 128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, size_layer, num_layers, embedded_size,\n",
    "                 dict_size, dimension_output, learning_rate):\n",
    "        \n",
    "        def cells(reuse=False):\n",
    "            return tf.nn.rnn_cell.LSTMCell(size_layer,initializer=tf.orthogonal_initializer(),reuse=reuse)\n",
    "        \n",
    "        self.X = tf.placeholder(tf.int32, [None, None])\n",
    "        self.Y = tf.placeholder(tf.int32, [None])\n",
    "        batch_size = tf.shape(self.X)[0]\n",
    "        encoder_embeddings = tf.Variable(tf.random_uniform([dict_size, embedded_size], -1, 1))\n",
    "        encoder_embedded = tf.nn.embedding_lookup(encoder_embeddings, self.X)\n",
    "        outputs, last_state = tf.nn.dynamic_rnn(tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(num_layers)]), \n",
    "                                                encoder_embedded, dtype = tf.float32)\n",
    "        attention_mechanism = tf.contrib.seq2seq.BahdanauAttention(num_units = size_layer, \n",
    "                                                                    memory = outputs)\n",
    "        rnn_cells = tf.contrib.seq2seq.AttentionWrapper(cell = tf.nn.rnn_cell.MultiRNNCell([cells() for _ in range(num_layers)]), \n",
    "                                                        attention_mechanism = attention_mechanism,\n",
    "                                                        attention_layer_size = size_layer,\n",
    "                                                       alignment_history=True)\n",
    "        decoder_outputs, decoder_last_state = tf.nn.dynamic_rnn(rnn_cells, encoder_embedded, \n",
    "                                                                initial_state = rnn_cells.zero_state(batch_size, tf.float32).clone(cell_state=last_state),\n",
    "                                                                dtype = tf.float32)\n",
    "        self.alignments = tf.transpose(decoder_last_state.alignment_history.stack(),[1,2,0])\n",
    "        W = tf.get_variable('w',shape=(size_layer, dimension_output),initializer=tf.orthogonal_initializer())\n",
    "        b = tf.get_variable('b',shape=(dimension_output),initializer=tf.zeros_initializer())\n",
    "        self.logits = tf.matmul(outputs[:, -1], W) + b\n",
    "        self.cost = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits = self.logits, labels = self.Y))\n",
    "        self.optimizer = tf.train.AdamOptimizer(learning_rate = learning_rate).minimize(self.cost)\n",
    "        correct_pred = tf.equal(tf.argmax(self.logits, 1,output_type=tf.int32), self.Y)\n",
    "        self.accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = Model(size_layer,num_layers,embedded_size,len(dictionary),dimension_output,learning_rate)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0, pass acc: 0.000000, current acc: 0.621582\n",
      "time taken: 2.827195882797241\n",
      "epoch: 0, training loss: 0.677005, training acc: 0.564157, valid loss: 0.647610, valid acc: 0.621582\n",
      "\n",
      "epoch: 1, pass acc: 0.621582, current acc: 0.677246\n",
      "time taken: 2.7505533695220947\n",
      "epoch: 1, training loss: 0.562928, training acc: 0.711529, valid loss: 0.616340, valid acc: 0.677246\n",
      "\n",
      "epoch: 2, pass acc: 0.677246, current acc: 0.702637\n",
      "time taken: 2.758648633956909\n",
      "epoch: 2, training loss: 0.416040, training acc: 0.807292, valid loss: 0.648595, valid acc: 0.702637\n",
      "\n",
      "epoch: 3, pass acc: 0.702637, current acc: 0.709961\n",
      "time taken: 2.7736120223999023\n",
      "epoch: 3, training loss: 0.282594, training acc: 0.877131, valid loss: 0.739679, valid acc: 0.709961\n",
      "\n",
      "epoch: 4, pass acc: 0.709961, current acc: 0.723633\n",
      "time taken: 2.758371114730835\n",
      "epoch: 4, training loss: 0.173980, training acc: 0.930043, valid loss: 1.054135, valid acc: 0.723633\n",
      "\n",
      "time taken: 2.789628267288208\n",
      "epoch: 5, training loss: 0.103915, training acc: 0.960227, valid loss: 1.415099, valid acc: 0.710449\n",
      "\n",
      "time taken: 2.78234601020813\n",
      "epoch: 6, training loss: 0.063510, training acc: 0.978456, valid loss: 1.265976, valid acc: 0.709961\n",
      "\n",
      "time taken: 2.75811505317688\n",
      "epoch: 7, training loss: 0.039564, training acc: 0.985204, valid loss: 2.029811, valid acc: 0.721680\n",
      "\n",
      "time taken: 2.77466082572937\n",
      "epoch: 8, training loss: 0.019326, training acc: 0.993016, valid loss: 2.283813, valid acc: 0.716797\n",
      "\n",
      "time taken: 2.761608123779297\n",
      "epoch: 9, training loss: 0.008160, training acc: 0.997159, valid loss: 2.394169, valid acc: 0.720703\n",
      "\n",
      "break epoch:10\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EARLY_STOPPING, CURRENT_CHECKPOINT, CURRENT_ACC, EPOCH = 5, 0, 0, 0\n",
    "while True:\n",
    "    lasttime = time.time()\n",
    "    if CURRENT_CHECKPOINT == EARLY_STOPPING:\n",
    "        print('break epoch:%d\\n'%(EPOCH))\n",
    "        break\n",
    "        \n",
    "    train_acc, train_loss, test_acc, test_loss = 0, 0, 0, 0\n",
    "    for i in range(0, (len(train_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(train_X[i:i+batch_size],dictionary,maxlen)\n",
    "        acc, loss, _ = sess.run([model.accuracy, model.cost, model.optimizer], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : train_Y[i:i+batch_size]})\n",
    "        train_loss += loss\n",
    "        train_acc += acc\n",
    "    \n",
    "    for i in range(0, (len(test_X) // batch_size) * batch_size, batch_size):\n",
    "        batch_x = str_idx(test_X[i:i+batch_size],dictionary,maxlen)\n",
    "        acc, loss = sess.run([model.accuracy, model.cost], \n",
    "                           feed_dict = {model.X : batch_x, model.Y : test_Y[i:i+batch_size]})\n",
    "        test_loss += loss\n",
    "        test_acc += acc\n",
    "    \n",
    "    train_loss /= (len(train_X) // batch_size)\n",
    "    train_acc /= (len(train_X) // batch_size)\n",
    "    test_loss /= (len(test_X) // batch_size)\n",
    "    test_acc /= (len(test_X) // batch_size)\n",
    "    \n",
    "    if test_acc > CURRENT_ACC:\n",
    "        print('epoch: %d, pass acc: %f, current acc: %f'%(EPOCH,CURRENT_ACC, test_acc))\n",
    "        CURRENT_ACC = test_acc\n",
    "        CURRENT_CHECKPOINT = 0\n",
    "    else:\n",
    "        CURRENT_CHECKPOINT += 1\n",
    "        \n",
    "    print('time taken:', time.time()-lasttime)\n",
    "    print('epoch: %d, training loss: %f, training acc: %f, valid loss: %f, valid acc: %f\\n'%(EPOCH,train_loss,\n",
    "                                                                                          train_acc,test_loss,\n",
    "                                                                                          test_acc))\n",
    "    EPOCH += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "heatmap=sess.run(model.alignments,feed_dict={model.X:str_idx(test_X[1:2],dictionary,len(test_X[1].split()))})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1oAAAI/CAYAAABu7KjDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XuYHXWd7/t3d4ckJIEEAyo3jSB8mYiY4SYqKAh6vDDidbio2wvb2ehRN+NRhpnxeH/2xtEZDo7uQR8cgVFRREfRYYt3REQFNQbQ+Y5c4nBRNDFcYkhIeq3zxyq2bVtJp4qia1X3+5VnPb1WrV9Vf9d6Ot397U/9fjXS7/eRJEmSJDVntO0CJEmSJGmmsdGSJEmSpIbZaEmSJElSw2y0JEmSJKlhNlqSJEmS1DAbLUmSJElq2Jy2C+iiv1p2cqfWxL+1v7HtEir7dW9D2yVUcs94997jJWM7tl1CZY8bW9J2CZUcen/3vsXG6Pq2S6hs/twtbZdQybKXzG27hMrmvOjktkuoZs68tiuobHTJI9suobKRnXdtu4RKxn92Vdsl1DL/qJePtF3DVDavuXlafzfeYdd9hv49ARMtSZIkSWqcjZYkSZIkNax757VIkiRJGh698bYrGEomWpIkSZLUMBMtSZIkSfX1e21XMJRMtCRJkiSpYSZakiRJkurrmWiVMdGSJEmSpIaZaEmSJEmqre8crVImWpIkSZLUMBMtSZIkSfU5R6uUiZYkSZIkNcxES5IkSVJ9ztEq1blEKyKWRMTr2q5DkiRJkramc40WsASo1GhFxNhDVIskSZIk/ZEunjp4FrBvRKwENgN3ZubxABHxQeDazDw/IlYDnwaeAfxdRJwGfB84hkGzdmpmXhkRjwM+Bsxl0Hi+KDN/Ps2vSZIkSeqm3njbFQylLiZaZwI3ZeYK4C1TjF2bmQdn5qeKx3My83DgdODtxbbTgHOK4x0K3PZQFC1JkiRp9uhiolXFpyc9/lzx8YfAsuL+1cDfRsRewOdMsyRJkqQKXAyjVBcTrYm28IevYf6k53836fGm4uM4RZOZmZ8EngfcB1wWEU9/COqUJEmSNIt0MdG6F9ipuP8LYHlEzAN2BI4FvlPlYBGxD3BzZn4gIh4FHAR8o8F6JUmSpJnLCxaX6lyilZlrgasi4nrgjcDFwPXFxx/XOOSfA9cXi2scCFzYVK2SJEmSZqcuJlpk5imTNp1RMmbZpMdHT7i/hmKOVmaexWAlQ0mSJEkV9Z2jVapziZYkSZIkDbtOJlqSJEmShoRztEqZaEmSJElSw0y0JEmSJNXnHK1SJlqSJEmS1DATLUmSJEn19cbbrmAomWhJkiRJUsNMtCRJkiTV5xytUiZakiRJktQwGy1JkiRJapinDkqSJEmqzwsWlzLRkiRJkqSGmWhJkiRJqs/FMErZaNVw3H3d+mL66byFbZdQ2S/G5rVdQiVrxza3XUJl8zsYaD+8361vWQ8f797XxdicftslVHbPfd36frHlP3/bdgmVjd1wTdslVDPave9v4wsXtV1CdQt3bruCSvq33tJ2CfUc9fK2K1BN3fqtRZIkSdJwcY5Wqe79yUeSJEmShpyJliRJkqTa+v3xtksYSiZakiRJktQwEy1JkiRJ9bnqYCkTLUmSJElqmImWJEmSpPpcdbCUiZYkSZIkNcxES5IkSVJ9ztEqZaIlSZIkSQ0z0ZIkSZJUX8/raJUx0ZIkSZKkhtloSZIkSVLDPHVQkiRJUn0uhlHqIUm0IuKiiFgVEX/ZwLH2iIhLau77yoj44FaeW//gKpMkSZKkco0nWhHxSOCwzHxshX3mZOaWsucy8w7gxU3VJ0mSJKlBXrC41JSNVkQsA76UmQcWj98MLAKOBr4PHAMsAU7NzCuBrwB7RsRK4A3AvcC5wALgJuDVmbkuIr4FrASOBC6KiM8DnwAWAl8ATs/MRRM/f0R8r/g8NxS1fAt4M4Nk7hxgPnAf8KrMzOIl7F2M2xP4eGa+c9LruxD4XGZ+vnj8CeDizPzCdr6HkiRJkvQHHmyiNSczD4+I5wBvB44DnsegMVoBEBGrgDdk5hUR8a5i3OnF/nMz89Bi3JeAczLzoog4bSuf79PAnwNvj4jdgd0z89qI2Bk4KjO3RMRxwP8AXlTsczhwILABuCYi/i0zr51wzI8Cfwl8PiIWA08GXvEg3xdJkiRpdhiyOVoR8SwGIcwYcF5mnjXp+XnAhcAhwFrgxMxcHREvBd4yYehBwMGZuTIiTgb+BugDdwAvy8w126rjwc7R+lzx8YfAsslPFo3Lksy8oth0AfDUCUM+PeH+k4DPFPc/uZXPdzG/P43wz4EH5m4tBj4TEdcDZwOPm7DPVzNzbWbeV9R75MQDFrXtFxG7AScDn93aaYySJEmShldEjAEfAp4NLAdOjojlk4adCqwrpjqdDbwXIDM/kZkrisDo5cAtRZM1h0HjdkxmHgSsAl4/VS3b02htmTRu/oT7m4qP49RLx35XZXBm3g6sjYiDgBP5faP2buCbxemNfzapxv6kw0x+DIOO9mXAq4B/rlKTJEmSNKv1etN727bDgRsz8+bMvB/4FHDCpDEnMAiAYBDcHBsRI5PGnFzsCzBS3BYW43ZmkGpt0/Y0WncCD4+IpUXMdvx27ANAZt4NrIuIo4pNLweu2Mrw7/H70/1O2sZhPw2cASzOzFXFtsXA7cX9V04a/4yIeFhE7Ag8H7iq5JjnU5zOmJk/3cbnliRJkjS89gRunfD4tmJb6ZjiTLa7gaWTxpwIXFSM2Qy8FriOQYO1nMH0o22astEqDvwu4AfAV4F/n2qfSV4BvK+Yq7WiOFaZ04E3FeMey+AFl7mEQSN28YRtfwf8z4j4MX+crP0A+CyDiO+zk+ZnAZCZdwI/Az62Xa9IkiRJ0sBwJVoPWkQ8EdiQmdcXj3dg0Gj9KbAHg77ir6c6znad7peZHwA+sI3n11DM0crM1QwWn3jguZXAESX7HD1p0+3AEZnZj4iTgNjK8e6cXHdmXg3sP2HTW4vt5zNIq8pqXvTA/YhYAOxH0bVKkiRJ6qTbgb0nPN6L35/5NnnMbcX8q8UMFsV4wEn8YV+wAiAzbwKIiIuBM6cqpPHraD0IhwAfLM57vAt49XR80mKVwo8CZxenOkqSJEnaTv3+eNslTHQNg4XuHsOgoToJOGXSmEsZnHV3NYOF9r6RmX2AiBhlsOjeURPG3w4sj4jdMvM3wDMYnA23TUPTaBXX4HpCC5/3a8Cjp/vzSpIkSWpWcbmn1wOXM1je/Z8z84biMlPXZualDEKWf4mIG4Hf8ofrQzwVuDUzb55wzDsi4p3AtyNiM/AL/nhdiD8yNI2WJEmSpA6ahnlTVWTmZcBlk7a9bcL9jcBLtrLvtyif9nQucG6VOh7sdbQkSZIkSZOYaEmSJEmqrz9cidawMNGSJEmSpIbZaEmSJElSwzx1UJIkSVJ9Q7YYxrAw0ZIkSZKkhploSZIkSarPxTBKmWhJkiRJUsNMtCRJkiTV5xytUjZaNRx8+K/aLqGSvXNh2yVU9ut7ulXzGua1XUJl9412L9C+e6ztCqr57Wj3vsXee3+3/u8B3D3Wra/lJT9Y0HYJle2180/aLqGy0Z3mt11CJSM7du/nyMiibn2/2HLzHW2XoFmme78FSJIkbUPXmiyp85yjVapbfwaUJEmSpA4w0ZIkSZJUn3O0SploSZIkSVLDTLQkSZIk1WeiVcpES5IkSZIaZqIlSZIkqT5XHSxloiVJkiRJDTPRkiRJklSfc7RKmWhJkiRJUsNstCRJkiSpYZ46KEmSJKk+F8MoZaIlSZIkSQ0z0ZIkSZJUn4thlJoRiVZELImI123j+e8WH5dFxCnTV5kkSZKk2WhGNFrAEuCPGq2ImAOQmU8uNi0DbLQkSZKkpvR703vriJly6uBZwL4RsRLYDGwE1gEHAPtHxPrMXFSM+5Ni3AXAB4ptRwPzgA9l5odbqF+SJEnSDDJTEq0zgZsycwXwFuBg4L9n5v4l467MzBWZeTZwKnB3Zh4GHAa8JiIeM52FS5IkSZ3W603vrSNmSqI12Q8y85btGPdM4KCIeHHxeDGwH7A9+0qSJElSqZnaaP1uO8eNAG/IzMsfymIkSZKkGatDKdN0mimnDt4L7FRj3OXAayNiB4CI2D8iFj4E9UmSJEmaRWZEopWZayPiqoi4HrgPuHMrQ1cB4xHxE+B84BwGKxH+KCJGgN8Az3/oK5YkSZJmiH6/7QqG0oxotAAyc6vLthcrDpKZm4GnT3r6b4qbJEmSJDVixjRakiRJklrgHK1SM2WOliRJkiQNDRMtSZIkSfWZaJUy0ZIkSZKkhploSZIkSaqvb6JVxkRLkiRJkhpmoyVJkiRJDfPUQUmSJEn1uRhGKRMtSZIkSWqYiZYkSZKk+vr9tisYSiZakiRJktQwEy1JkiRJ9TlHq5SNVg3fvWaPtkuoZOW87gWXa+aPt11CJfewqe0SKtuFHdouobK9e936lvWw3pa2S6jskQs2tF1CZVvGu/U9bveDf9d2CZXNOXB52yVUNrJoUdslVLNgYdsVVLdkadsVVLLDw3druwTNMt36rUWSJGkKnWuypK4z0SrVrT8DSpIkSVIHmGhJkiRJqq9volXGREuSJEmSGmaiJUmSJKm2fs/raJUx0ZIkSZKkhploSZIkSarPVQdLmWhJkiRJUsNMtCRJkiTV56qDpUy0JEmSJKlhNlqSJEmS1DBPHZQkSZJUn8u7lzLRkiRJkqSGmWhJkiRJqs/l3UvNyEQrItbX3O/0iFjQdD2SJEmSZhcTrT90OvBxYEPbhUiSJEmdYKJVakY3WhGxCPgCsAuwA/DWzPxCRCwELgb2AsaAdwOPAPYAvhkRazLzmJbKliRJktRxM7rRAjYCL8jMeyJiV+B7EXEp8Czgjsx8LkBELM7MuyPiTcAxmbmmxZolSZKk7ui76mCZmd5ojQD/IyKeCvSAPRkkV9cBfx8R7wW+lJlXtlijJEmSpBlmRi6GMcFLgd2AQzJzBXAnMD8z/wM4mEHD9Z6IeFuLNUqSJEnd1etN760jZnqjtRj4dWZujohjgEcDRMQewIbM/DjwPgZNF8C9wE6tVCpJkiRpxpjppw5+AvhiRFwHXAv8e7H98cD7IqIHbAZeW2z/CPDliLjDxTAkSZKk7dBzjlaZGdloZeai4uMa4EklQ1YDl5fs94/APz6kxUmSJEma8WZkoyVJkiRpmvS7M29qOs30OVqSJEmSNO1MtCRJkiTV5xytUiZakiRJktQwGy1JkiRJapinDkqSJEmqrd+hiwhPJxMtSZIkSWqYiZYkSZKk+lwMo5SJliRJkiQ1zERLkiRJUn1esLiUiZYkSZIkNcxES5IkSVJ9ztEqZaIlSZIkSQ0z0aphzdhY2yVU8tuRLW2XUNkGxtsuoZIunpm8oIN/Z1nUsTd64Wj3/u/Nnde9mkfu79b35DlL57ddQmUjixe3XUIl/Y0b2y6hspFN3auZjRvarqCS/vr1bZcwc3kdrVLd+01LkiRJkoaciZYkSZKk+pyjVcpES5IkSZIaZqIlSZIkqT6vo1XKREuSJEmSGmaiJUmSJKk+52iVMtGSJEmSpIbZaEmSJElSwzx1UJIkSVJtfS9YXMpGS5IkSdKMERHPAs4BxoDzMvOsSc/PAy4EDgHWAidm5uqIeCnwlglDDwIOBv4D+AywLzAOfDEzz5yqDk8dlCRJklRfrz+9t22IiDHgQ8CzgeXAyRGxfNKwU4F1mflY4GzgvQCZ+YnMXJGZK4CXA7dk5spin/dn5gHAnwJPiYhnT/W22GhJkiRJmikOB27MzJsz837gU8AJk8acAFxQ3L8EODYiRiaNObnYl8zckJnfLO7fD/wI2GuqQmy0JEmSJNU3RIkWsCdw64THtxXbSsdk5hbgbmDppDEnAhdNPnhELAH+DPj6VIXMmjlaEfHPwPHArzPzwGLbw4BPA8uA1cCfZ+a6tmqUJEmS1K6IeCKwITOvn7R9DoPm6wOZefNUx5lNidb5wLMmbTsT+Hpm7segK51yUpskSZKkCfq96b1t2+3A3hMe71VsKx1TNE+LGSyK8YCTKEmzgI8AP8/M/2973pZZ02hl5reB307aPPH8zAuA509rUZIkSZKadA2wX0Q8JiLmMmiaLp005lLgFcX9FwPfyMw+QESMAn9OMT/rARHxHgYN2enbW8isabS24hGZ+cvi/q+AR7RZjCRJktQ5QzRHq5hz9XrgcuBnwMWZeUNEvCsinlcM+yiwNCJuBN7EH57V9lTg1omnBkbEXsDfMljF8EcRsTIi/utUb8usmaM1lczsR8SUs+skSZIkDa/MvAy4bNK2t024vxF4yVb2/RZwxKRttwGTVyWc0mxPtO6MiN0Bio+/brkeSZIkqVP6vf603rpitjdaE8/PfAXwhRZrkSRJkjRDzJpTByPiIuBoYNeIuA14O3AWcHFEnAr8gsHEN0mSJEnbq0Mp03SaNY1WZp68laeOndZCJEmSJM14s6bRkiRJkvQQ6E15batZabbP0ZIkSZKkxtloSZIkSVLDPHVQkiRJUn0uhlHKREuSJEmSGmaiJUmSJKk+E61SJlqSJEmS1DATLUmSJEm19fsmWmVMtCRJkiSpYSZakiRJkupzjlYpEy1JkiRJapiJliRJkqT6TLRKmWhJkiRJUsNMtGr4Z37ZdgmVrFqzuu0SKtu45f62S6hkvNdru4TKDthl77ZLqOyEHfdpu4RKnjR3U9slVLb7U7a0XUJlo7vu2HYJlYwdcWTbJVQ2ss9BbZdQyeiceW2XUN3cbn0dA4zuvLTtEirpLV/bdgkzVt9Eq5SJliRJkiQ1zERLkiRJUn0mWqVMtCRJkiSpYSZakiRJkurr3lT1aWGiJUmSJEkNs9GSJEmSpIZ56qAkSZKk2lzevZyJliRJkiQ1zERLkiRJUn0mWqVMtCRJkiSpYSZakiRJkupzefdSJlqSJEmS1DATLUmSJEm1uepgORMtSZIkSWrYUDdaEfG8iDjzQex/ekQsaLImSZIkSRP0pvnWEUN96mBmXgpc+iAOcTrwcWBDMxVJkiRJ0tRaa7QiYhnwZeB7wJOBa4CPAe8EHg68FFgOHJqZr4+I84F7gEOBRwJnZOYlEXE08ObMPL447geBa4GdgT2Ab0bEmsw8JiKeWRx/HnAT8KrMXB8RZwHPA7YAX8nMN0/DWyBJkiR1nnO0yrV96uBjgb8HDihupwBHAm8G/qZk/O7F88cDZ23rwJn5AeAO4JiiydoVeCtwXGYezKAZe1NELAVeADwuMw8C3tPEC5MkSZI0e7XdaN2SmddlZg+4Afh6ZvaB64BlJeM/n5m9zPwp8IiKn+sIBgnZVRGxEngF8GjgbmAj8NGIeCGeZihJkiRtP+dolWp7jtamCfd7Ex73KK9t4viR4uMW/rBhnL+VzzUCfDUzT578REQcDhwLvBh4PfD0KSuXJEmSpK1ou9Fqwi+A5RExD9iRQcP0neK5e4GdgDUM5oJ9KCIem5k3RsRCYE8GpxcuyMzLIuIq4OZpfwWSJElSR/U7lDJNp7ZPHXzQMvNW4GLg+uLjjyc8/RHgyxHxzcz8DfBK4KKIWAVczWBe2E7Al4pt3wHeNI3lS5IkSZqBRvp9Vwmp6qg9j+3Um7bq7tVtl1DZxi33t11CJeO97v0p54Bd9m67hMpO2HGftkuo5OTRe9ouobJHHbNp6kFDZnTXndouoZKxIw5vu4TKRvY5qO0SKhmZM6/tEqqbu2PbFVQ2uvPStkuopHfP2rZLqGXefk8emXpUu9b+2dOm9XfjpV+8YujfE5gZpw5KkiRJakv3/t48LTp/6qAkSZIkDRsTLUmSJEm1uRhGORMtSZIkSWqYiZYkSZKk+ky0SploSZIkSVLDTLQkSZIk1eYcrXImWpIkSZLUMBMtSZIkSbWZaJUz0ZIkSZKkhploSZIkSarNRKuciZYkSZIkNcxEq4Yn7vDwtkuoZO6SsbZLqOzu8fvaLqGS+3r3t11CZXvP3aXtEiqb3x9pu4RK1m+c23YJld1z/Ya2S6hsZPSutkuoZMlO17ddQmWjS3Ztu4Rq5u7YdgXVzV/YdgWV9ToWY/T+s3v/9wDY78ltVzC1jv18ni4mWpIkSZLUMBMtSZIkSbV1LNycNiZakiRJktQwGy1JkiRJapinDkqSJEmqrd9zMYwyJlqSJEmS1DATLUmSJEm1uRhGORMtSZIkSWqYiZYkSZKk2vpesLiUiZYkSZIkNcxES5IkSVJtztEqZ6IlSZIkSQ0z0ZIkSZJUm9fRKmeiJUmSJEkNM9GSJEmSVFu/33YFw8lES5IkSZIaNlSJVkR8HtgbmA+ck5kfiYj1wDnA8cB9wAnABmAVsH9mbo6InYGfAPsDjwPOBRYANwGvzsx1EfEt4PvAMcAS4NTMvDIixoCzgKOBecCHMvPD0/SSJUmSpE5zjla5YUu0Xp2ZhwCHAm+MiKXAQuB7mfkE4NvAazLzXuBbwHOL/U4CPpeZm4ELgb/KzIOA64C3Tzj+nMw8HDh9wvZTgbsz8zDgMOA1EfGYh/JFSpIkSZrZhirRYtBcvaC4vzewH3A/8KVi2w+BZxT3zwPOAD4PvIpBg7QYWJKZVxRjLgA+M+H4n5twnGXF/WcCB0XEi4vHi4vPe0tDr0mSJEmasUy0yg1NoxURRwPHAU/KzA3FqX7zgc2Z+cAUu3GKmjPzqohYVuw3lpnXF43WtmyafBxgBHhDZl7e2IuRJEmSNKsN06mDi4F1RZN1AHDEduxzIfBJ4GMAmXk3sC4ijiqefzlwxVb2fcDlwGsjYgeAiNg/IhbWeQGSJEmSBEOUaAFfBk6LiJ8BCXxvO/b5BPAe4KIJ214BnBsRC4CbGZxWuC3nMTiN8EcRMQL8Bnh+tdIlSZKk2cnl3csNTaOVmZuAZ5c8tWjCmEuASyY8dyRwSWbeNWHMSkrSsMw8esL9NRRztDKzB/xNcZMkSZKkB21oGq2qIuIfGTRmz2m7FkmSJGm2cjGMcp1ttDLzDW3XIEmSJEllOttoSZIkSWpfv2+iVWaYVh2UJEmSpBnBREuSJElSbf1e2xUMJxMtSZIkSWqYiZYkSZKk2nrO0SploiVJkiRJDTPRkiRJklSbqw6WM9GSJEmSpIaZaEmSJEmqrd8z0SpjoiVJkiRJDTPRquGMfe9ou4RKbv/5krZLqGzN5l3aLqGSNWPd+680uqXtCqrbNNqtv5jd2t+x7RIqu2v13LZLqGz96FjbJVRy0CfWtF1CZbv/6nNtl1Bdr992BZWMLF7QdgmVjS7t1s/qzTesbruEWnY87rS2S5hSv1v/3aaNiZYkSZpZOtZkSZqZbLQkSZIkqWHdO99JkiRJ0tAYtsUwIuJZwDnAGHBeZp416fl5wIXAIcBa4MTMXF08dxDwYWBnoAcclpkbI2Iu8EHg6GL732bmZ7dVh4mWJEmSpBkhIsaADwHPBpYDJ0fE8knDTgXWZeZjgbOB9xb7zgE+DpyWmY9j0FRtLvb5W+DXmbl/cdwrpqrFREuSJElSbb3humDx4cCNmXkzQER8CjgB+OmEMScA7yjuXwJ8MCJGgGcCqzLzJwCZuXbCPq8GDii294ApVzYy0ZIkSZI0U+wJ3Drh8W3FttIxmbkFuBtYCuwP9CPi8oj4UUScARARDyzh/e5i+2ci4hFTFWKjJUmSJKm2fn9kWm8PoTnAkcBLi48viIhji+17Ad/NzIOBq4H3T3UwGy1JkiRJM8XtwN4THu9VbCsdU8zLWsxgUYzbgG9n5prM3ABcBhxcPLcBeOCigp8ptm+TjZYkSZKk2vr96b1N4Rpgv4h4TLFS4EnApZPGXAq8orj/YuAbmdkHLgceHxELigbsacBPi+e+yGBxDIBj+cM5X6VstCRJkiTNCMWcq9czaJp+BlycmTdExLsi4nnFsI8CSyPiRuBNwJnFvuuAf2DQrK0EfpSZ/1bs81fAOyJiFfBy4P+ZqpaR/na0hfpDvz72aZ16027/+ZKpBw2ZNZvmt11CJWvGureA52invooHNo0O1apGU9qx1703eafeeNslVLZ+dKztEio5aMnaqQcNmd2P7djfZTv4f29k8YK2S6hsdOkubZdQyeYbVrddQi2LP/a1of/ht/LRz5vW/3QrfnHp0L8nYKIlSZIkSY3r3p/hJUmSJA2Nh3glwM4y0ZIkSZKkhnUi0YqIy4BTioenZOb/KrYfDbw5M49v4HMcDdyfmd99sMeSJEmSZguXfCjXiUQrM5+TmXcBS4DXPUSf5mjgyQ/RsSVJkiTNIkORaEXEW4BNmfmBiDgbeEJmPj0ing6cCjwFOBQ4C9g3IlYCXwX+DVgUEZcABwI/BF6Wmf3iKs7vZ/AarwFem5mbImI1cGhmromIQ4sxrwROA8Yj4mXAGzLzyul6/ZIkSVJX9ZyjVWpYEq0rgaOK+4cyaJ52KLZ9e8K4M4GbMnNFZr6l2PanwOnAcmAf4CkRMR84HzgxMx/PoNl67dY+eWauBs4Fzi6ObZMlSZIkqbZhabR+CBwSETsDm4CrGTRcRzFowrblB5l5W2b2GFxYbBkQwC2Z+R/FmAuApz4UhUuSJEmzWb8/Mq23rhiKRiszNwO3MDiF77sMmqtjgMcyuKLztmyacH+cqU+H3MLvX3e3roorSZIkqROGotEqXAm8mcGpglcymDP148ycuI7JvcBO23GsBJZFxGOLxy8HrijurwYOKe6/qMaxJUmSJGmbhq3R2h24OjPvBDYy6bTBzFwLXBUR10fE+7Z2oMzcCLwK+ExEXAf0GMzBAngncE5EXMsgAXvAF4EXRMTKiDgKSZIkSVPq9Uem9dYVI30Xvq/s18c+rVNv2u0/X9J2CZWt2dStszrXjA3FAp6VjHbqq3hg02h3vrkC7Njr3pu8U2986kFDZv3oWNslVHLQkrVtl1DZ7scO099lt0MH/++NLF7QdgmVjS7dpe0SKtl8w+q2S6hl8ce+NvQ//L6/xwun9T/dE+/43NC/JzAky7tLkiRJ6qbu/WljenTsT1SSJEmSNPxMtCRJkiTV1qV5U9PJREuSJEmSGmaiJUmSJKm2Ll0kVRaeAAAgAElEQVREeDqZaEmSJElSw0y0JEmSJNXWa7uAIWWiJUmSJEkNM9GSJEmSVFsf52iVMdGSJEmSpIaZaEmSJEmqrddvu4LhZKIlSZIkSQ0z0ZIkSZJUW885WqVstGr4u5v2aLuESq7r39V2CZXdPXZ32yVUsn58Y9slVLbH3CVtl1DZ4WMPa7uESo6/v3tfF/setLbtEiobW9R2BdXMf9oBbZdQ2cgTDmu7hGp6HVxserR7JxmNLFzcdgmVzH38nW2XoFmme/+rJUmSJGnImWhJkiRJqs3l3cuZaEmSJElSw0y0JEmSJNXWwVmR08JES5IkSZIaZqIlSZIkqTbnaJUz0ZIkSZKkhploSZIkSarNOVrlTLQkSZIkqWEmWpIkSZJqM9EqZ6IlSZIkSQ0z0ZIkSZJUm6sOljPRkiRJkqSGtZ5oRcQbgdcCP8rMl27H+BXAHpl5WfH4ecDyzDwrInYDvgTMBd4I/DVwSmbetZ21vBI4NDNfX+vFSJIkSbNMz0CrVOuNFvA64LjMvG2qgRExB1gBHApcBpCZlwKXFkOOBa7LzP9aPL6y+XIlSZIkadtabbQi4lxgH+B/R8T5wFHF4w3AX2Tmqoh4B7Bvsf0/gacAO0bEkcD/BHZk0HidB/xd8dyhwJOAnzFIqNZExMsYpFxzge8Dr8vM8Yh4FYPk6y7gJ8Cm6XjtkiRJ0kzQc45WqVbnaGXmacAdwDHAMuDHmXkQ8DfAhROGLmeQep0MvA34dGauyMxPTzjWyknP3ffAcxHxJ8CJwFMycwUwDrw0InYH3smgeTuy+DySJEmS9KAM02IYRwL/ApCZ3wCWRsTOxXOXTmycajgWOAS4JiJWFo/3AZ4IfCszf5OZ9wOf3sYxJEmSJGm7DMMcre3xuwe5/whwQWb+9cSNEfH8B3lcSZIkaVbrt13AkBqmROtK4KUAEXE0sCYz7ykZdy+wU8Vjfx14cUQ8vDj+wyLi0Qzmaj0tIpZGxA7AS+oWL0mSJEkPGKZG6x3AIRGxCjgLeMVWxn0TWB4RKyPixO05cGb+FHgr8JXi+F8Fds/MXxaf92rgKgaLZ0iSJEnaTr1pvnXFSL9v2FfVm5ed3Kk37brx7bqM2FC5e/zBTMmbfuvHN7ZdQmV7zF3SdgmVHT72sLZLqOT4+7v3dbHv49a2XUJlY4varqCa+U87oO0SKht5wmFtl1BNr0u/ihVGh+lv39tnZOHitkuopL/uzrZLqGXH/+v1Q7+k3+ceecq0/m78wl99cujfE+jOHC1JkiRJQ6g30om+Z9p1788nkiRJkjTkTLQkSZIk1dapOTXTyERLkiRJkhpmoiVJkiSptg4uPzMtTLQkSZIkqWEmWpIkSZJq67noYCkTLUmSJElqmImWJEmSpNp6GGmVMdGSJEmSpIaZaEmSJEmqzetolTPRkiRJkqSG2WhJkiRJUsM8dbCGFfePtV1CJRvn7tx2CZX9dnR+2yVU8rs5m9suobK9Rxe2XUJle4x3629DvX73Jgevu23HtkuobEvHvifvtestbZdQ2Q4Pe1jbJVQzr1s/QwCYO6/tCirr9zp2mdo7/rPtCmYsl3cv163fWiRJkiSpA0y0JEmSJNXWsWxz2phoSZIkSVLDTLQkSZIk1eby7uVMtCRJkiSpYSZakiRJkmpz1cFyJlqSJEmS1DATLUmSJEm1uepgORMtSZIkSWqYiZYkSZKk2ky0yploSZIkSVLDTLQkSZIk1dZ31cFSJlqSJEmS1DATrUJEHA3cn5nfbbsWSZIkqSuco1XOROv3jgae3HYRkiRJkrqvE4lWRCwELgb2AsaAfwGemJkvjIgTgE8Bixk0jj/NzH0iYl/gQ8BuwAbgNZn57xGxG3Au8Kji8KcDtwOnAeMR8TLgDZl55fS9QkmSJEkzSScaLeBZwB2Z+VyAiFgM/LfiuaOA64HDGLye7xfbPwKclpk/j4gnAv8LeDpwDnB2Zn4nIh4FXJ6ZfxIR5wLrM/P90/aqJEmSpI7z1MFyXWm0rgP+PiLeC3wpM6+MiJsi4k+Aw4F/AJ7KIO26MiIWMTgN8DMR8cAx5hUfjwOWT9i+czFekiRJkhrRiUYrM/8jIg4GngO8JyK+DnwbeDawGfgacD6DRustDE4hvCszV5QcbhQ4IjM3Ttw4ofGSJEmStJ36bRcwpDqxGEZE7AFsyMyPA+8DDgauZDC/6urM/A2wFAjg+sy8B7glIl5S7D8SEU8oDvcV4A0Tjv1AM3YvsNN0vB5JkiRJM1snEi3g8cD7IqLHIMF6LXAD8AgGyRbAKuCRmflAU/1S4J8i4q3ADgwWzPgJ8EbgQxGxisHr/zaDhTC+CFxSLK7hYhiSJEnSduh5weJSnWi0MvNy4PKSp+ZNGPMXk/a5hcEiGpOPtQY4sWT7fwAHPehiJUmSJM16nWi0JEmSJA0nVx0s14k5WpIkSZLUJSZakiRJkmoz0SpnoiVJkiRJDTPRkiRJklSb19EqZ6IlSZIkSQ0z0ZIkSZJUm9fRKmejJUmSJGnGiIhnAecAY8B5mXnWpOfnARcChwBrgRMzc3Xx3EHAh4GdGazzcVhmboyIQ4DzgR2By4D/npnbPGvSUwclSZIk1dab5tu2RMQY8CHg2cBy4OSIWD5p2KnAusx8LHA28N5i3znAx4HTMvNxwNHA5mKffwJeA+xX3J411ftioyVJkiRppjgcuDEzb87M+4FPASdMGnMCcEFx/xLg2IgYAZ4JrMrMnwBk5trMHI+I3YGdM/N7RYp1IfD8qQqx0ZIkSZI0U+wJ3Drh8W3FttIxmbkFuBtYCuwP9CPi8oj4UUScMWH8bVMc8484R0uSJElSbTNoefc5wJHAYcAG4OsR8UMGjVhlJlqSJEmSZorbgb0nPN6r2FY6ppiXtZjBohi3Ad/OzDWZuYHBohcHF+P3muKYf8REq4an7fnLtkuoZPmvFrVdQmW/vH9B2yVU8us5C9suobJ72i6ghvG2C6joV/15bZdQ3V1tF1DdeL9b6wov+vHGtkuobJexH7ddQjVzuvd35NHF3fs5MvLwpW2XUMn4z1a3XUI9r2q7gKn1hivTugbYLyIew6AZOgk4ZdKYS4FXAFcDLwa+kZn9iLgcOCMiFgD3A08Dzs7MX0bEPRFxBPB94L8A/zhVId37TiRJkiRJJYo5V68HLgd+BlycmTdExLsi4nnFsI8CSyPiRuBNwJnFvuuAf2DQrK0EfpSZ/1bs8zrgPOBG4Cbgf09Vi4mWJEmSpNqmWnJ9umXmZQxO+5u47W0T7m8EXrKVfT/OYIn3yduvBQ6sUoeJliRJkiQ1zERLkiRJUm1DNUNriJhoSZIkSVLDTLQkSZIk1TZsc7SGhYmWJEmSJDXMREuSJElSbb1uXc5w2phoSZIkSVLDTLQkSZIk1dZz3cFSJlqSJEmS1DATLUmSJEm1mWeVM9GSJEmSpIbZaEmSJElSwzrXaEXEkoh4Xc19l0XEKRMevzIiPthcdZIkSdLs0pvmW1d0rtEClgC1Gi1gGXDKVIMkSZIk6cHo4mIYZwH7RsRK4KvFtmczmIf3nsz8dESMAH83eXux758U+14ArAP2iIgvA/sC/5qZZ0zvy5EkSZK6y+Xdy3Ux0ToTuCkzVwDfA1YATwCOA94XEbsDL9zK9jOBKzNzRWaeXRxvBXAi8HjgxIjYe1pfjSRJkqQZp4uN1kRHAhdl5nhm3glcARy2je1lvp6Zd2fmRuCnwKOno3BJkiRpJuhP860rut5oNWHThPvjdPN0SkmSJElDpIuN1r3ATsX9Kxmc7jcWEbsBTwV+sI3tE/eVJEmS9CC56mC5zjVambkWuCoirgeeBKwCfgJ8AzgjM38F/OtWtq8CxiPiJxHxl628AEmSJEkzXidPk8vMyUu0v2XS8/1i2+Ttm4GnT9r3/AnPH99clZIkSdLM56qD5TqXaEmSJEnSsOtkoiVJkiRpOJhnlTPRkiRJkqSGmWhJkiRJqq1LKwFOJxMtSZIkSWqYiZYkSZKk2vrO0iploiVJkiRJDbPRkiRJkqSGeeqgJEmSpNpcDKOciZYkSZIkNcxES5IkSVJtPRfDKGWiJUmSJEkNM9GqYe2vF7ZdQiWrN3erXoBfze3W3wDWjfqXnOkwNjLSdgmV3Dfara9jgHXjc9suobJ7xrr1Pj+67QJqGDugg1XvsEPbFVQysutubZdQ3S7dqnl0w31tlzBj+VtQuW79dJIkSZpKx5osSTOTiZYkSZKk2pyjVc5ES5IkSZIaZqIlSZIkqTavo1XOREuSJEmSGmaiJUmSJKm2vnO0SploSZIkSVLDTLQkSZIk1eYcrXImWpIkSZLUMBMtSZIkSbU5R6uciZYkSZIkNcxGS5IkSZIa5qmDkiRJkmpzMYxyJlqSJEmS1LBONloRsb74uEdEXFLcf2VEfLDCMSqNlyRJkvTHev3+tN66otOnDmbmHcCL265DkiRJkibqdKMVEcuAL2XmgZO2Pxd4K/BnwAhwLvCo4unTM/OqCWN3AlYB+2fm5ojYGfjJA48f+lchSZIkdVd3Mqbp1clTB7clIl4AnAk8JzPXAOcAZ2fmYcCLgPMmjs/Me4FvAc8tNp0EfM4mS5IkSVJdnU60SjwdOBR4ZmbeU2w7DlgeEQ+M2TkiFk3a7zzgDODzwKuA10xDrZIkSVLn9cy0Ss20RusmYB9gf+DaYtsocERmbpw4cELjRWZeFRHLIuJoYCwzr5+eciVJkiTNRDPt1MFfMDg98MKIeFyx7SvAGx4YEBErtrLvhcAngY89pBVKkiRJM0h/mv91xUxrtMjMfwdeCnwmIvYF3ggcGhGrIuKnwGlb2fUTwC7ARdNTqSRJkqSZqpOnDmbmouLjauDA4v75wPnF/R8DyyfscmLJMf7P+MKRwCWZeVfzFUuSJEkzU6/tAoZUJxutpkXEPwLPBp7Tdi2SJEmSus9GC8jMN0w9SpIkSdJkrjpYbsbN0ZIkSZKktploSZIkSaqtSysBTicTLUmSJElqmI2WJEmSJDXMUwclSZIk1eby7uVMtCRJkiSpYSZakiRJkmrr910Mo4yJliRJkiQ1zERLkiRJUm1esLiciZYkSZIkNcxES5IkSVJtrjpYzkarhl2Wbmi7hEoeccfctkuorLdlXtslVDI6Z6TtEirb1L2SO2duBycHL2RL2yVUNjY+1nYJlWze2L0fveP//ou2S6hkZEH3fu6Nrv9d2yVUNrL5/rZLqGZ9t35/U/d177u9JEmSpKHRd45WKedoSZIkSVLDTLQkSZIk1eaqg+VMtCRJkiSpYSZakiRJkmrrd3Dxp+lgoiVJkiRJDTPRkiRJklSb19EqZ6IlSZIkSQ0z0ZIkSZJUm9fRKmeiJUmSJEkNs9GSJEmSpIZ56qAkSZKk2rxgcTkTLUmSJElqmImWJEmSpNq8YHE5Ey1JkiRJatisSrQi4v8FXgb8BrgV+CHwNeBcYAFwE/DqzFzXWpGSJElShzhHq9ysSbQi4jDgRcATgGcDhxZPXQj8VWYeBFwHvL2dCiVJkiTNFLMp0XoK8IXM3AhsjIgvAguBJZl5RTHmAuAzbRUoSZIkdY0XLC43axItSZIkSZous6nRugr4s4iYHxGLgOOB3wHrIuKoYszLgSu2dgBJkiRJf6jX70/rrStmzamDmXlNRFwKrALuZDAf627gFcC5EbEAuBl4VXtVSpIkSZoJZk2jVXh/Zr6jaKq+DfwwM1cCR7RclyRJktRJw5YxRcSzgHOAMeC8zDxr0vPzGCyIdwiwFjgxM1dHxDLgZ0AWQ7+XmadN2vdSYJ/MPHCqOmZbo/WRiFgOzAcuyMwftV2QJEmSpGZExBjwIeAZwG3ANRFxaWb+dMKwU4F1mfnYiDgJeC9wYvHcTZm5YivHfiGwfntrmVWNVmae0nYNkiRJ0kwyZNfROhy4MTNvBoiITwEnABMbrROAdxT3LwE+GBEj2zposcbDm4C/AC7enkJm02IYkiRJkma2PYFbJzy+rdhWOiYztzBYt2Fp8dxjIuLHEXHFhAXzAN4N/D2wYXsLsdGSJEmSVFuP/rTeHkK/BB6VmX/KIL36ZETsHBErgH0z81+rHMxGS5IkSdJMcTuw94THexXbSsdExBxgMbA2Mzdl5lqAzPwhcBOwP/Ak4NCIWA18B9g/Ir41VSGzao6WJEmSpBntGmC/iHgMg4bqJGDyOg2XMrjE09XAi4FvZGY/InYDfpuZ4xGxD7AfcHNmXgv8E0CxMuGXMvPoqQqx0ZIkSZJUW3+ILiKcmVsi4vXA5QyWd//nzLwhIt4FXJuZlwIfBf4lIm4EfsugGQN4KvCuiNgM9IDTMvO3dWux0ZIkSZI0Y2TmZcBlk7a9bcL9jcBLSvb7LPDZKY69GpjyGlpgoyVJkiTpQRiy5d2HhothSJIkSVLDTLQkSZIk1dY30SploiVJkiRJDTPRkiRJklTbMK06OExstGq4b/3ctkuoZGN/rO0SKtswNtJ2CZXc161yAdgw0r1vipvbLqCiBR37OgYY6+/QdgmV3TPWrZMzYqzXdgmVje2/rO0Sqhnr3s+9kV13a7uE6nbbve0Kqrn1trYr0CxjoyVJkiSpNlcdLNetPwNKkiRJUgeYaEmSJEmqzTla5Uy0JEmSJKlhJlqSJEmSanOOVjkTLUmSJElqmImWJEmSpNr6JlqlTLQkSZIkqWE2WpIkSZLUME8dlCRJklRbz+XdS5loSZIkSVLDTLQkSZIk1eZiGOVMtCRJkiSpYSZakiRJkmpzjlY5E60SETHWdg2SJEmSuqsziVZELAQuBvYCxoB3A+8FDs3MNRFxKPD+zDw6InYDPgnsAVwNPAM4pBj3eWBvYD5wTmZ+pDj+euDDwHHA/w18Z1pfoCRJktRBztEq16VE61nAHZn5hMw8EPjyNsa+HfhGZj4OuOT/b+/ug+Sq6jSOf4fwEmQkBBAEBLMoPAiEQBLASOIGSFARRJQX2aAIFG5AYdFCUAqBSEpB1qXwZUUKFRBEIiAvooHIi8RIwouEhLcfFEtURMFQMRBDIJmZ/eOcJk2ne2a65870dPJ8UqmZvn373N89fc+559xz7h1gh7L3ToiIMcBY4DRJW+TlmwDzcvruZJmZmZmZWcNaqaO1EJgs6SJJEyJiaTfrjgd+DhARM4ElZe+dJulRYC5pZGunvLwDuLH4sM3MzMzM1l6dXV0D+r9VtExHKyKeBkaTOlzTJZ0LrGL1PgztKQ1JE0lTA8dFxCjgkbLPrYiIjqLjNjMzMzOzdU/LdLQkbQssj4hrgItJna5FwJi8yifLVp8DHJU/dxAwPC8fBiyJiOWSdgHePwChm5mZmZmttboG+F+raJmHYQAjgYsldQIrgZOBjYEfSboAuLds3WnAdZI+TXoYxt+BV0n3dU2V9CQQpOmDZmZmZmZmhWqZjlZE3AHcUeWtnassWwp8KCJWSRoH7B0Rr+f3PlIj/fZiIjUzMzMzW3e00n1TA6llOlp12gGYIWk94A3gpCbHY2ZmZmZm65C1sqMVEc8AezU7DjMzMzOztV0r3Tc1kFrmYRhmZmZmZmatwh0tMzMzMzOzgq2VUwfNzMzMzGxgdHV1NjuEQckjWmZmZmZmZgXziJaZmZmZmTWs0w/DqMojWmZmZmZmZgXziJaZmZmZmTWsy3+wuCqPaJmZmZmZmRXMI1pmZmZmZtYw36NVnUe0zMzMzMzMCuYRLTMzMzMza5jv0arOHa0GtA9f0ewQ6tK+bGizQ6jb8I7WGmztYEizQ6jbRuu1NTuEuq1ssZDbO1vvxPP2ro5mh1C3DTpaK59XvtF69cWq+HOzQ6hL28YbNjuEug1Z8XqzQ6hbW4v9kdquV5Y1OwRbx7ijZWZmZmZmDev0iFZVrTVsYGZmZmZm1gI8omVmZmZmZg3r8lMHq/KIlpmZmZmZWcE8omVmZmZmZg3zUwer84iWmZmZmZlZwdzRMjMzMzMzK5inDpqZmZmZWcM6/TCMqjyiZWZmZmZmVjCPaJmZmZmZWcP8MIzqPKJlZmZmZmZWMI9omZmZmZlZwzo9olWVR7TMzMzMzMwK5hEtMzMzMzNrmO/Rqq6lR7QkLZK0ZQHpLCsiHjMzMzMzM/CIlpmZmZmZ9YH/jlZ1LdPRkrQJMAN4FzAEuCC/daqkQ4ENgCMj4ilJmwM/BnYElgOfi4gFktqB7wJjgS5gWkTcWLaNLYHbgOkRcfsA7ZqZmZmZma1lWmnq4IeBFyJiVETsDszMyxdHxGjgB8AZedk04JGI2AM4G7g6L/8asDQiRub37i4lLmlr4HbgXHeyzMzMzMx6p6ura0D/t4pW6mgtBCZLukjShIhYmpfflH8+DIzIv48HfgoQEXcDW0jaFJgEfL+UYEQsyb9uANwFnBkRs/p1L8zMzMzMbK3XMh2tiHgaGE3qcE2XdG5+6/X8s4PGp0KuInXUPtSnIM3MzMzM1jGdXV0D+r9VtExHS9K2wPKIuAa4mNTpqmU2MCV/biJpeuErwCzg82VpDs+/dgEnALtIOqv46M3MzMzMbF3SMh0tYCTwgKT5wHnA9G7WPR8YI2kBcCFwXF4+HRgu6TFJjwL7lz4QER3AMcABkk7ph/jNzMzMzNY6XQP8r1W0tdINZYPF8/se0FKZ9te/bNbsEOr2UudGzQ6hLouHDGl2CHV7rZUus2Qr25odQX22WtVSVQUAW3asanYIdVvR1loHs4Yv6XmlQeadH2ytY7lt4w2bHULdhmzT5z8LOuDattu22SHUpePxp5odQkPav3njoD/7bfK2EQNaSfxr+aJBnyfQWiNaZmZmZmZmLaFl/o6WmZmZmZkNPq30gIqB5BEtMzMzMzOzgnlEy8zMzMzMGuZnPlTnES0zMzMzM7OCeUTLzMzMzMwa1kqPXB9IHtEyMzMzMzMrmEe0zMzMzMysYb5HqzqPaJmZmZmZmRXMI1pmZmZmZtYwj2hV5xEtMzMzMzOzgnlEy8zMzMzMGubxrOo8omVmZmZmZlawNs+pNDMzMzMzK5ZHtMzMzMzMzArmjpaZmZmZmVnB3NEyMzMzMzMrmDtaZmZmZmZmBXNHy8zMzMzMrGDuaJmZmZmZmRXMHa1+ImkzSac0O45aJF0naYGkLxaQ1raSbmjws5+V9L0a7y2rI51u81vSH/LPEZL+o/5I+089+1nxudMlva3oeBqI48eSXpL0WNmyzSXNkvRM/jm8SbF9TNJX+vD5QZHHg5mkX+fy95YyKGmipF8VtI2Jkj5Qx/qnSXpS0rW9XH9PSQeXvX7zuJH0DknzJD0iaUJpf+uIpWYd12z15msv02z43FdZPw9U3pXq4PJzWb3bHszfs/WdpEWStiwgnYbO99a63NHqP5sBdZ1sJA3pp1gqt/NOYO+I2CMiLunlZ9av9V5EvBARRxQWYGOq5ncp7ogoNSZGAIOqo9UHpwODoRNwJfDhimVfAe6KiJ2Au/LrARcRt0bEhX1IYrDk8aAVEQdHxD9poM6rw0Sgng7BKcDkiJjS04q5jtgTeLOjVXHcHAgsjIi9ImJ22f6uDSZSX772Rl+OgxE0sX4eJOcy64OBakeZ9VbNxrP12YXAeyTNB1YCL0bEIQD5qtdDEXGlpEXA9cBk4FuSpgLzgP1JJ6wTI2K2pN2AnwAbkjrIn4yIZ3J6I4BfRcTu+fUZQDvpJLpGWsCdwHY5tlOBV4HLSA3KZ4ETImKJpHuB+cB44DpJNwPXApsAtwCnR0R7+fYlzc3beTzHci9wRo75UmAo8BpwfEREzqvt83rbAddExLTyjJR0NXBTRNycX18LzIiIW7rJ7xXAEmAXYGdJyyKiPa/3vrzeVcB38rKJwEbA9yPih7W+1P4kqZ2Ur8OBDYBzIuIWSZsAM4B3AUOAC4CtgW2BeyQtjoj9mxEzQETcl4+BcoeR8hRSPt8LnFXkdvM2ZwJzSY3FB0llZBqwFTAF2BUYGxFfkHQl8AowFngncGZE3CBpInBGZfkENqUijyUdlNPfiFRWjo+IZZIuBD4GrALujIgzeoj9ZmB7Unm4NCIuz1c6LwUOIZWRw4DlwAJg54hYKWlT4FFgZ2A3apfbanXIEBo41iV9GXg9Ir4j6RJgVEQcIOkA4ERgv5yn5WVwFnA70J5HCHYHHgaOjYguSQcC/006Bz0InBwRr+f6cGxELJY0Nq/zWWAq0CHpWODUXI/VivcyYEfgN/k7n5BfLwc+FxELJJ0PvCcv/3Peh40ljQe+CWyc9+kK4Fv5vbHAOODJshiPBU4j1cvzgFMiokPS8cBXgX/m7+v1nvK5yn5UlvufAvtGxCckHQb8HBhGqlufiIgdJb0H+D7wjry/J0XEU5LeQTpWdsjJnw78tZ58rUPlcQDwEaALmB4R10tqI+XrW5azZv28BNhW0kzS9/XLiDizgBirqjyXli3/KHAOcCjQRkVeRsScsnXfTo0yGxEr+yv2ini/BhwL/AP4C6ns/ZYq9cUAxVPtHHYRFWU9IibmY/VnpLr3flLbaExeb416M6e/DPghMAn4PPD7fogX4FRJh5LOz0fmsrU58GPWrGPage+S6pEuYFpE3Fi2jS2B20jH/u19idcGN49o9Z+vAM9GxJ7Al3tY9+WIGB0RP8+v14+IfUgnw/PysqmkimVPUsF9vpdxVEvrY6XY8on1auCsiNgDWFi2HsCGETE2Ir5NagReGhEju9n+9cBRAJK2AbaJiIeAp4AJEbEXcC7wjbLP7AN8EtgDODJXuuV+RGpsIWkYqVFdWTFV5vdo4L8iYucq683O+34JqaG4NCL2BvYGTpL0bzX2rb+tAA6PiNGkRvK3c4Pkw8ALETEqNwBmRsR3gBeA/ZvZyerG1hHxt/z730kdw/7wXuDbpA71LqSr4eNJnfuzq6y/TX7/EFKjrqbKPM4nxnOASfk7egj4kqQtgMOB3XIZmt6LuE+IiDGksnxaTmMTYG5EjDOLRfsAAAkPSURBVALuIzWSXyV1Uj+aP/cp0kWHlXRfbquV+0aP9dmkzgo53nZJG+Rl95Wt92YZjIhSnbdXjmFXUkNkP0lDSaOgR+e6ZH3g5Fobj4hFpAbiJWV1Vk0RMZX8vZFGSB7JeXQ2Kc9KdiV9l8eQ6qTrc/rXl6U1v+K910rvSXofcDSwX653OoApud6bRuq8jc/baURlub+MNPIGKe8fI32P+5I6eQCXkzpMY0hl4H/z8ktJ+bc3qa69ot58rUN5XTw3xzyK1Ai+OOfPJ2osr6yfyesdDYwEjpa0fUFx9oqkw3NcB0fEYqrkZfn6PZTZgYi3FNcoUke2dD7trr7ob2ucw7pZ9zzg7ojYDbiB1R1aqF5vQqo75+X0+9TJ6iHexbnu/wGpfEEq69XqmK+R6tuR+b27S4lL2prUhjnXnay1nztag8P1Fa9vyj8fJjUUIF3ZOVvSWcC7y0/4PaiW1ptyx2WziPhdXnQV8MEasY0DfpF//1mN7c0ASlMvjiJVlJCuvP5C6T6eS0hX40tmRcTLeZ9uIjVO3pRj2ylf6ToGuDEiVtXYfskDEfFcD+sAHAR8Jl9BnQdsAezUi8/1hzbgG5IWkK4+bkfqoCwEJku6SNKEiFjapPgaEhFdpCt6/eG5iFgYEZ3A46Tpil2kPBtRZf2bI6IzIp6g/s7f+0mN5jn5eDkOeDewlNRJ/pGkT5CuavbkNEmPkhqi25OOuTeA0j1N5eX1CuD4/PvxwE96UW6rlftGj/WHgTH5yvzrpLpoLKmx31Pj/IGIeD5/P/NzLCJ9b0/XiL1I40kjQUTE3cAWeT8Abq2jHq3mQGAM8GDO0wNJncl9gXsj4h8R8QZr1u+9Va3cP5s7ePsA/0PKtwnA7HwF/QOkenY+6Qr/NjmtScD38vJbgU3z+v1tPHBdRHRExIvA70idw1rLq7krIpZGxArgCVKZGygHkEbiP1o2+tObvFyjzA5EsNl+wC0RsSJ3+m4jdUS6qy/6Wz3nsPGk0VoiYiZpVLOkWr0J6SLHjRSnVrzV6tVadcwk0ugy+b3SfmxAmk5/ZkSURnxtLeapgwNjFW/t1A6teP9fFa9L00w6yN9RRPxM0jzSVbJfS/rPXKh7Sn+NtOpUGVu3IuKvkl6WtAfpKuTU/NYFwD0RcXiennFv2ccqG+HVGuVXk6ZCfIrVJ7Ai4m4jXQG+o5fr96cppCk/YyJNOVkEDI2IpyWNJt1DMl3SXRHx9SbG2RsvStomIv6Wr1S/1E/bKZ+S1Vn2upPqx3v5+m35Z0/ls3z9WXkE5C0k7UNqaB8BfIHUQKsqT1WcBIyLiOV5qt9QYGXuJMJby/4cpYcETASGRMRjuaPVnWrlvqFjPR+Lz5FGlf9Amha1P2k08clexlEZSy3l30Wt76EoddVtVbQBV0XEV8sXSvp4H9MFoFq5J40gfoQ0Pfq3pJHBIaRR/PWAf+aRpErrAe/PnZXyWIsItb/VewwV6VlS53ln0gg29CIvq5XZgQl3cKpxLNdV1rupNwFWRERHP8cLfW9PQdrvh4EPkS4w2FrOI1r951Xg7fn3PwG7StpI6WlVB9abmKQdgf/L05luIU2zK3kR2ErSFpI2Ik2L6pV8pWaJpNLUoE9Tu/DPJU1JgNThqeV64ExgWEQsyMuGke4JgDwNsMxkpafUbQx8HJjDmq4kTUEij0ZUKs/v7lSudwdwcp4KhaSd8/zsZhgGvJQbtvuTr9xK2hZYHhHXABeTpkVC7/e5GW4ljfiQf97SzbrN1l35LM/juaSpb++FNI8/Hy/tpGP918AXSVN2ujMMWJIbC7uQRsp6cjVpFPknUHe5LenLsT6bNFXmvvz7VNJ0mfKLIr09HgMYUcrHitgXkUaJYHVdU0/a1eKeAm821BZHxCtV1msk/buAIyRtldPfXNK7SaOF/57r4w2AIxuIu1a5n02qB++PiH+QRiUFPJb36zlJR+bPt0kqHYt3ku7HLaVd6oz1Rx1SnuZs0nS/IXlGwgeBB7pZPtjqtD+RjsOrle6Thtp5WektZXYAzQEOlTQ0102HkC4q1FtfFKbGsbyI6mV9DqtvPziIdM8yNFZvFhlvLbXqmFmk+8VKaZb2ows4Adglz1CytZw7Wv0kIl4mTTF6jHSz9AzSnPoZwCMNJHkU8FierrA7Zfca5LnfXyedqGaR7oeqx3GkOfILSPPha42WnE66J2UB6Wp2reH/G0gdsRlly74FfFPSI6x5JegB0rD/AtK0wIcq3idPL3mSGietivy+uEZc5G10SHpU6dH2V5Cmo/wxf/aHVeIbKNcCYyUtBD7D6u9xJPBA/u7PY/U9QJcDMyXdM+CRlpF0HWk6mSQ9L+lE0v1PkyU9Q7oK2Zcn//WriPgLtcvnm3mcG7afJT0YZgFpn3chNQx/lZf9HvhSD5ucCawv6UlSvsztRZjXkhoc15Ut6225LenLsT6bNA3t/lwWV1AxbbC8DEqqWQbzSMDxpCluC0mjj5flt6cBl0p6iHTVuOQ24HBJ88sai71xPmna4wJSXh9XY717SJ3t+ZKO7k3C+YLPOcCdOf1ZpHtS/5a3ez+p0djTqF8t1cr9PNKU19K9cQtIT0QsdXinACfm6VWPkx6oAukcNFbpT3o8weqZBo3ma00VdfG4HOOjpHtUzoyIvwO/rLG8sn5uuoh4ipSvv1B62EitvKxUrcz2u4h4kHShawHwG9I0uKXUX18UqdqxXKusTwMOysfPkaR7fF+lsXqzyHhrOZ/qdcx0YHiuDx8lzQIAII++HQMcoEH8Z4CsGG1dXf1164StbZT+ntBrkZ4a9ingmIg4rKfPFbjthcDoVrtHyayvJB0BHBYRn252LGbWs2aWWUntkZ6G+jZSp/xzEfHHgY6jEXlWTkdErJI0DvhBjemwZi3B92hZPcaQbgJuIz22+ISB2KikSaQnD17iTpatayR9l3RfzsE9rWtmzTcIyuzlknYl3cN0Vat0srIdgBmS1iM9IOikJsdj1ice0TIzMzMzMyuY79EyMzMzMzMrmDtaZmZmZmZmBXNHy8zMzMzMrGDuaJmZmZmZmRXMHS0zMzMzM7OCuaNlZmZmZmZWsP8HW2o4fBGLMmUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f0951493b00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(15,10))\n",
    "sns.heatmap(heatmap[:,0,:],\n",
    "           xticklabels=test_X[1].split(),yticklabels=test_X[1].split())\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}