2-Layer-MLP-AlignMNiST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-04-20T21:52:04.548207",
     "start_time": "2016-04-20T21:52:04.544771"
    }
   },
   "source": [
    "## 2 Layer MLP w/AlignMNIST\n",
    "\n",
    "75 epochs of RMSProp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T22:21:42.303339",
     "start_time": "2016-05-04T22:21:41.057014"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-06T22:06:38.786095",
     "start_time": "2016-06-06T22:06:38.662396"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%run augmentmnist.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T22:21:42.607897",
     "start_time": "2016-05-04T22:21:42.605546"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 75\n",
    "batch_size = 125\n",
    "\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T22:21:43.245224",
     "start_time": "2016-05-04T22:21:43.241142"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Network Parameters\n",
    "n_hidden_1 = 1000 # 1st layer num features\n",
    "n_hidden_2 = 500 # 2nd layer num features\n",
    "n_input = 784 # MNIST data input (img shape: 28*28)\n",
    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
    "\n",
    "std_0 = 1.0/np.sqrt(n_input)\n",
    "std_h1 = 1.0/np.sqrt(n_hidden_1)\n",
    "std_h2 = 1.0/np.sqrt(n_hidden_2)\n",
    "\n",
    "logfile = \"2-layer-mlp-alignmnist\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T22:21:43.727964",
     "start_time": "2016-05-04T22:21:43.721861"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# tf Graph input\n",
    "x = tf.placeholder(\"float\", [None, n_input])\n",
    "y = tf.placeholder(\"float\", [None, n_classes])\n",
    "\n",
    "xt = tf.placeholder(\"float\", [None, n_input])\n",
    "yt = tf.placeholder(\"float\", [None, n_classes])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T22:21:44.308399",
     "start_time": "2016-05-04T22:21:44.305404"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def multilayer_perceptron(_X, _weights, _biases):\n",
    "    layer_1 = tf.nn.relu(tf.add(tf.matmul(_X, _weights['h1']), _biases['b1'])) \n",
    "    layer_2 = tf.nn.relu(tf.add(tf.matmul(layer_1, _weights['h2']), _biases['b2'])) \n",
    "    return tf.matmul(layer_2, weights['out']) + biases['out']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T22:21:44.956518",
     "start_time": "2016-05-04T22:21:44.921159"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Store layers weight & bias\n",
    "weights = {\n",
    "    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1], stddev=std_0)),\n",
    "    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2], stddev=std_h1)),\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes], stddev=std_h2))\n",
    "}\n",
    "biases = {\n",
    "    'b1': tf.Variable(tf.random_normal([n_hidden_1], stddev=0.1)),\n",
    "    'b2': tf.Variable(tf.random_normal([n_hidden_2], stddev=0.01)),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes], stddev=0.001))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T22:21:45.346716",
     "start_time": "2016-05-04T22:21:45.336865"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "mlp = multilayer_perceptron(x, weights, biases )\n",
    "mlp_test = multilayer_perceptron(xt, weights, biases )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T22:21:45.858972",
     "start_time": "2016-05-04T22:21:45.850052"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cost =  tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(mlp, y)) \n",
    "gs = tf.get_variable(\"global_step\",[],trainable=False,initializer=tf.constant_initializer(0))\n",
    "lr = tf.constant(learning_rate) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T22:21:46.354400",
     "start_time": "2016-05-04T22:21:46.217038"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_op = tf.contrib.layers.optimize_loss(cost, global_step=gs, learning_rate=lr,optimizer=\"RMSProp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T22:21:46.701651",
     "start_time": "2016-05-04T22:21:46.681418"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'ScalarSummary_2:0' shape=() dtype=string>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#with tf.name_scope(\"training accuracy\"):\n",
    "pred = tf.equal(tf.argmax(mlp, 1), tf.argmax(y, 1)) # Count correct predictions\n",
    "train_acc_op = tf.reduce_mean(tf.cast(pred, \"float\"))  # Cast boolean to float to average\n",
    "tf.scalar_summary(\"training accuracy\", train_acc_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T22:21:47.060583",
     "start_time": "2016-05-04T22:21:47.049444"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'ScalarSummary_3:0' shape=() dtype=string>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pred = tf.equal(tf.argmax(mlp_test, 1), tf.argmax(yt, 1)) # Count correct predictions\n",
    "test_acc_op = tf.reduce_mean(tf.cast(test_pred, \"float\"))  # Cast boolean to float to average\n",
    "tf.scalar_summary(\"test 0 accuracy\", test_acc_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T22:21:51.522596",
     "start_time": "2016-05-04T22:21:51.288843"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[34m5-layer-mlp-alignmnist\u001b[m\u001b[m\r\n"
     ]
    }
   ],
   "source": [
    "!rm -rf ./logs/{logfile}\n",
    "!ls logs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T21:57:37.534602",
     "start_time": "2016-05-04T21:57:37.532739"
    },
    "collapsed": true
   },
   "source": [
    "### Original MNIST Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2016-06-07T05:07:11.546Z"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\".\", one_hot=True)\n",
    "trX_0, trY_0 = mnist.train.images, mnist.train.labels\n",
    "teX_0, teY_0 = mnist.test.images, mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-06T22:06:44.056363",
     "start_time": "2016-06-06T22:06:44.054174"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "alignmnist = AlignMNIST()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2016-05-05T05:22:36.526Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_accuracies = []\n",
    "train_accuracies = []\n",
    "with tf.Session() as sess:\n",
    "    # create a log writer. run 'tensorboard --logdir=./logs/{logfile}'\n",
    "    writer = tf.train.SummaryWriter(\"./logs/{0}\".format(logfile), sess.graph) # for 0.8\n",
    "    merged = tf.merge_all_summaries()\n",
    " \n",
    "    tf.initialize_all_variables().run()\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        trX, trY = alignmnist.next_epoch()\n",
    "      \n",
    "        for start, end in zip(range(0, len(trX), batch_size), range(batch_size, len(trX), batch_size)):\n",
    "            sess.run(train_op, feed_dict={x: trX[start:end], y: trY[start:end]})\n",
    "                 \n",
    "        summary, trn_acc, tst_acc = sess.run([merged, train_acc_op, test_acc_op], feed_dict={x: trX, y: trY, xt: teX_0, yt: teY_0})\n",
    "        writer.add_summary(summary, epoch)  \n",
    "        \n",
    "      #  print(epoch, trn_acc, tst_acc)\n",
    "        train_accuracies.append(trn_acc)\n",
    "        test_accuracies.append(tst_acc)\n",
    "\n",
    "        writer.flush()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11c1d6c90>]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD7CAYAAACG50QgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4VNXWBvB303tCqAklAZWq9CaIgIIGpQlcL0gVpHix\nfCoKiiWIKF1AQAQBUZDeVZpApEkNhBp6SQgtkEJJn/f7Y2eSmcxMMgkTciDr9zx5yJy6Zsis2bP2\nPvsokhBCCJEz5MruAIQQQjw8kvSFECIHkaQvhBA5iCR9IYTIQSTpCyFEDiJJXwghcpA82R2AmVJK\nxo4KIUQGkVQZ2d5QLX2Shv756quvsj0GiVPilDglTvNPZhgq6QshhMhakvSFECIHkaSfAS1btszu\nEJwicbqWxOlaEmf2UpmtC7maUopGiUUIIR4FSinwUe7IFUIIkbUk6QshRA4iSV8IIXIQSfpCCJGD\nSNIXQogcRJK+EELkIJL0hRAiB5GkL4QQOYgkfSGEyEEk6QshRA4iSV8IIXIQSfpCCJGDSNIXQogc\nRJK+EMIwwsKAq1ezO4rHm1NJXynlq5QKUkqdVkoNs7PeXSm1UikVqJTao5SqYbHufaXU0aSf91wZ\nvBDi8fL118CoUdkdxeMt3aSvlMoFYBqAlwHUBNBdKVUt1WafAThEsjaAPgCmJu1bE0B/AA0A1AHQ\nTilV2XXhC2F869YBUVHZHUX64hPjszsErF8PXLiQ3VE83pxp6TcCcIbkJZLxABYD6JhqmxoAtgIA\nyVMAfJRSpQBUB7CXZCzJRADbAXR2WfRCPAKGvJuA+fOzO4q0HQg9AI9xHvCe7I2eK3ti5oGZOHHz\nxEON4cwZ4OJF/fMwkQ//nNnJmaRfDkCwxeOQpGWWApGUzJVSjQBUBFAewDEAzZVSxZVShQC8AqDC\ngwYtBACsPbUW526fy+4w0nTnDhD8UlNM3rwsu0NxKCQqBJ0Wd8KvnX7F5l6b0cqnFfaE7MHz857H\n3+f/fmhxrF8PdO6sE/DDvInepp03UbP3z0hMfHjnzE55XHScMQCmKKUCABwFcAhAIskgpdRYAJsB\n3DUvd3QQPz+/5N9btmz52N6j0uhiEmKQW+VG3tx5szsUh8Luh6Hnyp54p9E7+PbFb7M7HIe2B1yF\nKnsEl4p9jENH26PuMwWyOyQrd+Puov2i9niv8Xt4rfprAIAqJaqgf73+GL19NP468xdaV27t1LFI\n4nb0bXgU9IBSGbqDHwCd9Pv1I7b6J+L69TwoWzbDh8gwkhi2+y3cb/UH1m/viHatStnd7rfA39Cu\nSjsUL1g864NKg7+/P/z9/R/oGOneI1cp1QSAH0nfpMfDAZDk2DT2uQDgGZJ3Uy0fDSCY5Ew7+8g9\ncrMBSdy8fxP7r+zHzss7sePyDhwIPYCuNbpiQecF2R2eQ0M3DcXm85tRLH8x7HhzR3aH49DAHxZi\nw+UVyF+AKM8m2PaNzTgIl9izByhSBPB+6g72hOzBzss70apSK7T0aelwn0RTIjov7YwSBUtgToc5\nNol6T8geDPpjEAIHB6Z57jO3zmDh0YVYeHQhrt29hvjEeFRwqwBvN2/ULVsXY9uMRS6VdlEhOhoo\nXRp4f8U3+Pn3MKwaNBnPPuv007cyJ2AOKrpVRJsn2qS77eyDs/HZ6hmIulgZrbzbYMM3g222uXX/\nFjwnemJux7noWaunUzFcCL+AuYfmYtQLWdsrnVX3yN0P4EmllLdSKh+AbgDWpjqxm1Iqb9LvAwD8\nY074SbV9KKUqAngNwO8ZCVBYi0uMQ6Ip499DSeJC+AX8Fvgb3v7jbbz020uoOq0qCn9bGNWnV8ek\nPZOQP09++LX0w+UPLsP/oj/2huzNgmfgWFxinFPbhUSFYN7heVj+n+UIuBqA6PjoLI4s83Zf/xt1\n3Vtj8ivjsD1xPK5G3ciS87z+7S+oPb0BPL71xGcbRyEiJgLdlnfDzXs3He7z6ZZPERETgZntZkIp\nhbFjgWCLQm4Drwa4FHEJN+7Zj/nI9SNoNLsRms9rjoiYCPze+XdEDY/C7WG3sa77OnzS7BOsClqF\n/Vf2pxu/vz9Qty6w+uxiRJVfgfPnM9cA3BOyByO2jsDgPwej85LOuBhx0eG2Z26dwWdbP4P3gYXo\n/GQv7Axfane7JceXIHeu3Nh1eZdTMZDEwD8Goki+Ipl5ClmPZLo/AHwBnAJwBsDwpGWDAAxM+r1J\n0vqTAJYDcLPYdzt0bf8QgJZpnIMibVvPb2X5SeX5w94fnN4nJj6Gb65+k+UmlmOZ8WXYdWlXTv53\nMtefWc/jN47zTuwdu/vNOzSPTec0pclkclX4adp2YRvLTSzHqJiodLcdsHYAh20eRpJsNLsR/S/4\nZ3V4mWIymVjg0wqcsTSIJFmq1/+x3czBGT7OhfAL/HLrl0xITLC7/sSl68Tw4lwV+Df9volmyZJk\nnz7koOVD2X15d7v7zAmYwyenPsmwe2EkyeBgskABskwZcuPGlO06LOrARUcX2T1G39V9OWLLCMYn\nxjuMfcSWEfx408fpPsd33yU/HH2aZSeUpduXlfjuN0fS3Se1mPgY1pheg0uPLWV0fDRH/TOKHmM9\nOHTdSAZfu2e1bVxCHBvNbsSpe36guzt5ITiaarg7DwRdtTlu49mN6bfNj7V+rOVUHHMC5rDeT/XS\nfF1cJSlvOpXHzT8Z2jgrfyTpOxabEMthm4fRc4In31rzFl9f9rrT+/588Ge2mNeCZ26dyVACTzQl\nsu7MulxybEmG470fd5+j/hnF4Mhgp/d5c/WbLDG2BEdsGZHmdqfCTrHkuJK8ff82SfKjjR9x1D+j\nMhyjK92Pu293+amwU8w9tDzPntWv+zcTb7HAF6V47Poxp4+9IHABS44rSZ/JPpwTMMfuNh1++Jje\nbw9JfhweTn72GVm2wj1WGP8E1wattdp+24VtLDWuFE/ePJm87Ndfya5dyW3bSE9PcuRIMjGRnLpn\nKvuv6W9zzpj4GBYfU5whkSFpxh8QGsDKUyqn+7f35JPkB8vGc9C6QWw59h02/OA7u9vdi7vHL7Z+\nwciYSJt1X2z9gp0Wd7I619RfLzJP9/8w/1fu7Lu6Lzef28yExAR+ufVL+i7w5aVLJpYtq7f1+bAn\nu02aZnXMoJtBLDuhLO/H3Wfh0YXtntfSlagrLDmuJA9fPZzmdq4iSf8hG7NjjMOWsqucCjvF+j/V\nZ7vf2/H63es8FXaK3t97O7VvQmICq/xQJdMt4a3nt9Jnsg+j46Od3ifwWiBrTq/JMuPL8Gv/r53a\nJzo+msXHFOfekL30GOvBSxGXHG7732X/5ejto5Mfrwlawza/tnE6voz68/Sf3HJ+i8P1iwLW0H1U\nWbuv0cTtM5inS18mJurH166RBVpOZptffNM9b0R0BN9Y8QarT6vOgNAA7g3ZS88JnjZ/bzfu3mD+\nLz34xXjbD9glS0i32ttY6rvyjIiOIEmeDjvN0uNL8+9zf1tt27cvOX26/j00lGzenPT1JfdfPE7v\n771tkva6U+v43Nzn0n0eJpOJlSZX4qGrhxxuc/o06eVFNpvTjOvPrOeoRX/R/YPmdredf3g+i48p\nzhrTa/DsrbPJywOvBbLUuFK8EnWFJHn3LtmvH1mlCvn992S9Flc4cfdE1vupHj0neLLM+DIMjQrl\nH3+QL72kj/HRT2tZ4qPnrc43YssIfrjhQ5Lk8/Oe58azG+mIyWRip8WdOGLLCAYHk7/8ku7L88Ak\n6T9E0fHRzD0yt8Ovvq5SZ2YdTtg1IflNZzKZWHxMcV69Y/s1NLXlx5ezyc9NHqhE03FRR47dOTbd\n7RJNiZy0exJLjivJ+Yfn0/+CP2v/WNupc6w4sYIvzH+BpG6t9VjRw+52AaEBLDuhLO/G3k1eduv+\nLRb9tmimvkrfi7vHT//+lLfu37K7PjImkmXGl3FYdopNiGUJv6eI//Pmz/vn26xvOaMzfTr8ZrWs\nbbtYlhn9FDef2+wwruDIYPpM9uH//vgf78WllCV6rOjBL7Z+YbXtsM3D6PbG2zzkIKf+/TdZoOsg\nvjR1EG/dv8UqP1ThTwd+stnO25s8cSLlcVwc+fzz5JIlJnpO8LRKsCTZa2UvfrhkKj/9lIyJcfhU\nSJJDNw7l51s+d7h+yhSy+1vX6PadG2PiY3j42H2qz4omf5uz1GJeCy4/vpzT901nmfFluO3CNsYn\nxrPBrAb8+eDPJMnAQLJaNV3iunOHvHePLFSIjE76XD558ySPXj9Kkvz2W3LoUL089EYMMbw4z93Q\nHxyJpkRW/L5icqt9+Obh/GrbVw6fx7Ljy1htWjX+uTGaZcuSY8ak/bq4giT9hyggNIDwA7su7epw\nm01nN3H7xe2ZPse9uHss+E1BxsRbv6t8F/hy9cnVae5rMplY/6f66W6XnlNhp1hibAlev3s9eVlw\nMFm5MunuThYuTOYtdot533yZ9Wc0SU4OCYkJLD2+tE2ysKfLki7Jb9g7sXfoNdGLe0P2Wm0THh3O\n5nOb2+3PeHrG09wXsi9DzysiOoLPzX2OFSZVYN/Vfe1uM3zzcPZd3Zd9V/flRxs/slk/fvtk5uvn\nS48m61htYkOrdQmJCSz8tQe7vhlqtXzJErJqrx/Ye1Vvh7GN3TmWA9YOsFl+KeISPcZ68HLEZZLk\nzXs36f6dB929LyV/m7DHf08Ecw0tzwrfPsMPNnxgs/78ebJsWTJ12+CLL/RPz5U9rT4oouOj6T7G\nne8MD2Xp0mT9+uRZO//N9+6Ru3aR/wb/y+rTqjuMz9eXHDxrNv+77L/6+NGk6vEqfw9cbLXdmVtn\nWGpcKcYmxJIkN5/bzNLjS7Pd7+34wvwXaDKZGBFBlitn28pu0IDcscP23N266dKWWemBfTh43hSS\nugz2zIxnktetDVrr8Fvlrfu36DnBkwO+3kVPT3LrVodP16Uyk/RlwrVMOnrjKNpUboNN5zbhfvx9\nm/UkMXTzUIzeMTrT5wi8FogapWogf578Vssbl2uMPSF70tx3y4UtuB9/H+2rts/0+QE9ZrtnrZ4Y\n9McghEeHAwCWLgWaNQPOnQP8j51ExVGN8UTRmujLHXjC4wkAQO5cudGpaiesOLkizeNHxkRi8/nN\n6FxdX6hdJF8RjGo1Ch9u/NDcGMCOSztQZ2Yd1CpTC4PqD7I5xvMVn8f2S9udfk637t/Ci7++iFql\na+HY/45h24Vt2HRuk9U2FyMuYlbALHzT6huMbT0W8wPn49iNY8nrb0ffxshto9H0zgT0adYW1++E\nYd+VfcnrD107hPzxZdGwmqfVcdu3B675d8TaoD+QYEqwG9/qoNXoWqOrzfKKbhUxuP5gjNg6AgAw\ncfdE1M33H7RuWBG50ngnt2jshmlt5iDsSEN812q8zfpt24CWLYHUQ+tr1QKOHAFaV2ptdZHWxrMb\nUbtMbZwL9MSsWUCfPsCzzwLLl+v1Z88CH30EVKwIvPgiUOBWI0TFRuHkzZM2546OBnbuBM7nW41O\n1ToBAAoUAIpdexXLj/xpte0vh39Bz1o9kS93PgBA68qtsfPNnUgwJWB2+9lQSuGLL4C2bXVMlpo2\nBXbvtn1tjhzRz9OsbcXXseasHsXza+Cv6F27d/K6Zys8i71X9iLRlIjr1/XrZv55d9FE5DnfASc2\nNsX+/UCrVrbnMoyMfkpk1Q8esZb+0I1DOXr7aL4w/wWuOLHCZv3B0IOsMKkCi35r/2uqM6bumcqB\nawfaLP/r9F9s9UurNPd9cf6L/OXQL2lu46yomCgOXjeYZcaX4c8Hf2bjJolcv55cf2Y9S40rxbkB\nc7l8OdkmVSNo09lNbDS7UZrHnndoHjst7mS1LCExgbV/rM3fj/zOEVtG0HOCJ/849YfNvuaW6eKj\ni9lhUQennktoVChrTq/JTzZ9klz22nBmA30m+1jVy7st70a/bX7Jj6ftncbn5z2fvM+7f77Pot0H\nc9cucu1a8sne49lrZa/k7cfsGMOKA9/jH7Zhs39/svzX9bn1vG1zMDQqlO5j3JNbsyQZG0v+9pt+\nvlExUSw7oSw3nNlAj7Ee7NzvImfMcOqps2FDcv162+U9e5KzZtkuP3WKrFRJl5tKjC3BRJP+OvHG\nijc4fd90enmRFy/qbfft09vWq0eWKkV+8gl57hw5ahQ5YAD57l/v2u1w/+svsmmrKBb9tmhyvwNJ\n1nvhIt1Gl0wesZSQmMByE8sll2Xs2b9fjz4KC7Ndt2QJ2SHVn0h0tB6xZFme2h8Qy1yfevDkjSC6\nj3FP7iMwq/pDVe4+d5hPPkk2a0a2bEk+3yqe+T714oARxxgX5zC8LAEp7zw8L//2MtcGreX0fdP5\nxoo3bNYP+XMI/bb5seOijplOvn1W9eGsA9bvxuPHydPBYSz6bVGHQ/j2hexjhUkVrBKHKxwMPcj6\nM55l7sENOXyTTsY7L+0kqTvOihYlb1t8vsUlxLHE2BLJ5Qh72vzahkuPLbVZ/ve5vwk/sO2Ctrx2\n55rdffv0IVet0iMmPMZ6JCcls+M3jrPLki7stLhT8k/F7yvym3++senn6LOqD9/76z2S5O7Lu1lu\nYjmrvoOExATWnVmXCwIX8FTYKRb5ugQbv6BLXuHhZOGSt+g+xj25DNb619b0eHYNL1ywjXvVKvLJ\nfqOSz2fppwM/2Qyz/PBD/U7dsEE/nnVgFvONyse31g5g+fI6OTtj2jSye6oRnCaTLoecOWO7fUKC\nroVHRupkFxAawPtx9+n2nRuPXbxGd3frklB4uP4AjLbo075+XZcB1wT6s87MOjbnGDSIfOObZXz5\nt5etlvfoQZYbXZN7gveQ1A2MhrMa2uxvGWv9+o47T4ODyZIlreMNCCCfftr29SjUvR9rTW1sExOp\nR5k1+78Z7NMnZdmfp/9k49mNHcaWlSTpP0ReE714IfxCcsvMsu4eHR/NEmNL8GL4Rf56+Fe2/719\nps5Rc3pNHgw9SFJ3rPn5kXnykMOGkU9NfYpHrtkfy9xlSRdO/ndyps6ZnomTEvnckPnsvKQzL4Zf\ntFrXsaN1fZTUY7kdxXL1zlW6j3Hn/bj7vH2bnDnTev2BKwccdkKHhJC5c+vx3ST5xJQnrFqB8Ynx\nrP9TfX6+5XOuPLEy+cdRH4u5Jrvz0k42+bmJ3Q/qf4P/pecET77060ss23UM//wzZV2dOmS7Wf05\nevtoRsdHs8joIixUPMJurT0ykizoc5QVJlW0eX5tF7S1Gia7di1ZsSI5Y4ZuqZtM+gOo96re3Hbo\nIsuVs63FOxIWRrq5kREpDWqePk2WL+/4GA0b6rr8kD+HcPyu8cmd7ps3645eZ/TpQ343xraPJyhI\nJ+LOC3rwx/0/Wu0zYgTZ1O/j5I7rrku7cub+VH8gFn74gWzRIu3XomJF/XzNfvnF9kOQJNu+u4Hw\nAxceWWizbsicn1m4Vw9GWozc7LKki93O8YdBkr4Lnb11ltP3Tbe7zjxixPyGfW7uc1blh0VHFyV3\n+IRHh7Pot0UdXnTkqPRzN/YuC35TkLEJsQwK0m++l14ily4la9XSoydmH5xts9+psFMsNa6UVSvV\nlZo2pVWyszR/PtnJulLDdafWsflc+8PvpuyZktyhuXQpWawY0+yQtDRyJFmjBvlc0qjBN1e/afX/\nNW7nuOTOPUcSE8nx48n7ScPslx9fTvcx7qw7s67Ntwaz/mv6s/S33qxZO9oqwbz/PvnONwEsP6k8\nN53dxJrfN2GDBo7jf76FiZ7fPcGA0IDkZZExkSz6bdHkseCXLpGlS+ukm5hIPvOM/hAw+/FHsrfj\n/mC7XnuNnG3xZzNzJtmrl+Pt+/XT26w8sZK+C3z5+rLXOXP/TE6YQL5n+0XFrgMHdMIdsGZQ8kgw\nk0l34I6bEMfiY4rblFF+/pn0HeTP+j/V5817N+n2nRsjoiMYF0eOG6cvILub9Cd+5QpZooT+FpyW\nbt3IefNSHn/4IfmdncsBVqyOY9nBfaxGTpHk1atkiaon6Dm2UvKyG3dvJMeWHTKT9KUj14Fxu8bh\ni21fwESTzbqj14/i6dJPJ89V0qV6F6sOy7mH5qJf3X4AAPcC7mhWsRn+PPOnzXGWHFuCqtOq2p1W\n4fC1w6hZuiaW/J4PzZoBffsCGzboWQhDQoCqRex35k7fNx1v1XsLhfMVzuxTx+zZwKxZtstDQoCT\nJ4HWDubfatcO2LoVuHcvZVmbym1w9MZRXLt7zWb734/+jjeefgMAsH+/nnP+7Nn040tI0DFOmwYE\nBgImE/C8d0pn7plbZzB219jkzj1HJkwAPv4Y2LJFP+5Sowv61+2P6a9MdzhXzNS2U+Hj/w8++6SA\nVcdnixbAmR114e3mjaGbh8InsTVq1nT8HNr6KpSN6ITVQauTl204uwHPVXwOxfIXQ3w80L078OGH\nuhMyVy5g5Ejgyy/18wX0a/3CC+m/Xpb69AF+/TXl8bZtaXc6mjtzW/q0xK7Lu7Dh7AZ0rt4Zhw8D\ntWs7d8769YHy5QHP8JT3yR9/6Nk0n273D6qUqAKvol5W+/j4APdONcW58HOYsHsC2ldtD7cCbti9\nG5g6FfjmG6BMGaB5c/2eGDgQqFHD9tyWUnfmpu7ENXu5dV7EL/sF7w4uhD179IyfJNCvHzD4P1UR\nwwhcvaNv77Xw6EJ0qNoBbgXcnHsxjCCjnxJZ9QMDtfRv379N9zHu9JzgabeE8sPeH6w6WC9FXGKJ\nsSUYlxDHi+EXWWJsCauLdWYfnG0ztDM6Ppre33uzxNgS3HHJdizZlD1TOGjdIFatSm5PVZHo1o0c\nMX0/n55hXZCMioli8THF06yh37mj9w8IsL9+717dEVeihG2teMqU9FuWrVuTK1L1a3df3t3m6/vZ\nW2dZenzp5PH1LVrocy60/UZtY80askkT/bu3t/7KfvbWWXpN9GKiKZHPz3ue3//7fZrH2LdPt6L/\n9z9yyJA0N7Wyaxfp40PGp7os4OZN3aex4PAiwg/s8tE2jhvn+DiHDpEVmu2wurS/+/LuyWWCYcN0\nS9jym4/JpDtKly/Xy0uWJC87/q+2KzZW//+eO6ePV6YM7fY7mG3dmvJtquGshsnfYJ95RrfgnbV4\nMdm8pe7jaTP/ZRYc4MsGk31Z5Ycq/G6HbXP73Dn97eA/S//DXCNzJXd6f/mlfm1I3dLfuJGcNEkP\nD03PwYP626FZ6dK61m/P1at6nP0TT+i6f+/eethnXBz56sJXufz4cppMJj4942luu7DN+RfCxSDl\nHdeYtHsSe6zowbfWvMWpe6barB+4dqDNePGGsxpy87nN9NvmxyF/WmeRG3dvsNh3xay+Lo7ZMYad\nFnfi51s+5yebPrE5R+9VvTnrwGwWLKgTtaX588lOXWJZeHRhq7LRjH0z2HlJZ4fPKz6efOUVnWC9\nvGzf7BERegTG8uXk5Ml6O8uk89xz5Lp1Dg9PUl/V2bOn9bLlx5ez9a+tSerrBxYdXcTyk8onX1mb\nkKAT5scfkx/YDiO38corKV/TX3tNJxSTyUSviV4cunEoG89u7LCTmySjovRl/8uWkYcP6ze2s9q1\nS7lyNbWnnyZ3/hvLgWsH8oU2MQ7LYGRSwvVMoMd3pXju9jnGJsTSfYw7Q6NCuXu37ly9ccN2vz/+\nIGvW1Ansqaecj9vSu+/q/qHjx/UHWFpu3tRlN5OJ/PXwr9x4diNjYsiCBa07bNMTF6ef05qdQez9\n9V98ttdf/Ov0X1x/Zr3dAQdxcWS+fOTPB35hpcmVksttzz1nPTdQRsTHk0WK6MEG166RxYun3x+S\nmEhu2UK+9VZKf8Do7aP54YYPuS9kHytPqeywFPgwSNJ3gURTIp+Y8gT/Df6XvwX+xi5Luths8+zP\nz9pMbTBmxxgOXDuQ3t97W9VpzVr90oorT6wkSV6/e50lxpbg6bDT3Buy1+6FKzWm1+DfxwLo4WEb\n4/XrukPu2dlNk6cIMJlMrDG9ht1hgHo9OXgw+fLL+g01ZQpZtWrK8DaTiXz9db0NqRNxo0YpQ/mu\nXNFvkvSuvjRvF2vxPr4Xd4/FvitG/wv+bDGvBWv/WNuqQ/XYMZ2EN2/Wl/+n5cIF0sMjpWX39dcp\nLb9uy7sx79d5053bpndv/SY2P++yZe2PXkntyBG97X37U+1wyBAmt+4thzM60qcP2XRMf07aPYkb\nz25kk5/11xdfX/tDKM3xNm6s+1YGZ3zuNpJ6aGPlyrrzs1+/9LdP/VwCAvQHT0aNGkW++qr+Rnf+\nfPrbV6xInj6TkFzvv3NHXwzoTKvekVat9DDRzZt1oyYztl3YxiY/N+HgdYOzfd6nzCR9qemnsuHs\nBrgXcEfjco3RwrsF/rn0j/lDCYD+kDx24xieKfOM1X5danTBvMPzULxgcdT1rGtz3K41uibXM7/a\n9hV61eqFp0o8hQZeDRAeE46zt1OK2Xfj7uJixEUUvFsT3t62MZYuDVSpApRD4+Tpj/0v+gOAw/nT\nx40D/v1XX1iVNy/w3ntAhw76JzoamDMHOHECmDRJb587N/Dzz8BnnwGhocCKFbpmnz+/3cMn8/IC\nqlbVU+WaFcpbCC898RI6LemE12u+jgMDD6C5d/Pk9fv2AQ0b6trvoUNI8w5Gs2cDvXoBhQrpx3Xr\n6n0AoG/tvpj+ynTULO24mP777/p8kyfrx0oBvr66vyQ9Y8YA//d/QMGC9te3aAH88w8QEaH7JypW\nTPt4vr5AwrFOWH1qNVYHrUanqp1w4ABw7BjQu7f9fZTSNw7fvTvj9Xyz+vX1BVATJzp3EZG5rm8W\nGOh8Pd/SwIHA5s3AkCFApUrpb1+pEhB8OXdyvX/nTh27+f8+M5o1A3btclzPd0ajco1w5PoRLDm+\nBH1q90l/B6PJ6KdEVv3AIC39tgvact6hecmPK0+pbNVyvBB+gV4TvezuW2dmHbvlIDLlopuDoQdZ\nalwpq/le+q/RrT2zHZd2sOGshly2zHY0jNmXX5Lthy9Jviip85LOnLHP/lU6ixaRFSroYY6WEhP1\nkLUXX9T1YXujHz7/nOzcWQ/PW7PGfiypjR2rx19bunnvpsM5bt5+W0+KRepSi6NRGHFxuqVtOUdM\nSIiuUTvWTjb5AAAgAElEQVQzbDEwUG+bep6aJUt0CzQt587pFmpkGpMsXr2qv4Ft366/JaUnLIws\n6nGfxb4rxpLjSjLoZhBfe01/C0uLyUSOHm099DKjxozR3/NT/03Y88kn5DffpDz+v//T/8eZsX27\n429KqfXpo0fxmA0dqkdtPYi//tKt/T59rEcxZVSDWQ3ouyD9ifOyGqS847yrd67aDLMyz+1hOVVu\nv9X9OG1vynSra4PW2r1og9S1+7Rqyc3mNGO5ieVsxq2vPrna6grbyf9O5tt/vM0JE/RQQHv27CGf\naniRZcaX4eWIy/QY62EzA+Pp07qMUaqULk3YExOjP1jm284XRlLXbatW1XVdZ2u4p0/r5Jzg+KWw\nUr8+uVNf48X//tdxLMuW2Y4NN5nS7pAzO3NGlymW2Jkp+tYt3aeQ1vMbPFhPV5yeqlX1h9ibb6a/\nLalLNc9P68Jq06rx6FHdsfog5QtnhYbqK2WdsWCBLv2ZtWyZ+bp6Rvj56fH6ZvXq2Z8/JyPCw3Vd\n/+mn9aCFzJp/eL7dARgPW2aSfo4t73RZ2gVVplXBvEPzkodlztg/A/3q9kPBvCnf31v46BKP2dEb\nR/FM6WdsjgcApQqXQu5cuR2es2uNriiUtxDebvi21fLWlVvjQOgBRMREAAAOXD2A+p71cekS7JZ3\nAKBBA+D2hYowmYDPtn6Gns/0TL5Tz+HDQLdueoial5cu2zxjP2Tkzw+sWuW4nFCggB7iN3Kk/t0Z\nTz2lS1Dr16e/bUyMjq9u3ZTndeCA/W1/+gkYlGrqHaWsSzz2hIQAbdoAfn7A66/brvfwAJ5+WpcP\n7Ll2DViyBHj//XSfDlq0AH77DWkO17Tk6wuUD/k/fN78c3z3HfDBBw9WvnCWp6f9Ybn2WJZ3yMyX\ndzLKx0cP6wSA27eBM2eARo0e7Jju7vo9dfy48/9H9vSu3RvPVXzuwYLJLhn9lMiqHzzElv7JmydZ\nZnwZ7r68m41nN2bj2Y35z8V/6DHWgxfCL1htezH8IkuPL518kU+35d04/7CDpmg64hPjk+9UlNqr\nC19Nnqa5+rTqPHz1MDt0sB3+aKlHD7L2dx0JPzDopr4704QJupU9YYIepZJdtmzRLWt7I1As7dmj\nr2Y127aNfPZZ2+2Cg3UHsb3W+PDhjr/237ihp9kdPz7tOEaOJD+ynUiTpC5vvPNO2vubLVyovz/b\nm+PGnn//1RfbnTmjS2xplY+yS2ysnqPm/n09RLRMmYdz3n/+0R3WpH4f+LqomjJgQOZHPhkNpKXv\nnHmH5qF37d54tsKz2N1/NwbVH4SuS7uiecXm8HH3sdrW290bhfIWQlBYEAB9YZajln5qCQn6xyxP\nrjwoUaiE3W3bV2mPdafX4U7sHVyKvIQapWqk2dIHgFdeAeLPNcNLT7yEqiWrYtIkYOZMfaHTRx8B\nRYs6FWaWeOEF/e3hzTd169CR/ft1J65ZvXq6JZmQagLKRYuALl3sf9uoWxcICLBdfueObkl36QIM\nHZp2vI46cyMidId2evubtWih/03vQiGzhg31N5H33wf+9z+gWDHn9nuY8uXTAwdOnNDfIuvUeTjn\nrVQppaW/ZUvmO65Te/HFB//G8EjL6KdEVv3gIbX04xLiWHZCWatbxZH6wqbw6HC7+/Rd3Zcz9s1g\nbEIsC3xTwKk7SZ0/r+uGI9K++1+ykMgQeoz14NbzW5MnbypeXI+TduTmTbKIWxxvRkZx0iTdCZrR\ni3WyUlyc7tBMq2OyVy/bDrVq1XSnq6VatUh/f/vHOHNGD+9LbdQoXYt2ppM3IUF31Fq+fiaTbuFn\ndKqDlSudnw+H1P0YRYrYnx3SKHr2JOfO1a/pJ7aXlWSJhAQ9Vj86Wv9NHDzomuOaTBn7/zEySEs/\nfRvObkDl4pVRrWQ1q+VF8xeFewF3u/uYh24GhQXBx90HBfKkXdzevl3X0597Dli50rm4yhUrBx93\nH0zdNxX1Pevjzh0gNhYoYf+LAQCgZEng6ep58c7Aopg2TV+WX6GCc+d7GPLm1UMkR43SrXd7Urf0\nAdu6/pEjQHi4vuTensqVdYv81q2UZdHRepqGr76ynSfenty5gZdeAjZu1I9JXV/ftUsPbcyI115z\n7pxmAwcCY8em/X+d3cx1/YdVzwf0/0mFCsCePcCNG647r1IZ+/953OS4pD/n0Bz0q9MvQ/u09GkJ\n/4v+TpV25swB/vMfYP58YPp0naycmU8G0CWe1UGr0cCrQXJpJ70/znbt9LjzbdvSHxeeHZ54Avj+\ne92xfD/VvWYiI4HgYNsOtdRJf+FCoEcPOLxRSK5cOiEcPpyybP58/RXe2TILoEs8GzfquW3eflsn\nm61b9YdrVnrhBV3aMTJz0n+Y5R1Ad+bOnatv8pLb8RgJkQE5Kulfu3sN/hf98XpNO0M40mBu3S8/\nudxh0g8NTWmxbd+uW425cumkvG6dc+dpX0Xf5aq+lx6540wS//hj3foyYsI369lTt+bfecd6+cGD\nuh6fJ4/1csukbzLppN+zZ9rnsKzrJybqydQ++SRjcb70EvD333pirePHgU2b9GgPoUd/HTyo/86r\nVHl4561UCVi2zHX1fJHDkv6CIwvwWvXXUDR/xns4W/i0wNpTa22uxA0J0cns6aeBIkWAvXv1Falm\n7ds7Tvq3b+sShFk9z3r4pOknTnXimuXLl70dts6aMUO3nOfOTVlmvhI3tTp19FWpcXH6CteSJdMf\nXlevXsqwzVWr9JDRZs0yFmPZsrpUFBysO3WN2KmaXTw99Ydz9eq2H9JZqVIlPaz3xRcf3jkfdzkm\n6ZPUUx5nsLRj1tK7JUw0WbX0P/9clxUKFdJTDk+aBBQvbr1f69a61RoRYXvM6dP1KJu4OP1YKYWx\nbcYiT648Tif9R0WRInoqh2HDUsow+/bZH0VRuLAuCx07BixYkH4rH0gZq0/qKSc+/jhzddt164C/\n/tIxiBRK6RLPw6rnm/n46A8cy4aUeDA5JunvvbIXCaaETF9Q0apSKxQvUByViutJQ6KjdZI/cUIn\nmTJl7O9XqBDw/PO2wwFjYnTrt2hRXUpI7XFL+oBuJU6dCnTtqj8E7XXimjVooC+WWrVKzyvvzLEv\nXQL+/FP3FXTokLkYvbzSn18op/L11Re5PUytWun3WU7ueHW1h/hFLXvNCZiDfnX7pXlTjbT4uPvg\n/Pvnk2+uERgIVKvmONlb6tBBtyC7dUtZtmiRLmOUKpVS27b0OCZ9QCfwXbv0CJf793U5xZ4GDYBv\nv9Vlm3Ll0j9u3ry60/Z//wO++EI6/bJCRvtIXMHT0/p9Ix5cjmjpxyfGY2XQSvR4pscDHcdySOfB\ng3rGP2e0a6db+vHx+jGpR7R88IHjaQce16QP6CGQ9+7pVr6jz+AGDXSnoTOlHbN69XSprFcv18Qp\nxOMoR7T0dwXvgo+7Dyq4uW4Q+8GDzl/V5+WlO6R27dJDz7Zs0aNS2rTRtePffrPePjZWjzn38rJ7\nuEde/vz6dnnh4Y63qVVLt9w7d3b+uD166D4UZ+cIEiInyhEt/XWn1iUPh3SVjLT0AetRPJMm6Xuf\nKqVLPMePp3TmAnr0iJfX412iKF067c65ggX165KRETQtWtifUE0IkSJnJP3Trk360dF6xj9HM1fa\nY076J0/q8eRv6PuBo3BhXdc+dixl28e5tCOEyF6PfdI/FXYK9+LvoZ5nPZcd88gR3UrNSBmhbl3d\ncTlkiL7a03LfBg30NwczSfpCiKzy2Cf9dafXod1T7TI9aseejJZ2AF3Kad9e3+bubevp9FG/vnVn\nriR9IURWyRFJv33V7K3nmw0YAIwerevZlurXl5a+EOLheKyT/u3o2zh09RBerOTaa7gPHtQlmYyq\nV09fgZtanTr6Iq/YWP3Y2Xl3hBAiox7JpH/j3g0cCHVwTz0L68+sR6tKraxuf/igYmKA06cz1omb\nnkKFUqYdAIDLl6WlL4TIGo9k0v/+3+/RdE5TrDixIs3t1p5e69SonfBw3cH66adp3+UJyFwnrjPM\nnbkmk57ETVr6Qois8Egm/dWnVmP6K9Pxzvp3MP/wfLvbxCXGYdO5TXj1qVcdHofU865Xr65/X7dO\nzw2TlszW89Nj7sy9elVP2iYXGAkhssIjd0VuUFgQ7sTeQf96/dHcuzna/NYG9+Lv4X8Nre9CsePS\nDjzl8RQ8i3raPc7p07pj9d49newbNtT342zaVF8962jCrqxM+vPmSSeuECJrPXIt/TVBa9Cxakfk\nUrlQrWQ1bO+7HRP/nYhhm4fh8LXDSDQlAkj/gqy33tJzdO/dmzLTo48PsHo10L+/9WgaSwcOZE3S\nr11bX7h1+rQkfSFE1nnkkv7qU6vRqVqn5MeVilfCjjd3IDwmHN2Wd0OJcSXQdmFbLD622OFQzZAQ\nfYn/sGG2Ux00agTMmgV07KinQ7Bk7sStVcvVz0p35j75pJ4aWJK+ECKrPFLlnat3ruJU2Cm08Glh\ntdyrqBdmtZ8FQI/s2XV5F86Hn0ftMvbv+LB0KdCpk+N50197DTh/HmjbVt+5yXzD6iNH9K3isqre\n3qABsGSJvuWiEEJkBada+kopX6VUkFLqtFJqmJ317kqplUqpQKXUHqVUDYt1HyiljimljiilFiql\n8mU22LWn1qLtU22RL7fjQ5QuXBqvVX8NHzX9yOFVuIsXpz9H94cf6imRX35Z35QDyLp6vln9+nqq\nBmnpCyGySrpJXymVC8A0AC8DqAmgu1KqWqrNPgNwiGRtAH0ATE3a1wvAuwDqkawF/c0i07dEWH1q\nNTpV7ZT+hmk4d053lrZqlfZ2SgHffQc8+yzw6qu6wzerk775gi9J+kKIrOJMS78RgDMkL5GMB7AY\nQMdU29QAsBUASJ4C4KOUKpW0LjeAwkqpPAAKAQjNTKBRsVHYdXkXfJ/0zczuyRYv1rfrc+bmzkoB\nU6bokk7HjvrG3lmZ9GvV0rNuStIXQmQVZ5J+OQCWXZohScssBQLoDABKqUYAKgIoTzIUwEQAlwFc\nARBB8u/MBLr+zHo0926OovmLZmb3ZIsXO3fPVbNcuYDZs4GSJYGgoKzpxDUrWFBfjevmlnXnEELk\nbK7qyB0DYIpSKgDAUQCHACQqpdyhvxV4A4gEsFwp9QbJ3+0dxM/PL/n3li1bomXLlsmPXVHaOXZM\n35C7adOM7Zc7t7671aBBOjFnJQ+PrD2+EOLR5e/vD39//wc6hmI68w4opZoA8CPpm/R4OACSdDjG\nRCl1HkAtAL4AXiY5IGl5LwCNSb5jZx86iiU2IRZlJ5bFySEnUbZIWeeemR1ffKFvgDJhQqYPIYQQ\nhqGUAskMzRvvTHlnP4AnlVLeSSNvugFYm+rEbkqpvEm/DwCwneRd6LJOE6VUAaWH0rwI4GRGAgQA\n/4v+qFGqxgMlfNK5UTtCCPE4S7e8QzJRKfUOgE3QHxJzSJ5USg3SqzkLQHUA85VSJgDHAfRP2nef\nUmo5dLknPunfWRkNctO5TXjlyVcyupuVgwd14s/KjlghhDA6p2r6JDcAqJpq2U8Wv+9Jvd5i3UgA\nIx8gRlyIuIAm5Zs8yCGSW/kuvIGWEEI8ch6JaRiCo4JRwa3CAx1j/Xp9Fa4QQuRkj0bSjwxGRbfM\nTzBPAhcu6HnwhRAiJzN80o9NiEV4TDjKFC6T6WPcvKknNCv6YEP8hRDikWf4pH/lzhV4FvFE7ly5\n09/YgYsX5SpXIYQAHoGkfzny8gPX8y9e1HPlCyFETmf4pB8cGYwKxSTpCyGEKxg/6Uc9WCcuIElf\nCCHMjJ/0XdDSv3RJkr4QQgCPQtJ3wRh9aekLIYRm+KR/OfLyA7X0SRm9I4QQZoZP+g/a0g8L0/fC\nLVbMhUEJIcQjytBJ/27cXcQmxKJEwRKZPoaUdoQQIoWhk35wpG7lO7rBuTMk6QshRApjJ/0oGbkj\nhBCuZOikL1fjCiGEaxk66cvVuEII4VrGTvouuhpXhmsKIYRm+KQvY/SFEMJ1jJ30Ix9sjP6tW0De\nvIC7uwuDEkKIR5hhkz7JB74aV+r5QghhzbBJ/3b0beTLnQ9F82f+dlcyXFMIIawZNunLlMpCCOF6\nxk36D1jPByTpCyFEasZN+i64GldG7gghhDXjJv00Lsz691/Azy/9Y0hLXwghrBk26V+OcjwFw44d\nwMiRwD//ON7fPEZfkr4QQqQwbNIPjnTckRsaCrRqBQwaBMTE2N8/PBzInVvG6AshhCXjJv00avpX\nruiEX7Mm8O239veXVr4QQtgyZNI30YTQO6EoX6y83fVXrgBeXsDUqcCPPwLHj9tuI0lfCCFsGTLp\nX797He4F3JE/T36760NDgXLl9M/IkcDAgYDJZL2NjNwRQghbhkz6aU2/YDIBV68Cnp768eDButP2\np5+st5OWvhBC2DJk0k/ratxbt4AiRYCCBfXjXLmA2bP1EM5hw4DoaL1ckr4QQtgyZtJPY4y+uZ5v\nqWZN4OhRnejr1NFDOmXeHSGEsJUnuwOwJzjK8RQM5np+aqVLA0uWAKtXA926AdeuSdIXQojUjNnS\nT2e4ZuqWvqVOnYBjx3TJR8boCyGENUO29K/dvQbPop521zlq6VsqXhzo1y8LAhNCiEecIVv6kTGR\ncMvvZnddei19IYQQjhky6UfFRqFY/mJ21znT0hdCCGGfIZN+ZGwk3ApIS18IIVzNcEmfJO7E3kHR\nfPZvkygtfSGEyDynkr5SylcpFaSUOq2UGmZnvbtSaqVSKlAptUcpVSNpeRWl1CGlVEDSv5FKqffS\nOte9+HvInyc/8ubOa7MuLk7Pnlm6tJPPTgghhJV0R+8opXIBmAbgRQChAPYrpdaQDLLY7DMAh0h2\nVkpVBTAdQGuSpwHUtThOCIBVaZ0vrXr+tWs64efOnf4TE0IIYcuZln4jAGdIXiIZD2AxgI6ptqkB\nYCsAkDwFwEcpVSrVNq0BnCMZnNbJ0hu5I6UdIYTIPGeSfjkAlok6JGmZpUAAnQFAKdUIQEUAqedF\n/i+ARemdLL2RO9KJK4QQmeeqjtwxAIorpQIADAFwCECieaVSKi+ADgCWpXeg9EbuSEtfCCEyz5kr\ncq9At9zNyictS0byDoDka2CVUhcAnLfYpC2AgyRvpnUiPz8/nLh5AiE3QuBfwR8tW7a0Wi8tfSFE\nTubv7w9/f/8HOoYimfYGSuUGcAq6I/cqgH0AupM8abGNG4D7JOOVUgMANCPZ12L9IgAbSM5P4zwk\niTkBc7AreBfmdpxrs02vXkDr1kCfPhl6jkII8VhSSoGkysg+6bb0SSYqpd4BsAm6HDSH5Eml1CC9\nmrMAVAcwXyllAnAcQH+LoApBd+IOdCYgqekLIUTWcWrCNZIbAFRNtewni9/3pF5vse4+gNQjeRyK\njJXRO0IIkVUMd0WutPSFECLrGC7pR8bYH71z5w6QmAi42f8SIIQQwgmGS/pRcfZb+uZWvspQl4UQ\nQghLhkv6jq7IlXq+EEI8OMMlfUc1fannCyHEgzNc0nd0Ra609IUQ4sEZLulLS18IIbKO4ZK+1PSF\nECLrGCrpJ5oSEZ0QjcL5Ctusk5a+EEI8OEMl/ajYKBTNVxS5lG1Y0tIXQogHZ7ikb6+ebzLpu2ZJ\nS18IIR6M4ZK+vZE7YWFA0aJA/vzZEJQQQjxGDJX0I2Mj7bb0pbQjhBCuYaikHxUbZXfkjnTiCiGE\naxgq6UfGSEtfCCGykqGSvrT0hRAiaxkq6Tuq6V+/DpQtmw0BCSHEY8ZQST+t0TslS2ZDQEII8Zgx\nVNJ3VNOXpC+EEK5hqKQfFWe/pi9JXwghXMNQSV9a+kIIkbUMlfTt1fRJ4NYtoESJbApKCCEeI4ZK\n+vZG79y5o6dfkCkYhBDiwRkq6dsbpy+lHSGEcB1DJX17NX1J+kII4TqGSvr2avpSzxdCCNcxVNI3\n0YT8ua2L99LSF0II1zFU0i+WvxiUUlbLJOkLIYTrGCrpyxQMQgiRtQyV9OXCLCGEyFqGSvr2pmCQ\njlwhhHAdQyV9aekLIUTWMlTSl5q+EEJkLUMl/WL5pKUvhBBZyVBJ395ka7dvS01fCCFcxVBJP3VN\nPyoKKFAAyJcvmwISQojHjKGSvky2JoQQWctQSb+AksnWhBAiKxkq6SdGS0tfCCGykqGSflyUdUtf\nLswSQgjXMlTSvx8uLX0hhMhKTiV9pZSvUipIKXVaKTXMznp3pdRKpVSgUmqPUqqGxTo3pdQypdRJ\npdRxpVRjR+e5GyY1fSGEyErpJn2lVC4A0wC8DKAmgO5KqWqpNvsMwCGStQH0ATDVYt0UAH+RrA6g\nNoCTjs4VeUNa+kIIkZWcaek3AnCG5CWS8QAWA+iYapsaALYCAMlTAHyUUqWUUsUANCc5L2ldAsko\nRye6fbWo1eNbtyTpCyGEKzmT9MsBCLZ4HJK0zFIggM4AoJRqBKAigPIAKgEIU0rNU0oFKKVmKaUK\nOjrRtdC8Vo/DwqQjVwghXCmPi44zBsAUpVQAgKMADgFIBJAXQD0AQ0geUEpNBjAcwFf2DnL4sB/8\n/PTvLVu2RFhYS2npCyFEEn9/f/j7+z/QMRTJtDdQqgkAP5K+SY+HAyDJsWnscwHAMwAKA/iXZOWk\n5c8BGEayvZ19WLIkcfNmyrIyZYDAQKBs2Qw/LyGEeOwppUBSpb9lCmfKO/sBPKmU8lZK5QPQDcDa\nVCd2U0rlTfp9AIB/SN4leR1AsFKqStKmLwI44ehEkZFAXJz+XSZbE0II10u3vEMyUSn1DoBN0B8S\nc0ieVEoN0qs5C0B1APOVUiYAxwH0tzjEewAWJn0onAfwpqNzlS4NXLsGVKyoPwAKFQLy5nW0tRBC\niIxyqqZPcgOAqqmW/WTx+57U6y3WBQJo6Mx5PD2Bq1d10pfhmkII4XqGuiLXnPQBSfpCCJEVJOkL\nIUQOYqik7+UFhIbq3+XCLCGEcD1DJf3ULX0ZuSOEEK5l6KQvLX0hhHAtSfpCCJGDGCrpW9b0JekL\nIYTrGSrply6tO3ATEuSuWUIIkRUMlfTz5NGJ/sYNaekLIURWMFTSB1Lq+pL0hRDC9QyX9L28gCtX\ngPBwwMMju6MRQojHi+GSvqcncPIkUKSITLYmhBCuZsikf/SodOIKIURWMGTSP3JE6vlCCJEVDJf0\nvbyAoCBJ+kIIkRUMl/Q9PYH4eEn6QgiRFQyZ9AFJ+kIIkRUMl/TNN0GXjlwhhHA9wyX9fPl0K19a\n+kII4XqGS/qALvFI0hdCCNczZNLv0QOoUye7oxBCiMePIpndMQAAlFI0SixCCPEoUEqBpMrIPoZs\n6QshhMgakvSFECIHkaQvhBA5iCR9IYTIQSTpCyFEDiJJXwghchBJ+kIIkYNI0hdCiBxEkr4QQuQg\nkvSFECIHkaQvhBA5iCR9IYTIQSTpCyFEDiJJXwghchBJ+kIIkYNI0hdCiBxEkr4QQuQgkvSFECIH\ncSrpK6V8lVJBSqnTSqlhdta7K6VWKqUClVJ7lFI1LNZdTFp+SCm1z5XBCyGEyJh0k75SKheAaQBe\nBlATQHelVLVUm30G4BDJ2gD6AJhqsc4EoCXJuiQbuSbs7OHv75/dIThF4nQtidO1JM7s5UxLvxGA\nMyQvkYwHsBhAx1Tb1ACwFQBIngLgo5QqlbROOXkew3tU/ggkTteSOF1L4sxeziTjcgCCLR6HJC2z\nFAigMwAopRoBqAigfNI6AtislNqvlBrwYOEKIYR4EHlcdJwxAKYopQIAHAVwCEBi0rpmJK8mtfw3\nK6VOktzpovMKIYTIAEUy7Q2UagLAj6Rv0uPhAEhybBr7XADwDMm7qZZ/BeAOyUl29kk7ECGEEDZI\nqoxs70xLfz+AJ5VS3gCuAugGoLvlBkopNwD3ScYnlXD+IXlXKVUIQK6k3wsDeAnASFcELoQQIuPS\nTfokE5VS7wDYBN0HMIfkSaXUIL2aswBUBzBfKWUCcBxA/6TdywBYldSKzwNgIclNWfFEhBBCpC/d\n8o4QQojHR7YPpUzvwq/sopSao5S6rpQ6YrGsuFJqk1LqlFJqY1JZK1sppcorpbYqpY4rpY4qpd4z\nWqxKqfxKqb1JF+gdTerbMVSMlpRSuZRSAUqptUmPDRenvYseDRqnm1JqmVLqZNLfaGOjxamUqpL0\nOgYk/RuplHrPaHEmxfqBUuqYUuqIUmqhUipfRuPM1qTv5IVf2WUedFyWhgP4m2RV6OsSPn3oUdlK\nAPAhyZoAngUwJOk1NEysJGMBtCJZF0AdAG2ThvYaJsZU3gdwwuKxEeO0d9GjEeOcAuAvktUB1AYQ\nBIPFSfJ00utYD0B9APcArILB4lRKeQF4F0A9krWgS+bdkdE4SWbbD4AmANZbPB4OYFh2xpQqPm8A\nRyweBwEok/R7WQBB2R2jnZhXA2ht1FgBFAJwAEBDI8YIfX3JZgAtAaw16v87gAsASqRaZqg4ARQD\ncM7OckPFmSq2lwDsMGKcALwAXAJQPCnhr83Mez27yzvOXPhlJKVJXgcAktcAlM7meKwopXygW9J7\noP8IDBNrUsnkEIBrADaT3G+0GJN8D+Bj6IsKzYwYp+VFj28lLTNanJUAhCml5iWVTmYljegzWpyW\n/gvg96TfDRUnyVAAEwFcBnAFQCTJv5HBOLM76T/qDNMLrpQqAmA5gPepr49IHVu2xkrSRF3eKQ+g\nkVKqpp2YsjVGpdSrAK6TPAw9fYgjRvh/b0ZdjngFuqTXHAZ7PaFbo/UATE+K9R70t3mjxQkAUErl\nBdABwLKkRYaKUynlDj0Fjjd0q7+wUqqHnbjSjDO7k/4V6CkbzMonLTOq60qpMgCglCoL4EY2xwMA\nUErlgU74v5Fck7TYkLGSjALgD8AXxouxGYAOSqnzABYBeEEp9RuAawaLEySvJv17E7qk1wjGez1D\nAASTPJD0eAX0h4DR4jRrC+AgybCkx0aLszWA8yRvk0yE7ndoigzGmd1JP/nCL6VUPugLv9Zmc0yW\nFB6XQycAAAE4SURBVKxbfGsB9E36vQ+ANal3yCZzAZwgOcVimWFiVUqVNI8oUEoVBNAGwEkYKEYA\nIPkZyYokK0P/LW4l2QvAOhgoTqVUoaRvdlApFz0ehfFez+sAgpVSVZIWvQh9HY+h4rTQHfrD3sxo\ncV4G0EQpVUAppaBfzxPIaJwG6DjxBXAKwBkAw7M7Hou4fgcQCiA26cV+E7oD5e+keDcBcDdAnM2g\n5zk6DD3nUUDSa+phlFgBPJMU12EARwCMSFpumBjtxNwCKR25hooTulZu/v8+an7fGC3OpJhqQzfu\nDgNYCcDNoHEWAnATQFGLZUaM8yvoBtMRAPMB5M1onHJxlhBC5CDZXd4RQgjxEEnSF0KIHESSvhBC\n5CCS9IUQIgeRpC+EEDmIJH0hhMhBJOkLIUQOIklfCCFykP8HSqiNZSvAEPIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11c1d6cd0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(train_accuracies)\n",
    "plt.plot(test_accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.99040002"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(test_accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best error is still only "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.95999836921691895"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "100.0*(1.0-np.max(test_accuracies))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}