2-Layer-DK-MNiST.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-04-20T21:52:04.548207",
     "start_time": "2016-04-20T21:52:04.544771"
    }
   },
   "source": [
    "## 2 Layer MLP for Dark Knowledge Experiments \n",
    "\n",
    "75 epochs of RMSProp on pure MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:12.387973",
     "start_time": "2016-06-13T11:18:12.385629"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:13.232795",
     "start_time": "2016-06-13T11:18:13.227361"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%run augmentmnist.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:14.124853",
     "start_time": "2016-06-13T11:18:14.122398"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 75\n",
    "batch_size = 125\n",
    "\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:16.679306",
     "start_time": "2016-06-13T11:18:16.674365"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Network Parameters\n",
    "n_hidden_1 = 800 # 1st layer num features\n",
    "n_hidden_2 = 800 # 2nd layer num features\n",
    "n_input = 784 # MNIST data input (img shape: 28*28)\n",
    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
    "\n",
    "std_0 = np.sqrt(6.0/(n_input+n_hidden_1))\n",
    "std_h1 = np.sqrt(6.0/(n_hidden_1+n_hidden_2))\n",
    "std_h2 = np.sqrt(6.0/(n_hidden_2+n_classes))\n",
    "                 \n",
    "logfile = \"2-layer-dk-mnist\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:22.479820",
     "start_time": "2016-06-13T11:18:22.473705"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# tf Graph input\n",
    "x = tf.placeholder(\"float\", [None, n_input])\n",
    "y = tf.placeholder(\"float\", [None, n_classes])\n",
    "\n",
    "xt = tf.placeholder(\"float\", [None, n_input])\n",
    "yt = tf.placeholder(\"float\", [None, n_classes])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:23.142953",
     "start_time": "2016-06-13T11:18:23.139206"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def multilayer_perceptron(_X, _weights, _biases):\n",
    "    layer_1 = tf.nn.relu(tf.add(tf.matmul(_X, _weights['h1']), _biases['b1'])) \n",
    "    layer_2 = tf.nn.relu(tf.add(tf.matmul(layer_1, _weights['h2']), _biases['b2'])) \n",
    "    return tf.matmul(layer_2, weights['out']) + biases['out']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:24.248133",
     "start_time": "2016-06-13T11:18:24.210564"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Store layers weight & bias\n",
    "weights = {\n",
    "    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1], stddev=std_0)),\n",
    "    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2], stddev=std_h1)),\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes], stddev=std_h2))\n",
    "}\n",
    "biases = {\n",
    "    'b1': tf.Variable(tf.random_normal([n_hidden_1], stddev=0.1)),\n",
    "    'b2': tf.Variable(tf.random_normal([n_hidden_2], stddev=0.01)),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes], stddev=0.001))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:26.867795",
     "start_time": "2016-06-13T11:18:26.857687"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "mlp = multilayer_perceptron(x, weights, biases )\n",
    "mlp_test = multilayer_perceptron(xt, weights, biases )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:27.444407",
     "start_time": "2016-06-13T11:18:27.433801"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cost =  tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(mlp, y)) \n",
    "gs = tf.get_variable(\"global_step\",[],trainable=False,initializer=tf.constant_initializer(0))\n",
    "lr = tf.constant(learning_rate) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:28.825455",
     "start_time": "2016-06-13T11:18:28.689722"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "train_op = tf.contrib.layers.optimize_loss(cost, global_step=gs, learning_rate=lr,optimizer=\"RMSProp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:29.413716",
     "start_time": "2016-06-13T11:18:29.393385"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'ScalarSummary_2:0' shape=() dtype=string>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#with tf.name_scope(\"training accuracy\"):\n",
    "pred = tf.equal(tf.argmax(mlp, 1), tf.argmax(y, 1)) # Count correct predictions\n",
    "train_acc_op = tf.reduce_mean(tf.cast(pred, \"float\"))  # Cast boolean to float to average\n",
    "tf.scalar_summary(\"training accuracy\", train_acc_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:30.797955",
     "start_time": "2016-06-13T11:18:30.785805"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'ScalarSummary_3:0' shape=() dtype=string>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pred = tf.equal(tf.argmax(mlp_test, 1), tf.argmax(yt, 1)) # Count correct predictions\n",
    "test_acc_op = tf.reduce_mean(tf.cast(test_pred, \"float\"))  # Cast boolean to float to average\n",
    "tf.scalar_summary(\"test 0 accuracy\", test_acc_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:31.185782",
     "start_time": "2016-06-13T11:18:30.944891"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[36m2-layer-mlp-alignmnist-wdropout\u001b[m\u001b[m \u001b[1m\u001b[36m2-layer-modern-mlp-alignmnist\u001b[m\u001b[m\r\n",
      "\u001b[1m\u001b[36m2-layer-mlp-mnist-temp\u001b[m\u001b[m          \u001b[1m\u001b[36m5-layer-mlp-alignmnist-wdropout\u001b[m\u001b[m\r\n",
      "\u001b[1m\u001b[36m2-layer-mlp-mnist-watch\u001b[m\u001b[m         \u001b[1m\u001b[36m5-layer-mlp-infimnist\u001b[m\u001b[m\r\n"
     ]
    }
   ],
   "source": [
    "!rm -rf ./logs/{logfile}\n",
    "!ls logs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T21:57:37.534602",
     "start_time": "2016-05-04T21:57:37.532739"
    },
    "collapsed": true
   },
   "source": [
    "### Original MNIST Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:18:33.670259",
     "start_time": "2016-06-13T11:18:32.761681"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\".\", one_hot=True)\n",
    "trX, trY = mnist.train.images, mnist.train.labels\n",
    "teX, teY = mnist.test.images, mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:38:50.645025",
     "start_time": "2016-06-13T11:18:40.009449"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 0.94727272, 0.94)\n",
      "(1, 0.98223639, 0.97189999)\n",
      "(2, 0.98549092, 0.97259998)\n",
      "(3, 0.98634547, 0.97460002)\n",
      "(4, 0.99001819, 0.97509998)\n",
      "(5, 0.97818184, 0.96219999)\n",
      "(6, 0.99354547, 0.97710001)\n",
      "(7, 0.99456364, 0.97750002)\n",
      "(8, 0.99396366, 0.977)\n",
      "(9, 0.99727273, 0.98070002)\n",
      "(10, 0.99229091, 0.97490001)\n",
      "(11, 0.9964909, 0.97930002)\n",
      "(12, 0.99680001, 0.98079997)\n",
      "(13, 0.99760002, 0.98040003)\n",
      "(14, 0.99710912, 0.97689998)\n",
      "(15, 0.99874544, 0.98070002)\n",
      "(16, 0.99823636, 0.98210001)\n",
      "(17, 0.99699998, 0.98030001)\n",
      "(18, 0.99754548, 0.97920001)\n",
      "(19, 0.99874544, 0.98189998)\n",
      "(20, 0.99634546, 0.9781)\n",
      "(21, 0.99876362, 0.98110002)\n",
      "(22, 0.9946, 0.97640002)\n",
      "(23, 0.99874544, 0.98299998)\n",
      "(24, 0.99727273, 0.98019999)\n",
      "(25, 0.99750906, 0.97829998)\n",
      "(26, 0.99936366, 0.98339999)\n",
      "(27, 0.99959999, 0.9831)\n",
      "(28, 0.99881816, 0.98110002)\n",
      "(29, 0.99598181, 0.97970003)\n",
      "(30, 0.99958181, 0.98220003)\n",
      "(31, 0.99883634, 0.98339999)\n",
      "(32, 0.99938184, 0.9824)\n",
      "(33, 0.99923635, 0.98110002)\n",
      "(34, 0.99930906, 0.98250002)\n",
      "(35, 0.99969089, 0.98329997)\n",
      "(36, 0.99881816, 0.98199999)\n",
      "(37, 0.99967271, 0.98280001)\n",
      "(38, 0.99959999, 0.98220003)\n",
      "(39, 0.99940002, 0.98339999)\n",
      "(40, 0.99969089, 0.98269999)\n",
      "(41, 0.99923635, 0.97839999)\n",
      "(42, 0.99934548, 0.98329997)\n",
      "(43, 0.99952728, 0.98159999)\n",
      "(44, 0.99929088, 0.9824)\n",
      "(45, 0.99961817, 0.98339999)\n",
      "(46, 0.99907273, 0.98220003)\n",
      "(47, 0.99961817, 0.98210001)\n",
      "(48, 0.99956363, 0.98430002)\n",
      "(49, 0.9994182, 0.98280001)\n",
      "(50, 0.99983639, 0.9842)\n",
      "(51, 0.99970907, 0.98140001)\n",
      "(52, 0.99925452, 0.98229998)\n",
      "(53, 0.9993273, 0.98220003)\n",
      "(54, 0.99992728, 0.98509997)\n",
      "(55, 0.99987274, 0.98430002)\n",
      "(56, 0.99985456, 0.98269999)\n",
      "(57, 0.99989092, 0.98280001)\n",
      "(58, 0.99959999, 0.98299998)\n",
      "(59, 0.99992728, 0.98329997)\n",
      "(60, 0.9999091, 0.9831)\n",
      "(61, 0.99974543, 0.9831)\n",
      "(62, 0.99963635, 0.98290002)\n",
      "(63, 0.99985456, 0.98409998)\n",
      "(64, 0.99996364, 0.98439997)\n",
      "(65, 0.99989092, 0.98339999)\n",
      "(66, 0.99998182, 0.98519999)\n",
      "(67, 0.99998182, 0.98509997)\n",
      "(68, 0.99998182, 0.98509997)\n",
      "(69, 0.99998182, 0.98519999)\n",
      "(70, 0.99998182, 0.9853)\n",
      "(71, 0.99998182, 0.9853)\n",
      "(72, 0.99998182, 0.9853)\n",
      "(73, 0.99998182, 0.98540002)\n",
      "(74, 0.99998182, 0.9853)\n"
     ]
    }
   ],
   "source": [
    "test_accuracies = []\n",
    "train_accuracies = []\n",
    "with tf.Session() as sess:\n",
    "    # create a log writer. run 'tensorboard --logdir=./logs/{logfile}'\n",
    "    writer = tf.train.SummaryWriter(\"./logs/{0}\".format(logfile), sess.graph) # for 0.8\n",
    "    merged = tf.merge_all_summaries()\n",
    " \n",
    "    tf.initialize_all_variables().run()\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "      \n",
    "        for start, end in zip(range(0, len(trX), batch_size), range(batch_size, len(trX), batch_size)):\n",
    "            sess.run(train_op, feed_dict={x: trX[start:end], y: trY[start:end]})\n",
    "                 \n",
    "        summary, trn_acc, tst_acc = sess.run([merged, train_acc_op, test_acc_op], feed_dict={x: trX, y: trY, xt: teX, yt: teY})\n",
    "        writer.add_summary(summary, epoch)  \n",
    "        \n",
    "        print(epoch, trn_acc, tst_acc)\n",
    "        train_accuracies.append(trn_acc)\n",
    "        test_accuracies.append(tst_acc)\n",
    "\n",
    "        writer.flush()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:39:00.248868",
     "start_time": "2016-06-13T11:38:58.629080"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charlesmartin14/anaconda2/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:39:38.105650",
     "start_time": "2016-06-13T11:39:37.929988"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x11d7f6090>]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEACAYAAABfxaZOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xd4lFX2wPHvSei9SgtFEBCQIgpLUYzSFRtW7A0rtnVd\nsGfdXQUUBbsoAkpbRURAflINKwrCQigiAaTXQOhpkGTO7493ElJmkkkjb5jzeZ48ZO7bzkzCmZvz\n3ntHVBVjjDHBIaS4AzDGGHP2WNI3xpggYknfGGOCiCV9Y4wJIpb0jTEmiFjSN8aYIJJr0heRcSIS\nIyLrctjnPRHZIiJrRKRDhvZ+IhItIptFZGhhBW2MMSZ/Aunpjwf6+tsoIv2BZqraHHgE+MTbHgJ8\n4D22DTBIRC4scMTGGGPyLdekr6pLgaM57HI98KV339+AqiJSB+gMbFHVnaqaDEzz7muMMaaYFEZN\nvwGwO8PjPd42f+3GGGOKSVHcyJUiOKcxxphCUKoQzrEXaJjhcZi3rQzQyEe7TyJiiwAZY0weqWqe\nOtqBJn3Bfw9+FvAE8B8R6QIcU9UYEYkFLhCRxsB+4HZgUE4XcfvibxEREURERBR3GLkq6jh//x2u\nvhpOnoR27ZyvGjXgo4/grrsgIgKqVs35HA88AN9+G0G9ehGMGuWcT7y/YZs3w8cfw7ffOudp2BDC\nwqBBA0hJgRMnnK9jx+C336BKFef4q6+G5cthzBgYPty5hgjs3g2PPur8O3q0s/+JE3D8OBw9Clu3\nwqZNzteWLVC/PnTtCl26OF/vvhuBSAQ//ACdOztthw/DoUPO1/79sHMn1KwJzZo5x2/dChs2QJ06\n0KqVc97Dh+GKK+Dyy51t337rxPz441C9uvOcJ0+GK6+EgQMhNRXi452vEydg3z7nOezZAzExThwD\nB8K110Lt2pl/7h6P81pMnQrffOO8fpdfDgkJzvM+cQKio+G88+D55+GGGyA0NP+/E6dPw9q1Tmxd\nu0Lduv73zfr7qQpxcWdezzJl4MILoXz5/MdTGErC/3eRvBdWck36IjIFCAdqisgu4DWcXryq6lhV\nnSsiV4vIn0A8cD/OxlQRGQLMxykjjVPVjXmO0LhKUhLccQe8/DIMHnwmUQMMGQIvvOAkueHD4e67\nM29Pk5AAM2fCQw9Bz57w1786ifrOO2HSJFi3Dh58EObNc5LJnj1Ostu3z0kITZs6ibtKFXjrLbjg\ngjPn7tULrrsO7r8fvv4a+vWDN96Ap56C775zjs+JavaYW7Rw3sgSEmDOHOdN78ILnSR63nlOgmvS\nBCpUyHxcaqqT/P/4w4n5oosgJENB9a23YMIEuO8+5w304Yed5x4WlvvPITchIdCtm/P17rsQGQlR\nUVC5svNGWqWK8ybavr3vn1FelSkDnTo5X3kl4sRVubLzOpmilWvSV9U7AthniJ/2H4GW+YgrKCxd\n6iS8AQOcJNWwYe7HZKTqJN+UFBgxwvc+O3fCrbfCf/8LZcv63ue555wkNnhw7td88UVo3jx7wgen\nt/n557BiBdx7r5N47ror+zlmzXJ6zJUrOz3d3r3hk09g2jQnAd58c+ZY27fPPa6M2rVzerkjRsCC\nBU7Ca9MmsGNzSoAVKjiv5a23Bnau0FDnDaNFC9/bq1eHZ5+FZ55xfpYhRTRVslQp582wV6+iOb8p\nYVTVFV9OKO72008/Fdq5du9WrVdP9aOPVB9/XLV6ddW+fVVnzFD1eHI/PjlZ9d57Vbt0Ua1dW3XD\nBt9x3n23akiI6ty5vs+TkKBapYpq3bqq332X8zXnz1dt0EA1Njb3+ObNU73oIt/PpX9/1UmTCvf1\nLEoWZ+GyOAuPN2/mLdfm9YCi+ioJSb+wJCaqduqkOnz4mbb4eNWvvlJt1Ur1mWdyTvzx8aoDBjjJ\nMy5O9Z13VK+9Nvt+a9ao1qmj+uqrqoMH+z7XrFmqPXqo/u9/qrVqqS5d6nu/Q4echL9gQWDP0eNR\n7dBB9YcfMrcfOKBarZoTtzGmYPKT9EVdcvNURNQtsRSWQ4ec0ku9emfaVJ0bjAkJTjkjaznh6FGn\n5NGmDXz6afaba8eOOTfumjSBL76A0qXh1CmnPDNhgnOjME2/fk7pqH9/p7a7b1/28z3wgFMOeeYZ\n+PFHpywTGenU5TPGfNNNcP75MGpU4M9/6lSnbLNkyZm20aNhzRonVmNMwYhInkfvFHsPP+0Ll/X0\nPR7VY8fyd2xysuro0U7PuVo11X79VKdNc3r477+v2rZtzj3dkydVr7pK9dZbVU+dctq2bVMdOtQp\n5fz1r6qpqZmPmTLF+esh7S+EhQtVmzU7c3zbttl78cnJTozbt59pmzBBtXFj1W++Uf3Xv1RvuUW1\nRQvViy9WTUrK++vQuLHqsmVn2i6+2InNGFNw5KOnb6ts+jFxojOyJK+WLYNLL3VuVv78M+zd69zM\nHDfOGS3xz386I1cqVvR/jkqV4IcfnJEy11zj9NY7dYLkZPj1V6e3nfWm3223OcP0vvnG+ffvf3dG\nraSNVrnhBmf0SkZLlzo3j5s0OdN2773Ojd1x45xhfddfD9OnO0Mj/d0I9qdUKedcI0c6jzdsgIMH\nITw8b+cxxhQeK+/40aWLMwolJsYZleKPx+MMyVu6FBYudJL+22/D7bdnL93s3u3s37hxYDEkJ8O/\n/w2NGjnnyzokMKuffnKGQb76KnzwgZOo094coqLglluc8eJpcT39NNSqBa+8Elg8+REf75SFfv4Z\nxo932oYPL7rrGRNM8lPesaTvw7p1Tl29XTtnCKG/IXqPPOL0rGvWhMsuc75uvjn3iUlF6ZprYP58\n5+vKK8+0qzrJd/ZsaNvWedykifMXxUUXFW1MERGwa5cT07x5gQ+fNMbkzJJ+IXnySWcMdY0asHGj\nc0M1qz//dJJ8VFTmG7XFLTraKU29+Wb2bc8+6zyvV1+FVatg0CBnFmphTM7JSWys89dNy5awenXR\nXsuYYGJJvxAkJDh17tWrnVmS11/vzKrMatQoZ7kAX28IbrVkiZP4V692JnUlJ/uf1FXYXn/d+Uvj\n7rvPzvWMCQb5SfqFseDaOWX6dGe2aOPGTgkkPh62b3cSVkYzZzqzU0uSyy5zljTYscO5qTtu3Nm7\n9quvnr1rGWP8s9E7WXz2mbMGCjhlj549YdGizPscPAjr18NVV539+AoiNNQZ4z9ypDMfoHPn4o7I\nGHO2WdLP4I8/nFr9gAFn2nwl/dmzoW/fvA9hdIMbb3QmTN1wQ9Gt9WKMca+g/G+vClOmOKWc1NQz\n7Z9/7ix8Vrr0mbZevZyk7/GcaZs500maJVGvXs7QzxtvLO5IjDHFIehu5MbHO0Mt161zJkHFxDgr\nXd5xh7Ma4m+/ZV/etXlzZ+3zdu2cdb/r13fG3Bfn0MyC2LrVeY5FPWrHGFO08nMjN6h6+ps3O5Ou\nQkOdpXd//RW+/NJZfjcsDDp08L2ed1pvH5xx5l27ltyED84HfVjCNyY4BU3SnzPHGb0yZIiz2Ffa\n7Nbu3Z1yTVSUcxPXl549ndm2ULJLO8YYEzTlnVatnBUe+/bN+7GHDzt/Aezf7/xFsH69s46OMcYU\nJyvv5CAmJn8f5QbOMgsXXOB8vF3z5pbwjTElV1Ak/dOnndm11arl/xw9ezqzV620Y4wpyYIi6cfG\nOqtJFmRceq9ekJhoSd8YU7IFRdI/eBDOO69g57j8cnjhBecTqowxpqQKirV3CiPply/vfCiJMcaU\nZNbTN8aYIBJQ0heRfiISLSKbRWSoj+3VRGSGiKwVkeUi0jrDtqdFZL3366nCDD5QlvSNMcaRa9IX\nkRDgA6Av0AYYJCJZK9svAlGq2h64F3jPe2wb4EHgUqADMEBEfMx5LVqW9I0xxhFIT78zsEVVd6pq\nMjANuD7LPq2BxQCqugloIiK1gVbAb6p6SlVTgf8CAwst+gAdOpTz59waY0ywCCTpNwB2Z3i8x9uW\n0Vq8yVxEOgONgDDgd+ByEakuIhWAq4GGBQ3al4yrZWZlPX1jjHEU1uid4cAYEVkNrAeigFRVjRaR\nEcACIC6t3d9JIiIi0r8PDw8nPDw8oIv/8ouzUuZvv/nebknfGHMuiIyMJDIyskDnyHXtHRHpAkSo\naj/v42GAqqrfT1cVke1AW1WNy9L+b2C3qn7i45h8r73z8MPw9ddw7Jjv7eef76yS6WsFTWOMKamK\nau2dlcAFItJYRMoAtwOzsly4qoiU9n4/GFiSlvC9tX1EpBFwIzAlLwHm5tQpZ637hARnqQVfrKdv\njDGOXMs7qpoqIkOA+ThvEuNUdaOIPOJs1rE4N2wniogH2IAzYifNtyJSA0gGHlfVE4X5BObOhbZt\nnRUw9+7NPmM2Pt75pKyKFQvzqsYYUzKV+KWVb7oJ+veHqVNh2DDo3Tvz9u3b4corYceOwonTGGPc\nIuiWVj52zPlwk5tvhoYNYc+e7PscPGjDNY0xJk2JTvrTpzurX1ar5ny4ye7d2fexer4xxpxRopP+\n5Mlw553O92Fhvnv6hw5Z0jfGmDQlNunv3g3r1sHVVzuPcyrvWNI3xhhHiU36U6fCwIFQrpzz2Mo7\nxhiTuxKb9CdNgrvuOvPYX3nHkr4xxpxRIpP+6tXOyJ3LLz/TVqOGM1ErLi7zvpb0jTHmjBKV9D0e\nGDsW+vaFiIjMn3kr4ru3b0M2jTHmjBLzcYlbt8JDDznLLfz0E1x0UfZ90m7mZpyVaz19Y4w5o0Qk\n/W+/hUcegRdfhKefhtBQ3/tl7emr2lr6xpi8S05NxqMev9sVJe50HIcTDhObEMvhxMOkelKpWaEm\ntSrUomb5mtQoX4PSoaXPYtSBcX3SV4UXXnAmYuW20nLWETzHjjlr7pQtW6QhGnNOWLRtEV0bdqVC\n6QrFHUqB/Ou//2L9wfXUKl8rPQlnfU5JKUmZEvbhRO/33raklCRCQ/z0Lr0qlalEzfLeJF+hJqES\nmuk8rWu3JvK+yCJ8pvnj+qT/009O0r7iitz3bdgQoqLOPLbSjjGB+f3g71wz5RrqVa7H+/3fZ0CL\nAcUdUr5MXjeZSesm8doVr6Un4C2Ht5CYkphpv7KhZalZoSbNazanS/ku6W8Oab30SmUqIZKnJW2y\nccu6Zlm5Pul/+qlT2gnk9Q8Lg1kZFn22pG9MYN5Z9g4v93iZvzT4C0/MfYLPV3/OmH5jaFytcXGH\nlkmKJ4VjSceoVaFWtm1bj2zlmXnPsODuBXSo26EYosusoG8aRcXVo3diYmDevMzj8XOSdVauJX1z\nLjgQd4ABUwaw7+Q+v/vM2jSLPSd8TFQJwP6T+5kZPZPHLn2M3s16s/6x9Vxa/1IuGXsJ8/6cl9+w\nC0WKJ4V5f87j1Z9epdeXvagxogZNRjdh5C8jM9XcT6ee5vZvb+fVHq+6IuG7mauT/vjxzqzbatUC\n2z/rjVwbrmnOBUMXDmXn8Z0MmDKAk6eyf1LQpHWTuOnrm3jlp1fydf4PVnzAHW3voGaFmgCULVWW\nl3u8zMzbZ3LPzHvYfHhzgeLPr4TkBG78z40MWzSMVE8qz3Z5lh3P7OCPJ/5gxsYZ3DDtBo4mHgXg\npUUvUa9SPYZ0HlIssZYoquqKLyeUM1JTVZs2Vf3tNw2Yx6NavrxqXJzzOCJC9eWXAz/eGLdZunOp\nhr0TpieSTujgWYO136R+mpyanL595saZWuetOrp051KtPry67jq2K0/nP3nqpNYaWUv/PPynz+2f\nrfpMW77fUo8mHi3Q88irIwlHtPu47nrXjLv0dMrpbNtPpZzSp//vaW0yuomO+nWUhr0TpofiD53V\nGN3AmzfzlGtd29NfuBCqVIFOnQI/JusELSvvFI3dx3dTb1Q9en/Vm4jICOZvnc+JUwX7QLSPV35M\nqw9bsXzP8nwdn+pJ5bNVnzFxzcR8x7Dv5D66f9Gd26bfxpbDWwI65njScf7933/nOLwvo93Hd9Nj\nfA8em/MYqZ7UHPdN8aTwxNwneKv3W1QuW5mPrvkIgMfmPIaqsmjbIgbPHsycO+bQvVF37u9wP+8s\neyegONKMjxrPFY2voFmNZj63P9TxIfo068OgbwflGm+a06mnWbF3BaOXj+bB7x/ktZ9eY8r6Kaza\nt4qTp06S6knN9JXVvpP76DGhB50bdGbiDRN9DnssE1qG0f1G81bvtxi+dDhf3fiVzzq/yc61n5x1\n003Op2A9+mjeznPVVfDSS9CzJ9xyi/MBK7fdVsjBnmO+XPslF9e9mLZ12ga0/xdRXzBr0ywe6vgQ\nv+z6hV/3/MraA2tZfO9iOtbrmKdrqyqvL3mdr9Z9xfPdnueVn17hpctf4qm/PBXwjbBV+1bx+NzH\nKRVSir0n9vJc1+d48i9P+r2er/NuObyFPpP68ODFDxIqobyz/B1uanUTr17xKvUr1/d77anrp3LH\njDv415X/4qUeL+UY549//sh9M+/jqb88xeLti6lariqTB06mXKlyPvf/YMUHzNg4g0X3LEqP+eSp\nk1wx4Qo61uvIrE2z+OaWb7iiiTO0be+JvbT9uC1bntySXqrJSaonlebvN2fywMl0bdjV734pnhT6\nTepHh7odeLvP2+ntyanJ7D25l02xm9h0eBObYjex/uB6Vu9fTbMazegW1o12ddqxP25/+vYtR7aQ\nlJKUfg5VpXr56rSs2ZKWtVrSvEZzPlv9GY9e8ih/7/73gH4H/P1Mg0F+Pjmr2Ms6aV9kKO/s3ata\nrZrq8eN5/3Pn7rtVx493vu/RQ3Xx4ryfI9i0/aitXjP5moD3v/PbO/WzVZ9lapu4ZqK2+7idnko5\nFfB5UlJT9PE5j2uHTzrogZMHVFV165Gt2vHTjnrz1zfr8aScfwGOJh7VIT8M0Tpv1dHxUeM11ZOq\n249u1yajm+joZaMz7Xsk4Yg+NucxLf+v8nr/zPt1w8EN6dtW7Vul9d6ul+k5xcbH6vPzn9caI2ro\nzI0z/cZw/8z79YWFL2i9t+vpgq0L/D7Plxe9rA1GNdAlO5aoqmpScpLe8vUtGj4h3OfzjImL0doj\na+vvMb9n27bvxD695NNLdM6mOdm2Pfj9g/qPyH/4jTejbzZ8o93GdQto39j4WG02ppl2/LSjNh3T\nVKu+WVVD/xGqDUY10KsmXqWPzXlMRy8brfP/nJ/rzy0jj8ejB04e0CU7lujY/43V5+Y9p1PXTw34\n+GBHPso7xZ7s0wPJkPT/+U/Vhx/O34vw4ovO8aqqrVqp/p79/4zJIOF0gpb/V3mt+3ZdXXtgba77\nezwerT+qvm49sjVbe/9J/fX1yNcDuu6plFN62ze3afiEcD2WeCzTtsTkRH109qPa/L3muuPoDp/H\nH4w7qK0+aKUPfv+gxsbHZtq24+gOPX/0+frOr++ox+PRiWsmat236+pjcx7TLYe36OuRr2udt+ro\ngCkD9OOVH2vtkbX1u43f+bzO179/rX2+6uP3tQh7J0yjD0Xrom2LtO7bdXX38d2Z9tl4aKP2GN9D\nr5p4VfobW5qU1BR9dPajevEnF2v0oWiNPx2fvu2BmQ/oX3/8q+8XLwfRh6K19sjaGncqLlP7n4f/\n1NHLRuvENRP1h80/6PLdy7XT2E46448ZAZ87Ji5Gl+1epptjN+uRhCOa6knNc3ymcJ0zST88XHXe\nvPy9CB99pPrII873NWuqHjyYv/MUh13HdumAKQN0z/E9AR+TkpqS7T94XizbvUwv/uRiHbF0hN75\n7Z257h99KFobv9tYPR5Ptm27ju3SWiNr6fqY9bme56VFL2n/Sf01MTnR7z6jl43WJqOb6Paj2zO1\nH0s8ph0/7agvLnzR77E7j+3UZmOaaZsP2+gln16iK/asyLQ94XSCfrLyEw2fEK4/bf/J73lOJJ3Q\nym9U9tl7jT4UrQ3faZj+Wrzx3ze0y+dd9FTKKY0/Ha8vLnxRa46oqe8ue1dTUlN8nt/j8ejrka9r\ng1ENtOw/y2q5f5XTsHfCtP6o+nnqMWd0039u0jHLx6Sf/8MVH2rNETX1oe8f0rtm3KX9JvXTTmM7\n6TWTr/EblykZzpmk36mT6vLl+XsRZs9Wvfpq1eRk1dBQ1ZQS8jsdExejLd9vqT3G99DwCeEB/2d8\n9sdndeB/Bub7uh/89oE+9P1DejzpuNYcUVO3HdmW4/4frfhI7595v9/tn/7vU+00tlOmESZZHTh5\nQGuMqKE7j+3MNb73f3tfG7/bOD2u+NPxevkXl+vjcx73+caT0Z7je/TLNV8WOLH1n9Rfv/79a5+x\nZXwtUj2peu2Ua/WGaTdok9FN9Pbpt+veE3sDvo7H49G4U3G689hOPZxwON/xrtizQhu920i3Hdmm\nfb7qo53GdtKNhzbm+3zGvfKT9F05eichASrkc/mPtNE7sbHOGvv+Fmdzk2NJx+g7qS+3trmVxfcs\nBuDNpW/metzB+INMWDOBhdsWcjzpeL6uvWr/Ki6pfwlVylbh4UseZtSyUTnuv3jHYq5scqXf7YM7\nDqZy2cqMXj7a7z5v/PwGd7e7m0ZVG+Ua35DOQ/h7978TPjGc6Nhobv76ZhpXa8z7V7+f6827BlUa\ncHf7u3NdQyU317W8jlmbZ2VrX7htIb2b9k5/HCIhTLxhImVCy/DZtZ8x9aapOd4EzkpEqFimIo2q\nNqJG+Rr5jrdTg040r9Gciz6+iMsaXsavD/7KhbUuzP1AExwCeWcA+gHRwGZgqI/t1YAZwFpgOdA6\nw7Zngd+BdcBkoIyfa6S/ezVtqvqn72HDuTp0SLV6ddW1a1XbtMnfOc6m+NPx2n1cd31y7pPpPdc9\nx/ekj73OydAFQ/XxOY/r9VOv1wlRE3zuk5ScpG/98pbfXnG7j9ullz4OnDyg1YdX15i4GJ/7pnpS\nteaImtnq1lltPbJVa42spesOrMu2bcfRHVpjRA2/1/Dn45Ufa+nXS+v1U6/P8a+IorD7+G6tMaJG\npusmpyZr1Ter5vl5nC1bj2z1+fqbcwtFUd7BmbX7J9AYKA2sAS7Mss9I4BXv9y2Bhd7v6wPb0hI9\n8B/gHj/XSX8ides6I3jyw+NRLVdOdeZM1SuvzN85VJ2a75r9a/JwXY+OXDpSX498XdfsX5Nr6UHV\nSRx9v+qr93x3T7abYrOiZ2mjdxvpkYQjPo+NjY9NL5FMWz9N+37V1+d+E9dMVCLw+ed92k3cjHX1\nx+Y8pi8tesnnudbsX6Mt3m+R6/NSVZ22fpo2GNUg26Sf+2fery8vyt+MuV93/ZrjPYCidMmnl2jk\n9sj0x7/s+kXbf9y+WGIxJk1+kn4g5Z3OwBZV3amqycA04Pos+7QGFnsz9yagiYikLYAQClQUkVJA\nBcD/AiJeiYn5L++kTdCKiirYxKxxUeMY9O2ggPcf+ctIJq2fxJHEI9z4nxtpMqYJT859ku1Ht/s9\nZu6WuRxJPMK468YRIpl/FNe2vJbrW17P4NmD094UM3nvt/e48cIbaVS1EQNaDGDZnmUcij+UaR9V\nZfTy0TSv0dznGirrYtbRslbLTOPE/9btb3zyv098TrZavH0xVzW5KtfXAuC2i27jlR6v0Pur3uw9\nsReA6NhoZm+ezXPdngvoHFl1bdjV75j2onZdy+v4ftP36Y8XbltIr6a9iiUWYwoikKTfAMiwSj17\nvG0ZrQUGAohIZ6AREKaq+4BRwC5gL3BMVRfmdsGEBChfPoDI/GjYEFatKljSn7x+MtGx0elre+Rk\n6vqpfPS/j5h7x1ze7fcuW5/aytw75pKUksTQhUP9H/f7VO7vcD+lQnwvdjqy90h2HNvB4z88TnJq\ncnr78aTjfLjyQ4ZdNgyAimUqcnXzq5n+x/RMx/+862fik+P591X/5setP2Y7/6r9q+hYN/NkqqbV\nm9KnWR/GLB+Tbf/FOxZz1fmBJX2ARy59hEcvfZTeX/UmNiGWV356hb91/RvVygW4mJKLXNfyOmZt\nmpX+Brxg24JM9XxjSorCupE7HKguIquBJ4AoIFVEquH8VdAYp9RTSUTu8HeSiIgIXn01guTkCH79\nNTLfwYSFOR+ent+kv/XIVrYe2Ur3Rt1zXRZgyY4lPP3j08wZNIcGVZz3QhGhzXltGNl7JPO2zuNI\n4pFsx8Wfjmfulrnc3Ppmv+cuV6oci+9dzM7jO7l6ytXpb0AfrvyQ/s37c0GNC9L3HXTRIKZtmJbp\n+NHLR/P0X56md7Pe/LLrFxKTM68pvmqfcxM3qzd7vsl7K95jXcy69LYUTwo/7/yZ8CbhOb4eWf29\n+9+54cIbuOyLy/hl1y9+Z8q6Xfs67Un2JLMxdiMnT50kan8Ulze+vLjDMkEmMjKSiIiI9K98ya3+\nA3QBfszweBg+buZmOWYbUAm4GfgsQ/vdwAd+jlFV1RMnVCtVKlid64UXnLsVn3ySv+P/ueSf+sQP\nT+hLi17Ksf78x8E/9Ly3ztOFWxf63ee2b27TD1d8mK09pzp8VsmpyfrM/z2jLd9vqVH7o/S8t87T\nPw7+kWmfpOQkrTGiRvpN1q1HtmrNETX15KmTqqrabVw3nfdn5skPHT7poMt3+x4bOz5qvLb9qK0m\nJSepqury3cu17UdtA4o3K4/Ho8MWDNMp66bk63i3GPLDEH3z5zd19qbZeuWEAtwwMqaQUEQ1/ZXA\nBSLSWETKALcDmcaviUhVESnt/X4w8F9VjcMp63QRkXLijK/rCWzM6WIFLe2AU96B/PX0VZXJ6ydz\nZ9s76dawG7/u+dXvfjf+50ZG9BpBz6Y9/Z7vvg73MWHNhGztU3+fyqCLArtnUCqkFO/2e5dnuzxL\nl8+7cEXjK2hVu1WmfcqWKssNLW/g6w1fA866LQ9c/ACVylQCoF+zfpnq+kkpSWyK3US7Ou18XvPe\n9vfSrEYzXot8DfDW8/NQ2slIRHiz15sMahv4PRI3SivxZB2qaUyJEsg7A86QzU3AFmCYt+0R4GE9\n89fAJpyEPh2omuHY17zt64CJQGk/11BV1W3bVBs3Lti736xZTk9/ac4jHn1atW+Vnj/6fPV4PHo4\n4bBWeqOSzyGCaw+s1aZjmuY6SiclNUXrj6qfaQ2Vo4lHtcqbVbItPxCIFXtW+J2xO//P+dppbCc9\nnnRcqw/RR8rcAAAXJklEQVSvnmny0297ftPWH7bOdJ52H7fL8VoxcTFa9+26+vPOn7XXl730++jv\n8xzvueRUyimtNryahr0Tlm2GrzHFgaKanKWqP6pqS1VtrqrDvW2fqupY7/fLvdtbqerNqno8w7H/\n8La3U9V71RkB5FdBRu6kCQtz/s1PT3/yusnc0fYORIQa5WvQsEpD1sesz7bfnM1zGNB8QK4ThEJD\nQrm73d1MXHtmyd/vNn5Hz/N7UrVc1TzH16lBp/R7B1ldef6V7Dy+k1cWv0Kvpr0yTX66pN4lxMTF\nsPu4c09+1f5VXFIvez0/o/MqnsenAz7lnu/uYfme5fRo3CPP8Z5LyoSWoU+zPsSfjs/zaqLGuIXr\nZuQWZ3kn1ZPK1N+ncmfbO9PbujXsxq+7s5d4Zm+ezbUtrw3ovPe2v5ev1n1FiicFgGkbpnH7Rbfn\nLbgAlAopxS2tb+G9Fe/xTJdnMm0LDQmlT7M+zNvqlHhW7cs96YNT0riyyZW0qtWqRI66KWyDLhrE\nza1vLvAsX2OKi+uSfmH09GvWhFdecT6EJS8id0RSt1LdTPVyX3X9g/EH2XhoY8A931a1W9G4amPm\nb53PwfiD/LbnNwa0GJC34AL0wMUPcFOrm+galn199L7N+p5J+vt9j9zx5cNrPmTGbTMKNc6S6oYL\nb2DstWOLOwxj8s11Sb8wevoi8Prrzr/+HEk8wqmUU5napqyfkqmXD757+nO3zKVX016UCS0TcExp\nN3Sn/zGda1pcQ4XSBXxn86NjvY5Mv3W6z7JTn2Z9WLRtEfGn44mOjaZ9nfYBnbNcqXKEVQkr7FCN\nMcXAlUm/oD19X1SV6NhoRv4yksu+uIzGoxvT4J0GPDn3SaL2R5GUksR30d9lK7u0qNmCE6dOsP/k\n/vS2OZvncG2LwEo7aW5rcxvzts5j7KqxAY/aKWz1KtejUdVGjIsaxwU1LqB86QK+uxpjShzXJf3C\nKO/4ctv02+j1ZS92HNvByz1e5tDzh1g5eCU1K9Tkxv/cSMsPWtKhbodsN0lDJISuYV1ZtmcZ4Hz+\n58JtC+nfvH+erl+9fHX6X9CfXcd30adZn0J7XnnV74J+vPXrW3Yj0pgg5Xv+fzEqjPKOL0t3LWXF\n4BWZyhTnVz+fiPAIXr3iVX7a/hN1KtXxeWxaiWdgq4Es2bGE1rVbc17FvA8Ner7b83Rv2D1PZaHC\n1rdZX0b8MiKgm7jGmHOP65J+UfT0PeohNiHWb6IOkZAcJ1h1a9iNFxe9CHiHaubzJuwl9S8J+OZp\nUeneqDsVS1cs9jiMMcXDdeWdoujpH008SqUylfLdw+5UvxNrY9aSlJLE7M2zi2zkzdlQJrQMS+5b\nQpewLsUdijGmGLiup18UN3IPxh/MVzkmTcUyFWlVqxWT1k0iVVNpe17bQozu7LNevjHBy3U9/aIo\n78TEx/it1weqW8Nu/GPJPwKahWuMMW7luqSf1/KOqrJ011JmbPQ/eaigPX2ArmFd2XNiT8CzcI0x\nxo1cV94JtKe/98Revlz7JePXjOd06mlCJISBrQb63DcmLoY6FQvW07+s0WVUL1c9z+vJG2OMm7iy\np59b0h+3ehwXfXwR249tZ+INE9nw+Ab2ndzn82MFwSnvFLSn37BqQ3Y/u7vYPq7PGGMKg+t6+rmV\nd3Ye28mwRcP45YFfaF27dXp7uVLlOJp0lBrla2Q75mD8wUIZl16xTMUCn8MYY4qT63r6OZV3VJWH\n5zzMX7v8NVPCB2hQpQH7Tvr+zPXC6OkbY8y5wHVJP6ee/oQ1E4hNiOVv3f6WbVv9yvX9Jv2D8QcL\nPHrHGGPOBa4s7/jq6e87uY+hC4ey4O4FlA4tnW17Tkk/Js56+sYYAy7s6fsq76gqj855lMcufYz2\ndX0vB1y/Ui49/QKO3jHGmHOBK3v6Gcs7qZ5Uxvw2hu3HtjP91ul+j6tfuT6bDm/K1h5/Op5UTU3/\ngHBjjAlmru3pqyqzN82mw6cdmLFxBl/f/HWOa+f4K++k9fJtFq0xxri0p7/pxGoGznmK46eO88ZV\nbzCgRe5LH/hL+oWxBIMxxpwrXJX0VZ2k/8rS5xjQYgDPd3s+4A+gzqmnbzdxjTHG4aryzunTULo0\nxJ0+Sa+mvQJO+AB1K9XlQNwBPOrJ1F4YSzAYY8y5IqCkLyL9RCRaRDaLyFAf26uJyAwRWSsiy0Wk\ntbe9hYhEichq77/HReQpf9dJu4mbkJyQ5w8OL1uqLFXLVeVQ/KFM7TYxyxhjzsg16YtICPAB0Bdo\nAwwSkQuz7PYiEKWq7YF7gfcAVHWzql6sqh2BS4B44Dt/10q7iRufHE/F0nlf8sBXiceGaxpjzBmB\n9PQ7A1tUdaeqJgPTgOuz7NMaWAygqpuAJiJSO8s+vYCtqrrb34XSJmblp6cPvpO+9fSNMeaMQJJ+\nAyBjot7jbctoLTAQQEQ6A42AsCz73AZMzelCaeWd+NPx+VrczNcELVuCwRhjziis0TvDgTEishpY\nD0QBqWkbRaQ0cB0wLKeTvPdeBEeOKIkLElnebTlXXXlVnoLw2dO3JRiMMeeIyMhIIiMjC3SOQJL+\nXpyee5owb1s6VT0JPJD2WES2A9sy7NIfWKWqme+yZnHnnRFs2hbPsV5v5znhg5P01xxYk6nNavrG\nmHNFeHg44eHh6Y//8Y9/5PkcgZR3VgIXiEhjESkD3A7MyriDiFT19uYRkcHAElWNy7DLIHIp7YBT\n3ilTKX/1fPD29OPO9PSTU5M5fuq4zzX2jTEmGOXa01fVVBEZAszHeZMYp6obReQRZ7OOBVoBE0XE\nA2wAHkw7XkQq4NzEfTi3ayUmQpmK+Ru5A9nX1I9NiKVm+Zp5Gu9vjDHnsoBq+qr6I9AyS9unGb5f\nnnV7hm0JQNaRPD4lJECpCvEF6+lnSPo2cscYYzJz1YzcxEQoVT4h3x9LeF7F8ziccJgUTwrgnY1r\nI3eMMSadq5J+QgKEls9/eadUSClqVahFTFwMYOvuGGNMVq5L+iFl838jFzKXeGLibd0dY4zJyFVJ\nPzERQsrmb2JWmoxJ34ZrGmNMZq5K+gkJQOnC7elbeccYY85wVdJPTARK57+mDz56+nYj1xhj0rkq\n6Rd6T9+WYDDGmExc9clZCQlAaCH09OOspm+MMb64qqefmAgaWjg9fVXlYPxBalcMaF6YMcYEBVcl\n/YQESAkp+OidvSf2cizpGBVKV6BcqXKFGKExxpRs7kv6UrCefq0KtThx6gS7ju+yer4xxmThqqSf\nmAjJUrCafoiEULdSXdYcWGMjd4wxJgtXJf2EBDit+V9wLU3auvrW0zfGmMxclfQTE+G05n/BtTT1\nK9cn6kCUjdwxxpgsXJX0ExIgyWM9fWOMKSquS/qnUhMKVNMHaFC5AcdPHbeevjHGZOGqpH/6NCSk\nFE5PH7CevjHGZOGqpF+uHCQkF05NH7DRO8YYk4Wrkn6FChCfXHg9fSvvGGNMZq5K+uUrqNPTL2BN\n38o7xhjjm6uSfrmKpygdUprQkNACnadauWoM7zmcKmWrFFJkxhhzbnDVKpvlKhdsCYY0IsLQy4YW\nQkTGGHNucVVPv0ylgi22ZowxJmcBJX0R6Sci0SKyWUSydaFFpJqIzBCRtSKyXERaZ9hWVUS+EZGN\nIrJBRP7i7zplKhZOT98YY4xvuSZ9EQkBPgD6Am2AQSJyYZbdXgSiVLU9cC/wXoZtY4C5qtoKaA9s\n9Het0hUKttiaMcaYnAXS0+8MbFHVnaqaDEwDrs+yT2tgMYCqbgKaiEhtEakCXK6q473bUlT1hL8L\nlSpvPX1jjClKgST9BsDuDI/3eNsyWgsMBBCRzkAjIAw4H4gVkfEislpExopIeX8XKlXeavrGGFOU\nCmv0znBgjIisBtYDUUAqUBroCDyhqv8TkdHAMOA1XyfZtuYLSh3aScSfEYSHhxMeHl5I4RljTMkX\nGRlJZGRkgc4hqprzDiJdgAhV7ed9PAxQVR2RwzHbgbZARWCZqjb1tl8GDFXVa30co/2HfkWNzj8y\naeCkfD8hY4wJFiKCqkpejgmkvLMSuEBEGotIGeB2YFaWC1cVkdLe7wcDS1Q1TlVjgN0i0sK7a0/g\nD79PoGzBl2AwxhjjX67lHVVNFZEhwHycN4lxqrpRRB5xNutYoBUwUUQ8wAbgwQyneAqY7H1T2Abc\n7/dipQu+BIMxxhj/Aqrpq+qPQMssbZ9m+H551u0Ztq0FOgUUTWnr6RtjTFFy1YxcLVXwZZWNMcb4\n56qknxpqPX1jjClKrkr6nhCr6RtjTFFyVdJPFuvpG2NMUXJV0k8Rq+kbY0xRclXST8Z6+sYYU5Rc\nlfRPYzV9Y4wpSq5K+qc8tuCaMcYUJVcl/aRUK+8YY0xRclnSt/KOMcYUJVcl/YQU6+kbY0xRclfS\nT7Yhm8YYU5RclfQ96qF0SOniDsMYY85Zrkr6FctURCRPnwdgjDEmD1yV9K2eb4wxRctVSd9G7hhj\nTNFyVdK3nr4xxhQtVyV9G7ljjDFFy1VJ33r6xhhTtFyV9K2mb4wxRctVSd96+sYYU7RclfStp2+M\nMUXLVUnfevrGGFO0Akr6ItJPRKJFZLOIDPWxvZqIzBCRtSKyXERaZ9i2w9seJSIrcrqOjd4xxpii\nVSq3HUQkBPgA6AnsA1aKyPeqGp1htxeBKFUdKCItgQ+BXt5tHiBcVY/mdi3r6RtjTNEKpKffGdii\nqjtVNRmYBlyfZZ/WwGIAVd0ENBGR2t5tEuB1rKZvjDFFLJBk3ADYneHxHm9bRmuBgQAi0hloBIR5\ntymwQERWisjgnC5k5R1jjClauZZ3AjQcGCMiq4H1QBSQ6t3WXVX3e3v+C0Rko6ou9XWSeZ/P48Cc\nAwCEh4cTHh5eSOEZY0zJFxkZSWRkZIHOIaqa8w4iXYAIVe3nfTwMUFUdkcMx24G2qhqXpf014KSq\nvuPjGJ2+YTo3tb4pH0/DGGOCj4igqnlajz6Q8s5K4AIRaSwiZYDbgVlZLlxVREp7vx8MLFHVOBGp\nICKVvO0VgT7A7/4uZDdyjTGmaOVa3lHVVBEZAszHeZMYp6obReQRZ7OOBVoBE0XEA2wAHvQeXgf4\nTkTUe63Jqjrf37Wspm+MMUUr1/LO2SIiunLvSi6tf2lxh2KMMSVCUZV3zhobsmmMMUXLVUnfavrG\nGFO0XJX0raZvjDFFy1VJ33r6xhhTtFyV9MuXKl/cIRhjzDnNVUlfJE83oY0xxuSRq5K+McaYomVJ\n3xhjgoglfWOMCSKW9I0xJohY0jfGmCBiSd8YY4KIJX1jjAkilvSNMSaIWNI3xpggYknfGGOCiCV9\nY4wJIpb0jTEmiFjSN8aYIGJJ3xhjgoglfWOMCSKW9I0xJohY0jfGmCASUNIXkX4iEi0im0VkqI/t\n1URkhoisFZHlItI6y/YQEVktIrMKK3BjjDF5l2vSF5EQ4AOgL9AGGCQiF2bZ7UUgSlXbA/cC72XZ\n/jTwR8HDNcYYUxCB9PQ7A1tUdaeqJgPTgOuz7NMaWAygqpuAJiJSG0BEwoCrgc8LLWpjjDH5EkjS\nbwDszvB4j7cto7XAQAAR6Qw0AsK8294Fnge0QJEaY4wpsFKFdJ7hwBgRWQ2sB6KAVBG5BohR1TUi\nEg5ITieJiIhI/z48PJzw8PBCCs8YY0q+yMhIIiMjC3QOUc25Ay4iXYAIVe3nfTwMUFUdkcMx24B2\nOLX+u4AUoDxQGZihqvf4OEZzi8UYY8wZIoKq5tiZznZMAEk/FNgE9AT2AyuAQaq6McM+VYEEVU0W\nkcFAd1W9L8t5rgCeU9Xr/FzHkr4xxuRBfpJ+ruUdVU0VkSHAfJx7AONUdaOIPOJs1rFAK2CiiHiA\nDcCDeQ/fGGNMUcu1p3+2WE/fGGPyJj89fZuRa4wxQcSSvjHGBBFL+sYYE0Qs6RtjTBCxpG+MMUHE\nkr4xxgQRS/rGGBNELOkbY0wQsaRvjDFBxJK+McYEEUv6xhgTRCzpG2NMELGkb4wxQcSSvjHGBBFL\n+sYYE0Qs6RtjTBCxpG+MMUHEkr4xxgQRS/rGGBNELOkbY0wQsaRvjDFBxJK+McYEkYCSvoj0E5Fo\nEdksIkN9bK8mIjNEZK2ILBeR1t72siLym4hEich6EXmtsJ+AMcaYwOWa9EUkBPgA6Au0AQaJyIVZ\ndnsRiFLV9sC9wHsAqnoKuFJVLwY6AP1FpHMhxn9WRUZGFncIAbE4C5fFWbgszuIVSE+/M7BFVXeq\najIwDbg+yz6tgcUAqroJaCIitb2PE7z7lAVKAVoYgReHkvJLYHEWLouzcFmcxSuQpN8A2J3h8R5v\nW0ZrgYEA3p58IyDM+zhERKKAA8ACVV1Z0KCNMcbkT2HdyB0OVBeR1cATQBSQCqCqHm95Jwz4S1q9\n3xhjzNknqjlXW0SkCxChqv28j4cBqqojcjhmO9BWVeOytL8CxKvqOz6OKbFlH2OMKS6qKnnZv1QA\n+6wELhCRxsB+4HZgUMYdRKQqkKCqySIyGFiiqnEiUgtIVtXjIlIe6I3zV0GBAzfGGJN3uSZ9VU0V\nkSHAfJxy0DhV3SgijzibdSzQCpgoIh5gA/Cg9/B63vYQ77H/UdW5RfFEjDHG5C7X8o4xxphzR7HP\nyM1t4ldxEZFxIhIjIusytFUXkfkisklE5nnLWsVKRMJEZLGIbPBOgHvKbbH6m6Tnphgz8o44Wy0i\ns7yPXReniOzwToaMEpEVLo6zqoh8IyIbvb+jf3FbnCLSwvs6rvb+e1xEnnJbnN5YnxWR30VknYhM\nFpEyeY2zWJN+gBO/ist4nLgyGgYsVNWWOPMSXjjrUWWXAvxVVdsAXYEnvK+ha2LNYZKea2LM4mng\njwyP3RinBwhX1YtVNW3CoxvjHAPMVdVWQHsgGpfFqaqbva9jR+ASIB74DpfFKSL1gSeBjqraDqc8\nP4i8xqmqxfYFdAH+L8PjYcDQ4owpS3yNgXUZHkcDdbzf1wWiiztGHzHPBHq5NVagAvA/oJMbY8QZ\nWrwACAdmufXnDmwHamZpc1WcQBVgq492V8WZJbY+wM9ujBOoD+wEqnsT/qz8/F8v7vJOIBO/3OQ8\nVY0BUNUDwHnFHE8mItIEpye9HOeXwDWx+pmk56oYvd4FnifzzHE3xqnAAhFZKSIPedvcFuf5QKyI\njPeWTsaKSAXcF2dGtwFTvN+7Kk5V3QeMAnYBe4HjqrqQPMZZ3Em/pHPNXXARqQRMB55WZ35E1tiK\nNVbNPEmvs4i08RFTscYoItcAMaq6BshpCLEbfu7d1SlHXI1T0rscl72eOL3RjsCH3ljjcf6ad1uc\nAIhIaeA64Btvk6viFJFqOEvgNMbp9VcUkTt9xJVjnMWd9PfiLNmQJszb5lYxIlIHQETqAgeLOR4A\nRKQUTsL/SlW/9za7MlZVPQFEAv1wX4zdgetEZBswFbhKRL4CDrgsTlR1v/ffQzglvc647/XcA+xW\n1f95H3+L8ybgtjjT9AdWqWqs97Hb4uwFbFPVI6qainPfoRt5jLO4k376xC8RKYMz8WtWMceUkZC5\nxzcLuM/7/b3A91kPKCZfAH+o6pgMba6JVURqpY0okDOT9DbiohgBVPVFVW2kqk1xfhcXq+rdwGxc\nFKeIVPD+ZYeIVMSpQ6/Hfa9nDLBbRFp4m3rizONxVZwZDMJ5s0/jtjh3AV1EpJyICM7r+Qd5jdMF\nN076AZuALcCw4o4nQ1xTgH3AKe+LfT/ODZSF3njnA9VcEGd3nHWO1uCsebTa+5rWcEusQFtvXGuA\ndcBL3nbXxOgj5is4cyPXVXHi1MrTft7r0/7fuC1Ob0ztcTp3a4AZQFWXxlkBOARUztDmxjhfw+kw\nrQMmAqXzGqdNzjLGmCBS3OUdY4wxZ5ElfWOMCSKW9I0xJohY0jfGmCBiSd8YY4KIJX1jjAkilvSN\nMSaIWNI3xpgg8v+Go9zz57Cs5wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11d7f6290>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(train_accuracies, color='blue')\n",
    "plt.plot(test_accuracies, color='green')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:39:57.756392",
     "start_time": "2016-06-13T11:39:57.752987"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.98540002"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(test_accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number errors is 146"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-13T11:40:34.667306",
     "start_time": "2016-06-13T11:40:34.663352"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "145.99978923797607"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "10000*(1.0-np.max(test_accuracies))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}