2BwD-Layer-DK-AlignMNiST-3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-04-20T21:52:04.548207",
     "start_time": "2016-04-20T21:52:04.544771"
    }
   },
   "source": [
    "## 2 Layer MLP for Dark Knoweldge Experiments w/AlignMNIST\n",
    "\n",
    "75 epochs of RMSProp on AlignMNIST\n",
    "\n",
    "Bigger network with dropout yet, and trying to add some kind of weight constraint / max norm \n",
    "\n",
    "for now, looking at norms on each layer\n",
    "\n",
    "\n",
    "see  http://www.r2rt.com/posts/implementations/2016-03-29-implementing-batch-normalization-tensorflow/\n",
    "\n",
    "https://github.com/tensorflow/tensorflow/blob/b3621c95160a916d4d255f9f44318b9d465701af/tensorflow/contrib/layers/python/layers/layers.py\n",
    "\n",
    "\n",
    "https://www.reddit.com/r/MachineLearning/comments/2bopxs/question_about_the_maxnorm_constraint_used_with/\n",
    "\n",
    "\n",
    "\n",
    "https://arxiv.org/pdf/1207.0580.pdf\n",
    "\n",
    "https://arxiv.org/pdf/1503.02531v1.pdf\n",
    "\n",
    "http://www.kdnuggets.com/2015/04/preventing-overfitting-neural-networks.html\n",
    "\n",
    "https://www.reddit.com/r/MachineLearning/comments/2bopxs/question_about_the_maxnorm_constraint_used_with/\n",
    "\n",
    "and https://github.com/tensorflow/tensorflow/issues/608"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:48.245739",
     "start_time": "2016-06-14T14:41:46.031889"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:48.870518",
     "start_time": "2016-06-14T14:41:48.865396"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%run augmentmnist.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:49.595312",
     "start_time": "2016-06-14T14:41:49.592438"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 75\n",
    "batch_size = 125\n",
    "\n",
    "learning_rate = 0.001\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:50.617425",
     "start_time": "2016-06-14T14:41:50.612300"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Network Parameters\n",
    "n_hidden_1 = 1200 # 1st layer num features\n",
    "n_hidden_2 = 1200 # 2nd layer num features\n",
    "n_input = 784 # MNIST data input (img shape: 28*28)\n",
    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
    "\n",
    "std_0  = np.sqrt(6.0/(n_input+n_hidden_1))\n",
    "std_h1 = np.sqrt(6.0/(n_hidden_1+n_hidden_2))\n",
    "std_h2 = np.sqrt(6.0/(n_hidden_2+n_classes))\n",
    "\n",
    "logfile = \"2BwD-layer-dlk-alignmnist-3\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:51.485964",
     "start_time": "2016-06-14T14:41:51.479127"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# tf Graph input\n",
    "x = tf.placeholder(\"float\", [None, n_input])\n",
    "y = tf.placeholder(\"float\", [None, n_classes])\n",
    "\n",
    "xt = tf.placeholder(\"float\", [None, n_input])\n",
    "yt = tf.placeholder(\"float\", [None, n_classes])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:52.143697",
     "start_time": "2016-06-14T14:41:52.139324"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def multilayer_perceptron(_X, _weights, _biases):\n",
    "    layer_1 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(_X, _weights['h1']), _biases['b1'])),  0.2)\n",
    "    layer_2 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(layer_1, _weights['h2']), _biases['b2'])), 0.5)    \n",
    "    return tf.matmul(layer_2, weights['out']) + biases['out']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:53.006744",
     "start_time": "2016-06-14T14:41:52.971178"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Store layers weight & bias\n",
    "weights = {\n",
    "    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1], stddev=std_0)),\n",
    "    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2], stddev=std_h1)),\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes], stddev=std_h2))\n",
    "}\n",
    "biases = {\n",
    "    'b1': tf.Variable(tf.random_normal([n_hidden_1], stddev=0.1)),\n",
    "    'b2': tf.Variable(tf.random_normal([n_hidden_2], stddev=0.01)),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes], stddev=0.001))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:53.845347",
     "start_time": "2016-06-14T14:41:53.805116"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "mlp = multilayer_perceptron(x, weights, biases )\n",
    "mlp_test = multilayer_perceptron(xt, weights, biases  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:54.530086",
     "start_time": "2016-06-14T14:41:54.517890"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cost =  tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(mlp, y)) \n",
    "gs = tf.get_variable(\"global_step\",[],trainable=False,initializer=tf.constant_initializer(0))\n",
    "lr = tf.constant(learning_rate) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:55.337943",
     "start_time": "2016-06-14T14:41:55.201247"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "train_op = tf.contrib.layers.optimize_loss(cost, global_step=gs, learning_rate=lr,optimizer=\"RMSProp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:55.941728",
     "start_time": "2016-06-14T14:41:55.923963"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'ScalarSummary_2:0' shape=() dtype=string>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#with tf.name_scope(\"training accuracy\"):\n",
    "pred = tf.equal(tf.argmax(mlp, 1), tf.argmax(y, 1)) # Count correct predictions\n",
    "train_acc_op = tf.reduce_mean(tf.cast(pred, \"float\"))  # Cast boolean to float to average\n",
    "tf.scalar_summary(\"training accuracy\", train_acc_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:56.664747",
     "start_time": "2016-06-14T14:41:56.654137"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'ScalarSummary_3:0' shape=() dtype=string>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pred = tf.equal(tf.argmax(mlp_test, 1), tf.argmax(yt, 1)) # Count correct predictions\n",
    "test_acc_op = tf.reduce_mean(tf.cast(test_pred, \"float\"))  # Cast boolean to float to average\n",
    "tf.scalar_summary(\"test 0 accuracy\", test_acc_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:41:57.684406",
     "start_time": "2016-06-14T14:41:57.437300"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[36m2-layer-dk-alignmnist\u001b[m\u001b[m           \u001b[1m\u001b[36m2-layer-modern-mlp-alignmnist\u001b[m\u001b[m\r\n",
      "\u001b[1m\u001b[36m2-layer-dlk-alignmnist\u001b[m\u001b[m          \u001b[1m\u001b[36m2BwD-layer-dlk-alignmnist\u001b[m\u001b[m\r\n",
      "\u001b[1m\u001b[36m2-layer-mlp-alignmnist-wdropout\u001b[m\u001b[m \u001b[1m\u001b[36m5-layer-mlp-alignmnist-wdropout\u001b[m\u001b[m\r\n",
      "\u001b[1m\u001b[36m2-layer-mlp-mnist-temp\u001b[m\u001b[m          \u001b[1m\u001b[36m5-layer-mlp-infimnist\u001b[m\u001b[m\r\n",
      "\u001b[1m\u001b[36m2-layer-mlp-mnist-watch\u001b[m\u001b[m\r\n"
     ]
    }
   ],
   "source": [
    "!rm -rf ./logs/{logfile}\n",
    "!ls logs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T21:57:37.534602",
     "start_time": "2016-05-04T21:57:37.532739"
    },
    "collapsed": true
   },
   "source": [
    "### Original MNIST Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:42:03.305706",
     "start_time": "2016-06-14T14:42:02.443355"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\".\", one_hot=True)\n",
    "trX_0, trY_0 = mnist.train.images, mnist.train.labels\n",
    "teX_0, teY_0 = mnist.test.images, mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T14:42:04.415110",
     "start_time": "2016-06-14T14:42:04.411826"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "alignmnist = AlignMNIST()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T15:44:52.046565",
     "start_time": "2016-06-14T14:49:36.210726"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 0.81511664, 0.88749999)\n",
      "[54.569935, 62.339447, 6.9963851, 3.448611, 1.0789102, 0.082129896]\n",
      "(1, 0.86266667, 0.91829997)\n",
      "[56.05917, 65.244644, 6.5838289, 3.5065219, 2.1609182, 0.17845699]\n",
      "(2, 0.88026667, 0.92940003)\n",
      "[57.390232, 67.908058, 6.3219166, 3.5616927, 3.3341305, 0.27384359]\n",
      "(3, 0.89233333, 0.9386)\n",
      "[58.672428, 70.474121, 6.2681961, 3.6408212, 4.4861813, 0.36348552]\n",
      "(4, 0.89481664, 0.94099998)\n",
      "[59.879402, 72.952095, 6.3676305, 3.7453721, 5.5518999, 0.45345449]\n",
      "(5, 0.90166664, 0.94340003)\n",
      "[61.056767, 75.36821, 6.6044264, 3.8553052, 6.585577, 0.55994248]\n",
      "(6, 0.90773332, 0.95039999)\n",
      "[62.184849, 77.724831, 6.9394088, 3.992049, 7.5731235, 0.64984828]\n",
      "(7, 0.90786666, 0.94679999)\n",
      "[63.217308, 79.973152, 7.2663913, 4.1529312, 8.5597239, 0.75256306]\n",
      "(8, 0.90481669, 0.94709998)\n",
      "[64.219734, 82.17511, 7.6462321, 4.2979636, 9.5112324, 0.8630808]\n",
      "(9, 0.91521668, 0.95020002)\n",
      "[65.212883, 84.351112, 8.066247, 4.4625301, 10.426193, 0.94609499]\n",
      "(10, 0.90969998, 0.94870001)\n",
      "[66.108734, 86.468483, 8.4950523, 4.6553831, 11.355545, 1.048309]\n",
      "(11, 0.91565001, 0.94940001)\n",
      "[66.986855, 88.53553, 8.9402294, 4.8467298, 12.236042, 1.1433825]\n",
      "(12, 0.91455001, 0.95169997)\n",
      "[67.857887, 90.538864, 9.4048405, 5.0487452, 13.10191, 1.2374161]\n",
      "(13, 0.91781664, 0.95249999)\n",
      "[68.686852, 92.498749, 9.8125896, 5.2619033, 13.965233, 1.3266867]\n",
      "(14, 0.91971666, 0.9526)\n",
      "[69.505959, 94.384567, 10.328677, 5.4711604, 14.786835, 1.3974668]\n",
      "(15, 0.91638333, 0.94840002)\n",
      "[70.303291, 96.206253, 10.802207, 5.6913486, 15.6172, 1.4887288]\n",
      "(16, 0.91686666, 0.95429999)\n",
      "[71.070358, 97.976143, 11.285682, 5.9213619, 16.401409, 1.5643139]\n",
      "(17, 0.91918331, 0.9558)\n",
      "[71.822533, 99.679901, 11.714366, 6.1380196, 17.23484, 1.6405612]\n",
      "(18, 0.91869998, 0.95490003)\n",
      "[72.552368, 101.39856, 12.199872, 6.3537712, 18.028183, 1.7167299]\n",
      "(19, 0.91659999, 0.95300001)\n",
      "[73.290352, 103.07032, 12.666897, 6.5932903, 18.82575, 1.7845236]\n",
      "(20, 0.92036664, 0.95270002)\n",
      "[74.001213, 104.75574, 13.135871, 6.8153305, 19.611986, 1.8425934]\n",
      "(21, 0.92145002, 0.95730001)\n",
      "[74.697639, 106.32642, 13.637591, 7.0372286, 20.403564, 1.9064153]\n",
      "(22, 0.92321664, 0.95429999)\n",
      "[75.378006, 107.86785, 14.172114, 7.2620792, 21.141459, 1.966068]\n",
      "(23, 0.92068332, 0.95370001)\n",
      "[76.050995, 109.35846, 14.682657, 7.4923372, 21.912424, 2.0259686]\n",
      "(24, 0.91976666, 0.95209998)\n",
      "[76.682556, 110.82507, 15.121241, 7.7200146, 22.684597, 2.0832262]\n",
      "(25, 0.92063332, 0.95429999)\n",
      "[77.31221, 112.28627, 15.708577, 7.9551039, 23.380959, 2.1401856]\n",
      "(26, 0.92108333, 0.95179999)\n",
      "[77.944313, 113.69922, 16.182251, 8.1923094, 24.09444, 2.1669662]\n",
      "(27, 0.92463332, 0.95569998)\n",
      "[78.581642, 115.12203, 16.716074, 8.4065399, 24.813454, 2.205497]\n",
      "(28, 0.92186666, 0.95859998)\n",
      "[79.208786, 116.45968, 17.25577, 8.6439762, 25.522203, 2.2470915]\n",
      "(29, 0.91773331, 0.95469999)\n",
      "[79.788254, 117.77435, 17.752979, 8.8780413, 26.267006, 2.2789381]\n",
      "(30, 0.92281669, 0.95660001)\n",
      "[80.377579, 119.1104, 18.281847, 9.1166401, 26.969416, 2.3005657]\n",
      "(31, 0.92533332, 0.95899999)\n",
      "[80.972397, 120.42966, 18.792698, 9.3567734, 27.656218, 2.3509064]\n",
      "(32, 0.92079997, 0.95770001)\n",
      "[81.568573, 121.71946, 19.355625, 9.6017313, 28.366259, 2.3865311]\n",
      "(33, 0.92696667, 0.9562)\n",
      "[82.144722, 122.98081, 19.860567, 9.8347454, 29.066902, 2.4213989]\n",
      "(34, 0.92528331, 0.95670003)\n",
      "[82.716789, 124.28417, 20.398714, 10.081834, 29.769274, 2.4541461]\n",
      "(35, 0.9242, 0.95840001)\n",
      "[83.29187, 125.52774, 20.938486, 10.333005, 30.429003, 2.4942305]\n",
      "(36, 0.92398334, 0.95700002)\n",
      "[83.85611, 126.76325, 21.50816, 10.572257, 31.145134, 2.5327387]\n",
      "(37, 0.9242667, 0.95660001)\n",
      "[84.415726, 127.98941, 22.066324, 10.802129, 31.830999, 2.5595288]\n",
      "(38, 0.92016667, 0.95340002)\n",
      "[84.962296, 129.21895, 22.568316, 11.054328, 32.541031, 2.5931988]\n",
      "(39, 0.92423332, 0.95469999)\n",
      "[85.545006, 130.39685, 23.130333, 11.281258, 33.206898, 2.6241546]\n",
      "(40, 0.92366666, 0.95609999)\n",
      "[86.103355, 131.61763, 23.695093, 11.510935, 33.867378, 2.6653423]\n",
      "(41, 0.92455, 0.95789999)\n",
      "[86.677643, 132.74893, 24.245455, 11.743328, 34.524715, 2.675276]\n",
      "(42, 0.92451668, 0.95410001)\n",
      "[87.212524, 133.87428, 24.741199, 11.987894, 35.209175, 2.6949975]\n",
      "(43, 0.92116666, 0.95700002)\n",
      "[87.735733, 134.98067, 25.300257, 12.224077, 35.870392, 2.7258332]\n",
      "(44, 0.91930002, 0.95389998)\n",
      "[88.229958, 136.06815, 25.855301, 12.467381, 36.522182, 2.7486534]\n",
      "(45, 0.92661667, 0.9601)\n",
      "[88.78569, 137.16595, 26.417099, 12.695825, 37.136501, 2.7862618]\n",
      "(46, 0.92290002, 0.96030003)\n",
      "[89.328468, 138.25919, 26.953682, 12.943019, 37.784039, 2.8068116]\n",
      "(47, 0.92258334, 0.95670003)\n",
      "[89.848701, 139.3046, 27.514395, 13.167522, 38.41724, 2.8303139]\n",
      "(48, 0.92535001, 0.95529997)\n",
      "[90.386269, 140.29968, 28.025305, 13.397225, 39.066326, 2.8490038]\n",
      "(49, 0.92733335, 0.95929998)\n",
      "[90.87384, 141.35033, 28.578241, 13.616638, 39.702904, 2.8660417]\n",
      "(50, 0.9181, 0.95169997)\n",
      "[91.376289, 142.37071, 29.122089, 13.864511, 40.302883, 2.8894296]\n",
      "(51, 0.92605001, 0.958)\n",
      "[91.911751, 143.37984, 29.684347, 14.074899, 40.931458, 2.9108198]\n",
      "(52, 0.92473334, 0.95719999)\n",
      "[92.426727, 144.40573, 30.268461, 14.29416, 41.560516, 2.9132752]\n",
      "(53, 0.92326665, 0.9569)\n",
      "[92.953789, 145.39934, 30.81588, 14.534431, 42.19482, 2.9249189]\n",
      "(54, 0.92486668, 0.9576)\n",
      "[93.479431, 146.41206, 31.327526, 14.759096, 42.837517, 2.9569569]\n",
      "(55, 0.92580003, 0.95819998)\n",
      "[93.971565, 147.38495, 31.865746, 14.99767, 43.452827, 2.9678311]\n",
      "(56, 0.92601669, 0.9576)\n",
      "[94.493385, 148.41061, 32.397614, 15.213389, 44.056259, 2.9877491]\n",
      "(57, 0.92553335, 0.9612)\n",
      "[95.002487, 149.3727, 32.938843, 15.453576, 44.685135, 3.0112486]\n",
      "(58, 0.92676669, 0.95730001)\n",
      "[95.494804, 150.34189, 33.507671, 15.66866, 45.270321, 3.0443487]\n",
      "(59, 0.92483336, 0.95889997)\n",
      "[95.994713, 151.30365, 34.022457, 15.900341, 45.885952, 3.0500152]\n",
      "(60, 0.92734998, 0.95889997)\n",
      "[96.510216, 152.2558, 34.533279, 16.127514, 46.480461, 3.070816]\n",
      "(61, 0.92449999, 0.95929998)\n",
      "[97.010284, 153.21498, 35.092983, 16.347609, 47.049248, 3.0776284]\n",
      "(62, 0.92704999, 0.95929998)\n",
      "[97.500984, 154.15291, 35.636024, 16.564795, 47.624714, 3.0893929]\n",
      "(63, 0.92401665, 0.95639998)\n",
      "[97.975998, 155.07315, 36.164299, 16.788696, 48.209236, 3.1050465]\n",
      "(64, 0.92491668, 0.95480001)\n",
      "[98.449158, 156.00024, 36.70631, 17.023319, 48.778286, 3.1162803]\n",
      "(65, 0.92206669, 0.95730001)\n",
      "[98.993713, 156.89951, 37.271343, 17.239206, 49.361362, 3.1435463]\n",
      "(66, 0.92813331, 0.96030003)\n",
      "[99.490044, 157.7643, 37.833462, 17.461254, 49.926441, 3.152992]\n",
      "(67, 0.92518336, 0.9619)\n",
      "[99.968407, 158.64619, 38.303909, 17.691927, 50.516487, 3.1624649]\n",
      "(68, 0.92421669, 0.95539999)\n",
      "[100.49645, 159.50598, 38.828247, 17.920877, 51.081608, 3.1750205]\n",
      "(69, 0.92381668, 0.95679998)\n",
      "[100.9708, 160.39659, 39.324093, 18.117662, 51.655357, 3.1892529]\n",
      "(70, 0.92334998, 0.95539999)\n",
      "[101.4595, 161.2692, 39.861073, 18.326294, 52.21608, 3.2092633]\n",
      "(71, 0.92224997, 0.95520002)\n",
      "[101.94269, 162.10997, 40.400974, 18.550079, 52.739338, 3.2152977]\n",
      "(72, 0.92578334, 0.95639998)\n",
      "[102.45666, 162.9534, 40.922997, 18.756783, 53.30151, 3.2270765]\n",
      "(73, nan, 0.95829999)\n",
      "[102.45666, 162.9534, 40.922997, 18.756783, 53.30151, 3.2270765]\n",
      "(74, nan, 0.9594)\n",
      "[102.45666, 162.9534, 40.922997, 18.756783, 53.30151, 3.2270765]\n"
     ]
    }
   ],
   "source": [
    "test_accuracies = []\n",
    "train_accuracies = []\n",
    "norms = []\n",
    "with tf.Session() as sess:\n",
    "    # create a log writer. run 'tensorboard --logdir=./logs/{logfile}'\n",
    "    writer = tf.train.SummaryWriter(\"./logs/{0}\".format(logfile), sess.graph) # for 0.8\n",
    "    merged = tf.merge_all_summaries()\n",
    " \n",
    "    tf.initialize_all_variables().run()\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        trX, trY = alignmnist.next_epoch()\n",
    "      \n",
    "        for start, end in zip(range(0, len(trX), batch_size), range(batch_size, len(trX), batch_size)):\n",
    "            sess.run(train_op, feed_dict={x: trX[start:end], y: trY[start:end]})\n",
    "                 \n",
    "        summary, trn_acc, tst_acc = sess.run([merged, train_acc_op, test_acc_op], feed_dict={x: trX, y: trY, xt: teX_0, yt: teY_0})\n",
    "        writer.add_summary(summary, epoch)  \n",
    "        \n",
    "        \n",
    "        print(epoch, trn_acc, tst_acc)\n",
    "        train_accuracies.append(trn_acc)\n",
    "        test_accuracies.append(tst_acc)\n",
    "\n",
    "        nrms= [np.linalg.norm(weights['h1'].eval()),np.linalg.norm(weights['h2'].eval()), np.linalg.norm(weights['out'].eval()), np.linalg.norm(biases['b1'].eval()),np.linalg.norm(biases['b2'].eval()),np.linalg.norm(biases['out'].eval())]\n",
    "        print nrms\n",
    "        norms.append(nrms)\n",
    "        writer.flush()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:31:51.327745",
     "start_time": "2016-06-14T16:31:48.392102"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/charlesmartin14/anaconda2/lib/python2.7/site-packages/matplotlib/font_manager.py:273: UserWarning: Matplotlib is building the font cache using fc-list. This may take a moment.\n",
      "  warnings.warn('Matplotlib is building the font cache using fc-list. This may take a moment.')\n"
     ]
    }
   ],
   "source": [
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:31:51.529806",
     "start_time": "2016-06-14T16:31:51.329495"
    },
    "collapsed": false
   },
   "source": [
    "plt.plot(train_accuracies)\n",
    "plt.plot(test_accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:31:51.535330",
     "start_time": "2016-06-14T16:31:51.531173"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9619"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.max(test_accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number test errors is"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:32:07.217220",
     "start_time": "2016-06-14T16:32:07.212804"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "381.00004196166992"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "10000*(1.0-np.max(test_accuracies))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:32:40.544667",
     "start_time": "2016-06-14T16:32:40.541309"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[54.569935, 62.339447, 6.9963851, 3.448611, 1.0789102, 0.082129896]\n",
      "[102.45666, 162.9534, 40.922997, 18.756783, 53.30151, 3.2270765]\n"
     ]
    }
   ],
   "source": [
    "print norms[0]\n",
    "print norms[-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:33:49.528886",
     "start_time": "2016-06-14T16:33:49.525098"
    }
   },
   "source": [
    "#### What is the best way to implement the cutoff ?\n",
    "\n",
    "by the norm of the weights at each layer ?\n",
    "\n",
    "see also:  http://keras.io/constraints/\n",
    "\n",
    "Does this just reflect the loss decreasing ?\n",
    "\n",
    "Can we simply bound the norm?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:57:05.122672",
     "start_time": "2016-06-14T16:57:04.910441"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEACAYAAAC9Gb03AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XeclFWe7/HP6UTniIBIbOhumpxVRCkRJCnqqKhrhtHZ\ncZ114o46Oxfujt5RZ3dnnHXmeh3DGGAUZBxBBQGhDEjOdEM3OUnuHKur6tw/Tj1d1QFoOlXo3/v1\nel5V9VT6dQtfD7/nPOdRWmuEEEKErjB/FyCEEKJtSdALIUSIk6AXQogQJ0EvhBAhToJeCCFCnAS9\nEEKEuEsGvVLqDaXUaaXUTp99w5RS65RS25RSG5VSo32ee0YptU8ptUcpdXNbFS6EEKJpmjKifwuY\nUm/fS8BcrfUIYC7wOwCl1EBgFpANTAP+rJRSrVeuEEKIy3XJoNdafwMU1tvtBpI895OBE577M4H3\ntdZOrfVhYB8wtnVKFUII0RwRzXzfT4DPlVL/BShgnGf/VcA6n9ed8OwTQgjhJ809GPtD4CmtdS9M\n6L/ZeiUJIYRoTc0d0T+stX4KQGv9oVLqdc/+E0BPn9f1wNvWqUMpJYvsCCFEM2itL+vYZ1NH9Mqz\nWU4opSYAKKVuwvTiAZYA9yqlopRSfYH+wMaLFBvw29y5c/1eg9QpdQZzncFQYzDV2RyXHNErpRYA\nNiBNKXUUM8vmMeCPSqlwoAp43BPcuUqphUAuUAM8oZtbmRBCiFZxyaDXWv/TBZ4a3dhOrfVvgd+2\npCghhBCtR86MvQSbzebvEppE6mxdUmfrCYYaIXjqbA7lr86KUkq6OkIIcZmUUug2OhgrhBAiSEnQ\nCyFEiJOgF0KIECdBL4QQIU6CXgghQpwEvRBChDgJeiGECHES9EIIEeIk6IUQIsRJ0AshRIiToBdC\niBAnQS+EECFOgl4IIUKcBL0QQoS4plxh6g3gFuC01nqoz/4fAU8ATuBTrfXTnv3PALM9+5/SWq9o\ni8KFEKKtVTmr2HB8A18e+ZK1x9ZSUVNx0dc/NPQhHhv1WDtV13RNuTj4W8D/AO9YO5RSNuBWYIjW\n2qmU6uzZnw3MArIxFwZfpZTKkIXnhRDBwAp2+2E79iN2Np3YxMArBmLrY+PJMU+SHJ180ff3TOrZ\nTpVenqZcSvAbpVTvert/CLygtXZ6XnPOs/824H3P/sNKqX3AWGBDK9YshBCtwuFysPHERlYfWs3q\nQ6vZ/N1mBnUZxITeE/j5tT/n+t7Xk9gp0d9ltlhTRvSNyQRuUEr9H6AS+LnWegtwFbDO53UnPPuE\nEMLv3NrNjlM7WHVwFasPr2bt0bVkpGUwsc9E/u26f2N8r/EhEez1NTfoI4AUrfU1SqkxwCIg/XI/\nZN68ebX3bTZbSF+zUQjR/rTW5J/PZ83hNbWj9s6xnZmUPonHRj7G/O/NJzUm1d9lXpTdbsdut7fo\nM5p0zVhP62apdTBWKfUZ8KLW+kvP433ANcBjAFrrFzz7lwNztdYNWjdyzVghRFs4XHS4NtTXHF5D\nmArjxj43MrHvRCalT6JHYg9/l9gizblmbFNH9MqzWf4BTAS+VEplAlFa6/NKqSXAfKXUf2NaNv2B\njZdTkBBCXI4z5WdYc2gNXxz6gi8OfUGZo4yJfScysc9E5tnm0S+lH0pdVi6GnKZMr1wA2IA0pdRR\nYC7wJvCWUmoXUA08BKC1zlVKLQRygRrgCRm2CyFaU1FVEfbD9toR+7HiY1zf+3pu6nsTPxr7IwZ3\nGdzhg72+JrVu2uSLpXUjhGgCh8vBumPrWHlwJasOriL3bC7X9ryWiX0mMrHvREZcOYKIsOYebgw+\nzWndSNALIQKK1pq883msOLCCFQdW8NWRrxjQeQCT0yczKX0S1/a8luiIaH+X6TcS9EKIoHS+4jxf\nHPqCFQdWsPLgStzazZR+U7i5383c1Pcm0mLT/F1iwJCgF0IEBafbyfrj6/l8/+d8fuBz8s7ncX2v\n67m5383c3O9mstKypM9+ARL0QoiAdbzkOMv2LWP5geWsPrSavsl9mdJvClP6T2Fcz3FEhUf5u8Sg\nIEEvhAgY1c5q1h5by/L9y1m2fxknS09yc7+bmdZ/Gjf3u5mu8V39XWJQkqAXQvjVocJDLN+/nOUH\nlmM/bCe7czZT+k1hWsY0xnQfQ3hYuL9LDHoS9EKIdlXlrOKrI1+xbN8ylu1fRmFVIVP7T2Va/2lM\nTp8sB1HbgAS9EKLNHSo8xLL9Jti/PPwlQ7oOYVr/aUzrP40RV44gTMn1jNqSBL0QotVVO6vNqN0T\n7gWVBUztP5Xp/aczud/kgF8ULNRI0AshWsXJ0pN8kv8JS/OXYj9sZ3CXwWbUnjGNkVeOlFG7H0nQ\nCyGaRWvN9lPb+XTfpyzNX0r++Xym9p/KrZm3MrX/VBm1BxAJeiFEk5VUl7DywEo+2/cZy/YvIz4q\nnukZ07k181au7329zGsPUBL0QoiLOlR4iKX5S1mav5QNxzcwruc4pmdMZ1r/aWSkZfi7PNEEEvRC\niDpqXDV8e+xbPt33KZ/u+5RzFeeYkTGDWzNvZXK/ycRHxfu7RHGZJOiFEBRWFrJ8/3KW5C/h8/2f\n0zelL7dk3MKMzBmM7j5aDqQGOQl6ITqoAwUHalsym05swtbHxsysmUzPmE73hO7+Lk+0ojYJeqXU\nG8AtwGnrmrE+z/0M+B3QWWtd4Nn3DDAbcAJPaa1XXOBzJeiFaCan28m3x76tnQJZVFXEjIwZzMya\nyaT0ScRGxvq7RNFG2iroxwNlwDu+Qa+U6gG8DmQBo7TWBUqpbGABMAboAawCMhpLdAl6IS5PcVUx\nnx/4nCV5S1i2fxm9k3pza+at3JJ5C6O6j5KWTAfRJhcH11p/o5Tq3chTvwd+ASzx2Xcb8L7W2gkc\nVkrtA8YCGy6nKCGEcaLkBB/nfczHeR+z7tg6xvcaz8ysmbww6QV6JPbwd3kiSDTrQotKqZnAMa31\nrnoXB7gKWOfz+IRnnxCiCbTWbDu1jU/zzYlL+wv2MyNzBo+PfJzFsxbLLBnRLJcd9EqpGOBZYHJL\nv3zevHm19202GzabraUfKUTQqaypZOXBlSzNW8pn+z8jLjKOGRkz+O1Nv+WG3jcQGR7p7xKFH9nt\ndux2e4s+o0mzbjytm6Va66FKqcGY3nsFoDC9+BOYFs1sAK31C573LQfmaq0btG6kRy86snMV5/gk\n/xM+zvuY1YdWM/LKkczMnMmMzBlkpmX6uzwRwNpseqVSqg8m6Ic08twhYKTWulApNRCYD1yNadms\nRA7GCgGYhcI+2vsRi/csZvN3m5mUPonbs25nRuYMWUtGNFmbHIxVSi0AbECaUuooZoT+ls9LNGZk\nj9Y6Vym1EMgFaoAnJM1FR3a0+CiLcxezeM9ics7mMCNjBk+OeZIp/afIFEjRbuSEKSFa2f6C/bXh\nfrDwILdl3cadA+/kpr430Smik7/LE0FOzowVwg+01uw+s5u/7/k7f9/7d86Un+H2rNu5a+BdTOgz\ngYiwZk1uE6JREvRCtBNrGuSHuR+yeM9iqp3VfC/7e9yZfSfX9LhGLoIt2owEvRBtSGvNlpNbWJSz\niEW5iwhTYdw18C7uGngXo64cRb1zSoRoExL0QrQyrTVbT25lYc5CFuUuIjwsnFkDZ3HXwLsY3m24\nhLtodxL0QrQCK9wX5S5iYc5CwsPCuXvg3cwaNIthXYdJuAu/kqAXogX2F+znvZ3vMX/XfLTWzBo0\nS8JdBBwJeiEu09nysyzKXcS7O9/lYOFB7h10Lw8MfYDR3UdLuIuAJEEvRBOUVpfyj73/YMHuBaw7\nto5pGdN4cOiDTE6fLOvKiIAnQS/EBTjdTlYcWME7O95h2f5lTOg9gfsG38fMrJnERcX5uzwhmkyC\nXggf1lz3d3e8y992/430lHQeHPogswbNIi02zd/lCdEsbbLWjRDB5mjxUebvnM+7O9+l2lXNA0Me\n4OtHvyYjLcPfpQnhFzKiFyGhpLqExbmLeXfnu+w4vYO7B97Ng0MfZFzPcXJQVYQUad2IDsXpdvLF\nwS94Z+c7fJr/KbY+Nh4a9hAzMmbI4mEiZEnQiw5h1+ldvLPjHebvmk+PxB48POxh7hl8D51jO/u7\nNCHanPToRcgqrCxk/q75vLntTc5WnOXBoQ/yxUNfkH1Ftr9LEyLgyYheBCy3drPm0Bre2PYGn+37\njGkZ05g9fDYT+06U1SFFh9UmrRul1BvALcBprfVQz76XgFuBauAA8KjWusTz3DOYa8c6gae01isu\n8LkS9KJR+wv28/b2t3ln5zukRKcwZ8Qc7h96v1xuTwjaLujHA2XAOz5BPwlYrbV2K6VeALTW+hmf\na8aOwVw0fBVyzVjRBKXVpSzMWchb299iX8E+7h9yPw8Pe5hh3Yb5uzQhAkqb9Oi11t8opXrX27fK\n5+F64E7P/ZnA+1prJ3BYKbUPGAtsuJyiRMegteabo9/w5vY3+WjPR9j62PjFuF8wPWO6LEUgRCtq\njYOxs4G/ee5fBazzee6EZ58QtU6VneLt7W/zxrY3iAiLYM6IObxw0wt0je/q79KECEktCnql1K+A\nGq313y754kbMmzev9r7NZsNms7WkHBHAXG4Xnx/4nNe3vs7qQ6u5M/tO3r79ba7pcY2c0CTERdjt\ndux2e4s+o0mzbjytm6VWj96z7xHgMWCi1rras+9pTL/+Rc/j5cBcrXWD1o306DuGo8VHeXPbm7y5\n7U26xXfj+yO/z72D7yWxU6K/SxMiKLXlPHrl2awvmgr8ArjBCnmPJcB8pdTvMS2b/sDGyylIBL8a\nVw2f5H/C69teZ92xddw3+D6W3LeE4d2G+7s0ITqkSwa9UmoBYAPSlFJHgbnAs0AUsNLzz+71Wusn\ntNa5SqmFQC5QAzwhw/aOI/98Pm9sfYO3d7xNZlomc0bMYdHdi4iNjPV3aUJ0aHLClGiRKmcVi3MX\n89rW18g7l8dDwx5izog5ZHXO8ndpQoQkWetGtJu95/by6uZXeW/ne4zuPprHRj7GrVm3EhUe5e/S\nhAhpstaNaFMut4tP8j/hlU2vsOv0LuaMmMPmxzfTJ7mPv0sTQlyEBL24pLPlZ3lj2xu8uvlVrky4\nkifHPMldA++SpYCFCBIS9OKCNp7YyJ82/YkleUu4fcDtfDjrQ0Z3H+3vsoQQl0l69KIOh8vBopxF\nvLzhZc5VnOOHo3/I7BGz5RqrQgQIORgrmu1M+Rle3fwqr25+lYFXDOSpq59iesZ0WQ5YiAAjB2PF\nZdt2chsvb3iZj/M+5u6Bd7PiwRUM7jLY32UJIVqRBH0H5HK7WJK3hD9s+AMHCw/yxOgn2PejfXIp\nPiFClAR9B1JSXcKb297kjxv+SNf4rvz46h/zvezvyZLAQoQ4CfoO4EjREV7e8DJv73ibSemTWHDn\nAq7pcY2/yxJCtBMJ+hC27eQ2fvft71i+fzmzR8xm2w+20Supl7/LEkK0M5l1E2K01qw6uIqXvn2J\nPWf38NTVT/H4qMdJik7yd2lCiFYgs246MKfbyeLcxby49kWqXdX8Ytwv+Kch/yRrzwghZEQf7Bwu\nB29vf5sX175It/hu/PK6XzIjcwZhKszfpQkh2oCM6DsQK+Cf//p5sjpn8dfb/8r4XuP9XZYQIgBd\nctinlHpDKXVaKbXTZ1+KUmqFUipPKfW5UirJ57lnlFL7lFJ7lFI3t1XhHZXD5eC1La+R+T+ZfLjn\nQxbcuYDPH/hcQl4IcUGXbN0opcYDZcA71jVjlVIvAue11i8ppX4JpGitn1ZKDQTmA2OAHsAqIKOx\nHo20bi5PtbOaN7e9yQtrXyC7czb/a8L/YlzPcf4uS4iQ5HLBsWOQl+fd9u8Hh+Pi73vgAZgzp21r\na5PWjdb6G8/FwX3dBkzw3H8bsANPAzOB97XWTuCwUmofMBZocHFw0TTVzmpe3/o6L6x9gaFdh/LB\nXR/IHHghWoHbDcePw759dbf9++HQIUhLg6ws7zZjBsTEXPwze9dPygDR3B59F631aQCt9SmlVBfP\n/quAdT6vO+HZJy6Tw+XgrW1v8fzXzzOk6xD+PuvvjLlqjL/LEiKoaA3ffWdG5Pn5JsSt7eBBSEmB\njAyz9e8P111n7vfrB7EhdKnj1joYKz2YVlLjquGdHe/wm69+Q1bnLBbdvYire1zt77KECGgFBSa8\nrRH5vn3elktsrBmRZ2aaMB83ztymp0N8vL8rbx/NDfrTSqmuWuvTSqluwBnP/hNAT5/X9fDsa9S8\nefNq79tsNmw2WzPLCX5Ot5P3dr7Hb776Db2TevPe996TA6xC+HA4TIjv2QN795oRen6+CfWaGu+o\nvH9/mDQJnnzSBHxKir8rbxm73Y7dbm/RZzRpHr1Sqg+wVGs9xPP4RaBAa/3iBQ7GXo1p2axEDsZe\nlMvtYsGuBfzHV/9B94Tu/IftP5jQZ8Kl3yhECHI6zUFQq71y4IC5zcszffNevSA7GwYM8I7SMzPh\niitAXdbhyeDVJgdjlVILABuQppQ6CswFXgAWKaVmA0eAWQBa61yl1EIgF6gBnpA0b5zWmr/v+Tu/\nXvNrUmNS+X+3/D9u7HMjqqP8aRUdlssFR440PAC6f7/Z37Wr6ZFbo/Nx47x99Ohof1cfnOTM2Ham\ntWblwZU8+8WzuLWb5yc+z9T+UyXgRUjRGs6erdtisbZDh8wI3Apv34Oh6ekS5pcilxIMcJtObOKX\nq37JidITPHfjc9w58E5ZqkAEtbIyMxKvH+Z5eRAWVnd6otVm6dfv0tMUxYVJ0Aeofef38avVv2Lt\nsbXMmzCPR0c8SkSYrD4hgoPDYUbh1hRF34OgBQUmuH2DPCPD9NA7ywXL2oQEfYA5W36WefZ5fJDz\nAT+99qf8+JofExsZQpNzRciwWi3WaHzvXrPl5cHRo9CjR8Mwz8iAnj3NyF20H1nULEBUOat4ef3L\n/O7b33H/kPvZ++ReuR6rCAgVFWYk7hvk1ug8IsI7Gh8wAGbPNuHevz9EyWrXQU1G9K1Ia80HOR/w\n9KqnGXHlCF6c9CKZaZn+Lkt0MFrDqVPeIPcN9VOnTKvFCnNrlJ6RAamp/q5cNIW0bvxo68mtPLX8\nKcod5fx+yu9lLrxocw6HmWe+Z4/3JCIr0KOjvQdBrVAfMAD69DEjdxG8JOj94Ez5GX71xa9Ymr+U\n5yY+x6PDHyU8LNzfZYkQ4XabE4h855xbB0OPHjUnEA0Y4D2JyBqly+g8dEmPvh053U7+tPFPPPf1\nczw49EH2PrmX5Ohkf5clglR5uRmNWyNz6/bgQRPa1sHPrCyw2Uy7JT1deueiaWRE3wz2w3Z+tOxH\ndI3ryv9M+x+yr8j2d0kiCGgNJ0823mo5f94c9MzO9m4DBph9cXH+rlwEEmndtLETJSf4+cqf8+2x\nb/nvm/+b72V/T85oFQ1Y7ZbcXO+Wk2PCvVOnukFutVp69YJw6fiJJpCgbyNOt5NXNr7Cc189xw9G\n/YBnr3+WuCgZZnV0BQV1Z7ZYPfQDB8yKiQMHNtzS0vxdtQh2EvRtYN2xdfzw0x+SFpvGn6b/iQGd\nB/i7JNGOtDYHPX3bLdZtVZV3RO57MlH//h1nnXPR/iToW1FRVRFPr3qaJXlL+M+b/5P7Bt8nbZoQ\npjWcOOFts+TkwO7d5nF8PAwa1HB2y5VXdpylcUXgkFk3rUBrzcKchfzk859wW9Zt5P5LrsymCSFa\nw5kzJsR9t9xcs9DWoEGmxTJ6NDzyiHkc7BeuEEJG9D4OFx3miU+f4GjxUV679TXG9Rzn75JEM1nX\nCs3JMSG+Z4/3wKjWMHgwDBlibgcNMpv0z0UwkNZNM7ncLl7Z+Aq/+eo3/Ozan/GzcT8jKlwmKAcD\nreH48botFyvQY2LqHgi1Zrt07SotFxG82j3olVI/AeYAbmAX8CgQB3wA9AYOA7O01sWNvDcggj7n\nTA5zlsyhU0Qn/nLrX2RtmgBltVys3vnu3d77VqBbI3PrvozQRShq16BXSnUHvgEGaK0dSqkPgM+A\ngcB5rfVLvteTbeT9fg16h8vBb7/+La9seoXnbnyOx0Y9JhcBCRAFBd4Q9711u02ADx5ct+Ui656L\njsQfB2PDgTillBuIAU4AzwDWil5vA3agQdD706YTm5i9ZDa9k3qz7Qfb6JHYw98ldUhVVabFsmuX\nd9u9G0pLvYE+aBDcfru5Ly0XIZqnpa2bfwWeByqAFVrrB5VShVrrFJ/XFGitGyyx5I8RfUVNBXPX\nzOWdne/w+ym/lymT7cTtNlco8g30Xbvg8GEz53zIkLpbr14S6EJcSLuO6JVSycBtmF58MbBIKXU/\nUD+9/d+IB745+g2Pfvwoo7uPZtcPd9Elrou/Swo51jrovtMWd+0yo/a0NO9Ml9tvh1//2pxkJIty\nCdH2WtK6mQQc1FoXACilPgLGAaeVUl211qeVUt2AMxf6gHnz5tXet9ls2Gy2FpTTuIqaCn71xa/4\nIOcD/jT9T9yRfUerf0dH5HCYAN+xA7ZvN7c7dpiRuDVtcexYmDPHtF+SkvxdsRDByW63Y7fbW/QZ\nLTkYOxZ4AxgDVANvAZuAXkCB1vpFfx+MtUbxY68ayx+n/pG0WJmG0RzFxSbEt20zob59u1nfpU8f\nGD7cbMOGmU366EK0LX9Mr5wL3AvUANuA7wMJwEKgJ3AEM72yqJH3tlnQVzur+fWaX/Pezvf484w/\nc/uA29vke0KNta6LNUK3Qv3MGRg61AT5iBEm2AcPhli5zrkQ7U5OmAJ2nd7FAx89QHpKOq/d8hpX\nxF3R6t8RCqzWixXmVrDHxHhH6MOHm2Dv10+W0BUiUHTooHdrN39Y/wd++81veWnSSzwy/BGZUeNR\nXW0Oim7ZAps3m9s9e8wVihprvQghAleHDfoTJSd4+B8PU+ms5N073iU9Jb1VPjcY1dSYULcCffNm\nE+r9+8OoUd5t2DBpvQgRjDpk0H+892N+8MkPeGLMEzx7/bNEhHWcBTndbnOhi40bzbZpkwn5Pn3M\n6otjxpjbYcNMS0YIEfw6VNBX1FTws89/xvIDy5n/vfkdYqXJM2dgwwZYv97cbt5sLhw9dqwJ9TFj\nYORIueiFEKGswwT97jO7uefDexjWdRj/d8b/JSk69CZpV1ebg6MbNnjD/fx5uPpq7zZ2LFwhx5qF\n6FBCPui11ry25TX+fc2/87vJv+PhYQ+HxAFXrc3FpNetM9v69aYFk5HhDfVrrzVnkobJumtCdGgh\nHfRFVUU8vvRx8s/n8/5d7wf1tVsdDjNa//Zb7+ZwmDC3ttGjIU6uPy6EqCdkg37Ld1uY9eEspvWf\nxn/e/J9ER0S3cXWtq6TEhPk335ht82YzN33cOLjuOhPs6elyRqkQ4tJCLui11ry6+VXm2ufy5xl/\n5q6Bd7VTdS1z9ix8/TV89ZXZ8vPNCP3662H8eBPsiYn+rlIIEYxCKujLHGU8vvRxcs/msujuRWSk\nZbRjdZfn5En48kvvduKEGanfcIPZRo+WVRqFEK3DHxceaRN7z+3ljg/uYHzP8aybs46YyMCaBH72\nLKxZA6tXm9uzZ81ofcIE+P73zZmmsmSAECJQBNyI/qM9H/GDT37AC5NeYPaI2X6orKGSEjNSX73a\nbEeOmGCfOBFuvNEs+CWzYYQQ7SGoWzcut4tfr/k183fNZ/GsxYzuPtovdYGZw75uHaxaZbacHDPF\n8aabTLiPGgURAflvISFEqAvaoC+qKuKeD+/B6Xby/p3vt/uKk1qbA6affgorVsDatTBwIEyaZMJ9\n3DiIDq6JPkKIEBWUQX+o8BAzFsxgcvpk/mvKf7XbWjUOh5kR88knJuArKmDGDJg61bRjUlIu/RlC\nCNHegi7o1x9bzx0f3MEz45/hR1f/qM2/88wZ+OwzE+6rVsGAAXDLLWYbNkzmsQshAp8/rjCVBLwO\nDAbcwGwgH/gAc9Hww5grTBU38l7d+aXOvHXbW9ySeUuza7gYrWHvXliyxGw5OaYdc8stMH06dJHr\ngwshgow/gv6vwJda67eUUhFAHPAscF5r/dKlrhm75bstjLxyZLO/vzFam+UFFi6ExYuhshJmzoTb\nbjPTHzt1atWvE0KIdtWuQa+USgS2aa371du/F5igtT6tlOoG2LXWDRamac0Lj2gNO3fCBx/AokVm\nnfa77zbbyJHSkhFChI72PmGqL3BOKfUWMAzYDPwY6Kq1Pg2gtT6llGqzBsmBA/C3v5mtvBzuuQfe\nf1/CXQghfLUk6COAkcC/aK03K6V+DzwN1B+mX3DYPm/evNr7NpsNm812yS89c8aM3OfPh0OHYNYs\n+MtfzPoxEu5CiFBjt9ux2+0t+oyWtG66Auu01umex+MxQd8PsPm0btZorbMbeX+TWzeVlfDxx/Du\nu2aO+623wv33mwOrcuKSEKIjadfWjSfIjymlMrXW+cBNQI5newR4EXgY+Lh5n2+ugfrWW+bA6qhR\n8NBDZjQvl8oTQoima+msm2GY6ZWRwEHgUSAcWAj0BI5gplcWNfLeRkf058+bkfvrr0NVFTzyiAn4\nXr2aXaYQQoSMoDthyvpurc2iYa+9Zk5omjEDHnvMTIeUvrsQQngFXdBXVWkWLIA//MEsSfDP/wwP\nPgipqX4pSQghAl7QBX3Xrprhw+HHP4YpU2T0LoQQlxJ0Qb97t2bQIL98vRBCBKWgC3p/fbcQQlyM\n1prq49VU5FZQnlNOeU45FXkV6OqLZ1a32d246odXtWltEvRCCHEZ3DVuKvdVUp5bTkVuBRV5FVTs\nNbfh8eHEDYojblAcsQNjicuOIyz24peSi7oyiugebXvxCgl6IYRohNvppupAFeW7zejcuq08UEl0\nz2hiB5kgj82OJTYrlpisGCKTI/1ddqMk6IUQHZp2a6oOeQLdCvWccirzK4nqHmVG6IM9o/RBJtTD\nY8L9XfZlkaAXQnQIVg+9PKfc20ffVU55bjmRaZEmzAfXbbuExwVXoF+IBL0QIuQ4zji87ZZd3pF6\nWExYbQ+9dqQ+OI6IpNBeAEuCXggRtGqKarwtl93lVORUUL67HO3UxA6KJX5IfG2Yxw6KJapzlL9L\n9gsJeiEyjdIwAAAZpElEQVREwHOVuyjfUzfMy3eX4yxymoOiPm2XuEFxRF0ZhZKzKWtJ0AshAoa7\n2k3F3grKdpWZQPe0XxwnHcRkxdQ5MBo3JI7o3tGoMAn0S5GgF0K0O6011SeqKd9RTtnOMsp3mtuq\ng1VE940mbkjdEXp0v2jCIi4+H11cmAS9EKJNOUuc3gOiu8sp21VG+a5yVKQifmg88cPiTbAPjSN2\nQCzh0aEx0yWQSNALIVqFdmuqjlSZ0fmOMsq2m81xxkHcwDjvKN1z26lbJ3+X3GFI0AshLpuz1IzS\ny3aUUbbDjNDLd5UTnhhO/JB44od7t5j+Mahw6aP7k1+CXikVBmwGjmutZyqlUoAPgN7AYcwVpoob\neZ8EvRDtSGuN4zsHZdvLKN1WStk2zyj9lMP0z4fG1Wm/RKYG5hIAHZ2/gv4nwCgg0RP0LwLntdYv\nKaV+CaRorZ9u5H0S9EK0EbfTTWV+ZW3LxdrQED8i3rsNjyc2I1ZG6UGk3YNeKdUDeAt4HvipJ+j3\nAhM8Fw/vBti11gMaea8EvRCtwFnqNL307WW1/fTynHI6de9E3LA4EkYkmNbLiHiZk3653G44dgz2\n7oX9+6G6+uKvv/pquO66Ni2pOUHf0nOFfw/8Akjy2ddVa30aQGt9SinVpYXfIYTAu2CX7xTGsh1l\nOE46iBscR/wwM0Lv9kg34obEEZEQ2ksBtKrqati3D3JzYc8es+3dC/n55tqmWVmQkQGxsRf/nAEN\nxrQBodl/EpRSM4DTWuvtSinbRV56wWH7vHnzau/bbDZstot9jBAdh6vSZaYvbvO2Xcp3lRORFmGW\nAhgSxxV3XUHf5/oSkxEj89KbQms4c8YEen6+CfK9e02oHzsGffpAdrbZZsyAn//cBHxCgl/Lttvt\n2O32Fn1Gs1s3Sqn/AzwAOIEYIAH4CBgN2HxaN2u01tmNvF9aN0IANYU1puWyzWylW0upOlhFTGaM\nabt4eulxQ+MCdo30gFJRAQcO1A3zvDyzRUZCZqYZnWdmeoO9Xz+ICo61c/w2vVIpNQH4madH/xLm\nYOyLcjBWCC+tNdXHqk2YbyutHak7C5xmxsvweBPsI+OJGxhHWCcZpV+Q1mYUnpNjttxc00M/cAAK\nCszoPCvLtFIGDDD3MzMhLc3flbdYoAR9KrAQ6AkcwUyvLGrkPRL0ImS5q91m4a4d3l562bYywqLC\nGsx6iUmPkTVeLsQ6GJqbazYr2Pfsgfh4GDTIbNnZJsj794erroKw0P2fpJwwJYQfOEudJsi3elsv\nlfmVRKdHmznpQ70HSjtdKWeQNsrhMCNyq2dubXl5kJgIAweaQPe9TU31d9V+IUEvRBtzFjsp3VpK\n6ZZSyraaUK8+Vk3coDjiR/q0XgbHBd0l6tpFWZkJc2t2i3V79Cj07u1ttQwY4O2fJyVd+nM7EAl6\nIVqRs9hZewZp6ZZSSjeVUn282vTSRyYQP8rcxmbHEhYZuq2Cy2bNbsnP9x4EtdouZ86YFsvAgSbE\nrdv+/YPmYKi/SdAL0UyOc47aEbp16zjlMEsCjDSBnjDGE+oyldGoqTFTFa0551ao5+ebHrl1ADQr\ny9tu6dsXwuVfOi0hQS9EEzhOO8wIfWspZVtMqDuLnbVtl/gRnpF6liwNAJiTifLzvX1z64DowYPQ\ns6cZkVszW6ytc2d/Vx2yJOiFqMdZ4jShvrGUkk0llG4sxVXmImGUt/USP1JmvgBQVOQdnfveWicT\n+bZbBg0ygR4T4++qOxwJetGhuSpclG0rM4G+uZTSzZ6e+tB4Eq9OJGFsAoljE4lOj+646724XObA\np9Vi8Q31sjLvQVDfg6LSPw8oEvSiw3A73JTnlFO2tax2pF6RV0HcwDgSxph+esLoDtxTr6kxQW7N\nP7dmuOzbZ9oqVovF9+zQHj2go/4PMIhI0IuQ5K5xU767nNJNZuZL6dZSKvZUEJ0eTcIIE+qJVycS\nNyyu4126zuWCQ4e8Z4fu3g27dplA79nT22axWi5ZWRAX5++qRQtI0Iugp92ayv2VlG4qpWRjCaWb\nSinbUUZ07+g6I/X4ofGEx3agULdOKLIOiFqj9Lw86NLFO6tl8GAYMsS0XC610qIIShL0IuhUf1dd\nG+ilG01fPTwpnMQxid5gH5VARGIHWXK3pKTh/PPcXDNqt2a4+M4/HzDA76srivYlQS8CmuOcwxwk\n3VRae7DUXe02oe45UJowJoGoLh3gwN+5c96TiKzbPXtM0GdkeHvo1kg9IwOio/1dtQgAEvQiYLjK\nXZRu9rZfSjeVUlNQQ8LohDpbdJ8QngGjNZw4UTfIrc3pbLh+y4AB5oBoCC/IJVpOgl74hdVXL9lQ\nQsn6EkrWlZgZMIPjzLTGMWa0HpMRonPVrSVzrTaL70g9JsYb5lbbJTsbunaVGS6iWSToRbuoPlVN\n6QbPaN3TV49IjjCBfm0iidckEj8iPvRmwDgcplduzT/3HalbS+YOHOjdBg0KifXPRWCRoBetzlnm\npGxLWW2ol2wowVXuMv30UO2rl5R4Z7ZYW14eHD9uWiuZmWbzXQs9JcXfVYsOQoJetIjb4aZsZ5l3\nvvqmUioPVhI/NN4b6lcnENMvJjT66sXF3rVbfC9qUVho+uX12y19+8oZosLv2jXolVI9gHeAroAb\n+IvW+o9KqRTgA6A3cBhzhaniRt4vQe9n1SerKV5bTMm3JRR/W0z5znJi+sV4pzWOMfPVw6KC+OCg\n221G4taURes2J8dccs537RZr691bDoiKgNXeQd8N6Ka13q6Uige2ALcBj2KuGfuSXDM2cGiXpnx3\nOcVriyn+tpiStSU4i50kjksk6bokksYlkTA6gfC4IO2ru91w5Ig5M9S35bJnj7lCke8p/1bbRQJd\nBCG/tm6UUv8AXvFsE7TWpz3/M7BrrQc08noJ+jbkOOvwzoJZb6Y4Rl0ZRdK4JJKuSyJxXKJZhjfY\nZsE4neaAqO9UReuAaEqKOTPUd9qiXKFIhBh/Xhy8D2AHBgPHtNYpPs8VaK0bXNxRgr71aG2mNxZ/\nU2y2r4txnHaQONbMgEm8xvTWozoHUX+5tNSs15KXVzfUDxyAbt3q9s6tUJdAFx1Ac4K+xeeVe9o2\nHwJPaa3LlFL10/uCaT5v3rza+zabDZvN1tJyOgS3w03ZtjLThvmmmOK1xYRFhZF0fRJJ45Po8VQP\n4gbFBcdFM86c8R4E9V2/pajInA1qra54xx3w7LOm/SJruIgOxG63Y7fbW/QZLRrRK6UigE+AZVrr\nlz379gA2n9bNGq11diPvlRF9EzlLnZSsK6kdrZduLiW6X7TprY83rZjoXgF8erzDYUbie/d6Lzln\nHRR1u+seCM3ONmEuZ4iKNlDjdlPsdFLscplbp5Min63Q2mpqKKz3fLHLhfsSmfVvvXoxt0+fNv0Z\n2r11o5R6Bzintf6pz74XgQKt9YtyMLZ5qk9We9sw3xRTsbeChFEJJtSvNwdOI5ICbJEvreHkSRPg\nvltenjlI2qtXw8vNZWXBFVfIGaLisjjc7jph7BvOBZ77BT7PWfeLnE4cbjdJERG1W7LPlhQeTkpk\nJCkREbVb7XMRESRGRBBxiT+rkUoR1cYDlPaedXMd8BWwC9Oe0cCzwEZgIdATOIKZXlnUyPsl6D2q\njldR/GUxRfYiir4souZcjXe0Pj6J+FEBdJap02muFep7qTnrfqdOdWe2ZGSYcO/XzzwnhEeVy1V3\nBG2NqOsFdIHPrTWyrtG6QRinREaS6nmc6gnr1EZCOy48POjPAZETpoKAdmvKc8sp/sZMcSz+phhn\nqZPkCcm1W9yQOP/OhrEW48rLM2ugW1t+vmnBdO/uXSLX96BoaoNj7iKEVfqEdWMj6Pqjad9Ad3vC\nOtknqH1DOSUykrR6AZ4SGUlyRASxYWFBH9YtIUEfgLRLU7ajjKIvzWi9+OtiIlMjSRqfROJ1iSSN\nTzLTHP3xB7eiAg4fNtMVDx70XqFo924zAreuF2pt1sFRuSB0yLDaIL6j5yKfQC5qZGRtvUZDbVin\negK5zii6kZC2Xh/TwcO6JSToA4Db4aZ0SynFXxVT9FURxWuL6dS9E8kTkkmakETyDcl06t6ObQzr\nUnPWQVBry883p/r37m1O7e/b13uFosGDTe9cBAXrAGP9gK4f4OfrjbYLa2qo9oysGwtpa7Sd5hPk\nVngnR0QQEx4g7cQORoLeD9xON2VbyyhaU0Th6kJKvi0hpn8MSTckmXAfn9Q+C34VFJh559ZBUGuG\ny/79Zt65dSA0M9N7e9VVMrMlQLi1priR1kf93vV5T3Cfr6nhvGdkXeVzgLGxvnVqvVvf0XZ8CPSs\nOxoJ+nag3ZryXeUUrimkaHURRV8VEd0zmuQbk0memEzyDclEpka2wRdrOH3a9MgPHDABbt3u3w81\nNd6DoFaYZ2eb+zLvvN043O7a0bPvLJBCz77zNTWNjq5LnE7iw8MbHkT0uZ8cEUFaZKTZPPeTJaw7\nHAn6NqC1pnJfJYWrCyn6oogiexERKREkT0wmZWIKybbk1huxu1xmiuKhQ2Z0bm1WmEdHmxks/fp5\n++bW/S5dZJpiK6pyuepM1SvwCen6/eoCn5F2ldvdYPTsOwvECuj6o+uk8HAi5F9Xogkk6FtJzfka\nClYWULiikMKVhWitSbkpxQT7xGSie7bg5KTqanPgs36IHz5srlKUmgp9+pgDn9YBUCvUk5Nb60fs\nELTWlLpctSNoq91x/hIzRAqdTlxa1xlZp3mC2Tek0+odgEyLjCRBRteijUnQN5Pb4aZkXUltuFfs\nrSB5QjIpU1JInZxKTOZlrr9eU2NG5daBT99AP33anDxUP8j79jX7ZUZLA1prylwuE9L1Avt8I6Nr\n33nX0WFhtaFstTzqH1hsbIZIR5/CJwKXBH0TWe2YguUFFKwooPirYmKzYkmZnELKzSkkjUu6+Brs\nWps1Wqx++ZEjDbfu3esujWuNynv1gogAO6u1nfiOsOvPBDlfr29dP7QjlWrQn+7sGV031g5J9fSv\n2/osRSHamwT9RbjKXRSuLqRgWQEFywtwO9ykTkkldUoqKTelEJlW7wBqZaUJbGuOuXV78KAJ906d\nvP3yPn1MgPfubbb0dNNPD1FOt5til6v2YGNjI+pCp7PR56wRthXQKfWC2rdNkuIz+o6WqXxCABL0\nDVQerOT8p+c5/+l5StaWkDA6gdTpqaROTSWuL6jjx70j8MOHvduhQ2aOec+epqWSnu69TU834R7k\n/XLf0bUVyoWXCGprql+Fy1W7Tkj9E2XS6rVA6oe4jLCFaJngC/qvvjIj4O7dW6Wd4Xa4KV5znvMf\nHuf8yhKcxS7SBpSQetUxUuP2EHH+qDngeeyYGbFfdZUZjftu1glE3bsH/BxzK6wbW+Cp/rohhfXa\nIsUuV+3o2rdH7dv+aGx2SIpnOl+Y9K+F8IvgC/prrzWj6XPnzEk9XbpA585mS0szByYjIrybywVV\nVWbmSlUVlJVRc6qcgkNdOHcqncKyAcRwnLSEXFJ7HCehnwvV/Uq40mfr0cOM1NPSAmI6oltrSpwN\nl0qtf1u/BVLoOVkmxjP32jq93PesxvoBXf/MxsgA/x+ZEKKh4At667urq80iWufO1d2qq81qiU6n\nmckSEQHR0VSVJ3AurzPncpIpPRJN8qgwOt8cT9od3YgaeGW7j8SrfU5Bb2zzHXHXXwSq2Okktl5Y\nJ/uEtbU1doajhLUQHU/wBn0TlOeWc3bxWc59dI7qY9Wk3ZpG59s6kzI5hfDY5h2oq3G7KXO5KHW5\nam9LnE5KPBclKHGaiw2UeALZ92IF1uMipxOnz0p8ST5hbT2uM9qu19tOjoiQE2WEEE0WdEHvdrtx\naY1Ta2o8t773K/ZVUrroHJWLC9BFTrg1Cef0RCrHxlAVpqlyu6l0u+vclrtcVLjdVLhclHsCvP5W\n7nmdW2sSPD3n+PBwEjxbUkQEidZthLkgQf37ST5BLnOuhRDtJaCCXik1FfgDEAa8obV+sd7zmjVr\nCMNclSVCKSLDwuh6Bsav1ly3yk3qWc3WmyLYOTmC48PC6RQRTnRYWO3WKSyMmLAwYsK9++PCwogL\nDyc2PJzYsDASPCFubXE+W5RSEtCiAfN3wmy+981jN+D27Hf5PG54q7XL8xrv67yfoRu8p+Fz+iL3\n3Q0+x1tv7U+C95LN1nO+Nbp8fhbf19f/Od0Nvtf7s1nfXf/3V782LvA99X+/jT223uNu8Lus/9+r\n7s/alO9x16n34nVeWpcu99Ct24NNem1zBUzQK6XCgHzgJuA7YBNwr9Z6r89rtMvtJkwpqk9Vc+6j\nc5z52xnKc8rpfEdnut7XlWRbst8vcG2324PiouVr1qxhwoTr0brGszlrb93uuo8b3l7oNb5BYP0l\n8w2v+o+djYRI3eBav/4IY8d2Q2un533ORj7L9/0u6v4Fv9B3131/46Fbt666n+39LNBs3w7DhyvA\nbGZAEIb5o23tC0epcMxYxnocVu+x9zV1n1OA9/XmcWPfEXaBGsz+zZsLGTPmijrv9+Xdj8/n+X6v\n9/O9r2/4c/rW4v15wmuf836H9b3ez1+//jDXXJNeW0Pdunx/Nt/HYT6vt94T7lO/ou73qgY/a+Of\n29j3mN//2rU5XHfdkIvWeSkxMf2Jixtwyde1RHOCvq1O0RwL7NNaHwFQSr0P3Abs9X3RiT8c5+zi\ns1TkVJA6LZWeP+9J6pRUwjoFTs96zZrVXH/9GNzu6tpNa+t+leexw+c5B263w3Nb4/O4upHX1Pi8\n1uEJ1wu9v8Znn2+Ym+3tt2tQKgylIgkLi0SpCJSK9GwRtY+9z136+YaB4P3Lbf6Ch/s8jvDZF4FS\nUY2GyZYtOdhsA+q8p+5nhdX5XO93egOq4Xc39rh+sNbd7/vZDX+uMOz2/43NNq89/6g1y0cfzWPO\nnHn+LuOi/vrXedx776/9XcYlbd++i9tvv8PfZbSJtgr6q4BjPo+PY8K/jorcCnr/qjcpE1PqhLvW\nrnrB6qgXsg6f+1W43VW4XJW43VWNvNb7Gu9W6XO/2idYva+3PvvIESdr1/6OsLBolOpEWJh3a/xx\nFEpFeW4jPeHpfU1EREq910R5wtX72Lyn7uf4fl7DMI9g9ernsdn+dxv952w9SUll9OjxpL/LEKJD\n8euiK8WP3U6huxr3lqo64ay1yxNy0YSFRV0gVKM892M8r4v2PI6u83xkZEK9531fH+0JcOuzrHCN\nqf2s1auf44YbAj9A5ViDEOJC2qpHfw0wT2s91fP4aUD7HpBVSgXG0pVCCBFkAuVgbDiQhzkYexLY\nCNyntd7T6l8mhBDiotqkdaO1dimlngRW4J1eKSEvhBB+4LcTpoQQQrQPv8xjVEpNVUrtVUrlK6V+\n6Y8aGqOUekMpdVoptdNnX4pSaoVSKk8p9blSKsnPNfZQSq1WSuUopXYppf41QOvspJTaoJTa5qlz\nbiDWaVFKhSmltiqllngeB1ydSqnDSqkdnt/pxgCuM0kptUgptcfz5/TqQKtTKZXp+T1u9dwWK6X+\nNQDr/IlSardSaqdSar5SKqo5NbZ70HtOpnoFmAIMAu5TSrXtGQZN9xamLl9PA6u01lnAauCZdq+q\nLifwU631IOBa4F88v7+AqlNrXQ3cqLUeAQwHpimlxhJgdfp4Csj1eRyIdboBm9Z6hNbamq4ciHW+\nDHymtc4GhmHOnwmoOrXW+Z7f40hgFFAOfEQA1amU6g78CBiptR6KabXf16watdbtugHXAMt8Hj8N\n/LK967hIfb2BnT6P9wJdPfe7AXv9XWO9ev8BTArkOoFYYDMwJhDrBHoAKwEbsCRQ/7sDh4C0evsC\nqk4gETjQyP6AqrNebTcDXwdanUB34AiQ4gn5Jc39u+6P1k1jJ1Nd5Yc6mqqL1vo0gNb6FNDFz/XU\nUkr1wYyW12P+wwdUnZ52yDbgFLBSa72JAKwT+D3wC7yLpUBg1qmBlUqpTUqp73v2BVqdfYFzSqm3\nPG2R15RSsQRenb7uARZ47gdMnVrr74D/Ao4CJ4BirfWq5tQYOGsNBI+AOHqtlIoHPgSe0lqX0bAu\nv9eptXZr07rpAYxVSg0iwOpUSs0ATmutt3PxxUz8/vsErtOm1TAd07K7ngD7fWJGniOBP3lqLcf8\nqz3Q6gRAKRUJzAQWeXYFTJ1KqWTM0jG9MaP7OKXU/Y3UdMka/RH0J4BePo97ePYFqtNKqa4ASqlu\nwBk/14MyC9F8CLyrtf7Yszvg6rRorUsAOzCVwKvzOmCmUuog8DdgolLqXeBUgNWJ1vqk5/YspmU3\nlsD7fR4HjmmtN3seL8YEf6DVaZkGbNFan/M8DqQ6JwEHtdYF2qy29xEwrjk1+iPoNwH9lVK9lVJR\nwL2Y3lOgqL9M3RLgEc/9h4GP67/BD94EcrXWL/vsC6g6lVKdrdkASqkYYDKwhwCrU2v9rNa6l9Y6\nHfNncbXW+kFgKQFUp1Iq1vOvOJRScZi+8i4C7/d5GjimlMr07LoJyCHA6vRxH+Z/8JZAqvMocI1S\nKloppTC/y1yaU6OfDjJMxZw5uw942l8HOxqpawFmWeVqzy/5UcyBkFWeelcAyX6u8TrABWwHtgFb\nPb/P1ACrc4intu3ATuBXnv0BVWe9mifgPRgbUHViet/Wf/Nd1t+bQKvTU9MwzIBuO/B3IClA64wF\nzgIJPvsCqk5gLmaAtBN4G4hsTo1ywpQQQoQ4ORgrhBAhToJeCCFCnAS9EEKEOAl6IYQIcRL0QggR\n4iTohRAixEnQCyFEiJOgF0KIEPf/AdG5HRD4UygeAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x125aa77d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i in range(6):\n",
    "    plt.plot([nrm[i] for nrm in norms])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}