2BwD-Layer-DK-AlignMNiST-4.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-04-20T21:52:04.548207",
     "start_time": "2016-04-20T21:52:04.544771"
    }
   },
   "source": [
    "## 2 Layer MLP for Dark Knoweldge Experiments w/AlignMNIST\n",
    "\n",
    "75 epochs of RMSProp on AlignMNIST\n",
    "\n",
    "Bigger network with dropout yet, and basic weight max norm constraints\n",
    "\n",
    "the initial values are\n",
    "\n",
    "     [54.569935, 62.339447, 6.9963851, 3.448611, 1.0789102, 0.082129896]\n",
    "\n",
    "\n",
    "see  http://www.r2rt.com/posts/implementations/2016-03-29-implementing-batch-normalization-tensorflow/\n",
    "\n",
    "https://github.com/tensorflow/tensorflow/blob/b3621c95160a916d4d255f9f44318b9d465701af/tensorflow/contrib/layers/python/layers/layers.py\n",
    "\n",
    "\n",
    "https://www.reddit.com/r/MachineLearning/comments/2bopxs/question_about_the_maxnorm_constraint_used_with/\n",
    "\n",
    "\n",
    "\n",
    "https://arxiv.org/pdf/1207.0580.pdf\n",
    "\n",
    "https://arxiv.org/pdf/1503.02531v1.pdf\n",
    "\n",
    "http://www.kdnuggets.com/2015/04/preventing-overfitting-neural-networks.html\n",
    "\n",
    "https://www.reddit.com/r/MachineLearning/comments/2bopxs/question_about_the_maxnorm_constraint_used_with/\n",
    "\n",
    "and https://github.com/tensorflow/tensorflow/issues/608"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:19.999981",
     "start_time": "2016-06-14T22:23:17.956800"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:20.621195",
     "start_time": "2016-06-14T22:23:20.615596"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%run augmentmnist.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:21.316919",
     "start_time": "2016-06-14T22:23:21.314059"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 75\n",
    "batch_size = 125\n",
    "\n",
    "learning_rate = 0.001\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:21.983566",
     "start_time": "2016-06-14T22:23:21.977648"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Network Parameters\n",
    "n_hidden_1 = 1200 # 1st layer num features\n",
    "n_hidden_2 = 1200 # 2nd layer num features\n",
    "n_input = 784 # MNIST data input (img shape: 28*28)\n",
    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
    "\n",
    "std_0  = np.sqrt(6.0/(n_input+n_hidden_1))\n",
    "std_h1 = np.sqrt(6.0/(n_hidden_1+n_hidden_2))\n",
    "std_h2 = np.sqrt(6.0/(n_hidden_2+n_classes))\n",
    "\n",
    "logfile = \"2BwD-layer-dlk-alignmnist-4\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:22.619037",
     "start_time": "2016-06-14T22:23:22.612123"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# tf Graph input\n",
    "x = tf.placeholder(\"float\", [None, n_input])\n",
    "y = tf.placeholder(\"float\", [None, n_classes])\n",
    "\n",
    "xt = tf.placeholder(\"float\", [None, n_input])\n",
    "yt = tf.placeholder(\"float\", [None, n_classes])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:23.265796",
     "start_time": "2016-06-14T22:23:23.261998"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def multilayer_perceptron(_X, _weights, _biases):\n",
    "    layer_1 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(_X, _weights['h1']), _biases['b1'])),  0.2)\n",
    "    layer_2 = tf.nn.dropout(tf.nn.relu(tf.add(tf.matmul(layer_1, _weights['h2']), _biases['b2'])), 0.5)    \n",
    "    return tf.matmul(layer_2, weights['out']) + biases['out']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:23.832707",
     "start_time": "2016-06-14T22:23:23.795953"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Store layers weight & bias\n",
    "weights = {\n",
    "    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1], stddev=std_0)),\n",
    "    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2], stddev=std_h1)),\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes], stddev=std_h2))\n",
    "}\n",
    "biases = {\n",
    "    'b1': tf.Variable(tf.random_normal([n_hidden_1], stddev=0.1)),\n",
    "    'b2': tf.Variable(tf.random_normal([n_hidden_2], stddev=0.01)),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes], stddev=0.001))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:24.387675",
     "start_time": "2016-06-14T22:23:24.355312"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "mlp = multilayer_perceptron(x, weights, biases )\n",
    "mlp_test = multilayer_perceptron(xt, weights, biases  )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:24.906749",
     "start_time": "2016-06-14T22:23:24.894959"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cost =  tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(mlp, y)) \n",
    "gs = tf.get_variable(\"global_step\",[],trainable=False,initializer=tf.constant_initializer(0))\n",
    "lr = tf.constant(learning_rate) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:25.614174",
     "start_time": "2016-06-14T22:23:25.448280"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "train_op = tf.contrib.layers.optimize_loss(cost, global_step=gs, learning_rate=lr,optimizer=\"RMSProp\", clip_gradients=1.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:26.045529",
     "start_time": "2016-06-14T22:23:26.024857"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'ScalarSummary_2:0' shape=() dtype=string>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#with tf.name_scope(\"training accuracy\"):\n",
    "pred = tf.equal(tf.argmax(mlp, 1), tf.argmax(y, 1)) # Count correct predictions\n",
    "train_acc_op = tf.reduce_mean(tf.cast(pred, \"float\"))  # Cast boolean to float to average\n",
    "tf.scalar_summary(\"training accuracy\", train_acc_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:26.628793",
     "start_time": "2016-06-14T22:23:26.617017"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'ScalarSummary_3:0' shape=() dtype=string>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_pred = tf.equal(tf.argmax(mlp_test, 1), tf.argmax(yt, 1)) # Count correct predictions\n",
    "test_acc_op = tf.reduce_mean(tf.cast(test_pred, \"float\"))  # Cast boolean to float to average\n",
    "tf.scalar_summary(\"test 0 accuracy\", test_acc_op)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:27.420590",
     "start_time": "2016-06-14T22:23:27.179472"
    },
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[1m\u001b[36m2-layer-dk-alignmnist\u001b[m\u001b[m           \u001b[1m\u001b[36m2-layer-modern-mlp-alignmnist\u001b[m\u001b[m\r\n",
      "\u001b[1m\u001b[36m2-layer-dlk-alignmnist\u001b[m\u001b[m          \u001b[1m\u001b[36m2BwD-layer-dlk-alignmnist\u001b[m\u001b[m\r\n",
      "\u001b[1m\u001b[36m2-layer-mlp-alignmnist-wdropout\u001b[m\u001b[m \u001b[1m\u001b[36m5-layer-mlp-alignmnist-wdropout\u001b[m\u001b[m\r\n",
      "\u001b[1m\u001b[36m2-layer-mlp-mnist-temp\u001b[m\u001b[m          \u001b[1m\u001b[36m5-layer-mlp-infimnist\u001b[m\u001b[m\r\n",
      "\u001b[1m\u001b[36m2-layer-mlp-mnist-watch\u001b[m\u001b[m\r\n"
     ]
    }
   ],
   "source": [
    "!rm -rf ./logs/{logfile}\n",
    "!ls logs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T21:57:37.534602",
     "start_time": "2016-05-04T21:57:37.532739"
    },
    "collapsed": true
   },
   "source": [
    "### Original MNIST Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:29.657263",
     "start_time": "2016-06-14T22:23:28.532372"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\".\", one_hot=True)\n",
    "trX_0, trY_0 = mnist.train.images, mnist.train.labels\n",
    "teX_0, teY_0 = mnist.test.images, mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:29.661930",
     "start_time": "2016-06-14T22:23:29.659034"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "alignmnist = AlignMNIST()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:30.178887",
     "start_time": "2016-06-14T22:23:30.175854"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# final norms...[102.45666, 162.9534, 40.922997, 18.756783, 53.30151, 3.2270765]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T18:12:43.510102",
     "start_time": "2016-06-14T18:12:43.502711"
    }
   },
   "source": [
    "#see https://github.com/tensorflow/tensorflow/issues/424"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:31.700263",
     "start_time": "2016-06-14T22:23:31.696205"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.python.framework import ops\n",
    "from tensorflow.python.ops import array_ops\n",
    "from tensorflow.python.ops import control_flow_ops\n",
    "from tensorflow.python.ops import math_ops\n",
    "\n",
    "from tensorflow.python.ops import variable_scope as vs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:32.648501",
     "start_time": "2016-06-14T22:23:32.619158"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def max_norm_constraint(_weights, _biases):\n",
    "    \n",
    "    norms0 = np.array([54.569935, 62.339447, 6.9963851, 3.448611, 1.0789102, 0.082129896])\n",
    "    norms0 = norms0*1.2\n",
    "\n",
    "    n0 = tf.constant(norms0[0], dtype=tf.float32)\n",
    "    n1 = tf.constant(norms0[1],  dtype=tf.float32)\n",
    "    n2 = tf.constant(norms0[2],  dtype=tf.float32)\n",
    "    n3 = tf.constant(norms0[3],  dtype=tf.float32)\n",
    "    n4 = tf.constant(norms0[4],  dtype=tf.float32)\n",
    "    n5 = tf.constant(norms0[5],  dtype=tf.float32)\n",
    "    \n",
    "    pred = tf.greater(tf.sqrt(tf.reduce_sum(tf.square(_weights['h1']))) , n0)\n",
    "    _weights['h1'] = control_flow_ops.cond(pred, lambda: tf.div(_weights['h1'], n0), lambda: _weights['h1'])\n",
    "    \n",
    "    pred = tf.greater(tf.sqrt(tf.reduce_sum(tf.square(_weights['h2']))) , n1)\n",
    "    _weights['h2'] = control_flow_ops.cond(pred, lambda: tf.div(_weights['h2'], n1), lambda: _weights['h2'])\n",
    "    \n",
    "    pred = tf.greater(tf.sqrt(tf.reduce_sum(tf.square(_weights['out']))) , n2)\n",
    "    _weights['out'] = control_flow_ops.cond(pred, lambda: tf.div(_weights['out'], n2), lambda: _weights['out'])\n",
    "            \n",
    "    pred = tf.greater(tf.sqrt(tf.reduce_sum(tf.square(_biases['b1']))) , n3)\n",
    "    _biases['b1'] = control_flow_ops.cond(pred, lambda: tf.div(_biases['b1'], n3), lambda: _biases['b1'])\n",
    "    \n",
    "    pred = tf.greater(tf.sqrt(tf.reduce_sum(tf.square(_biases['b2']))) , n4)\n",
    "    _biases['b2'] = control_flow_ops.cond(pred, lambda: tf.div(_biases['b2'], n4), lambda: _biases['b2'])\n",
    "\n",
    "    pred = tf.greater(tf.sqrt(tf.reduce_sum(tf.square(_biases['out']))) , n5)\n",
    "    _biases['out'] = control_flow_ops.cond(pred, lambda: tf.div(_biases['out'], n5), lambda: _biases['out'])\n",
    "    \n",
    "    return 0\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T22:23:33.694523",
     "start_time": "2016-06-14T22:23:33.582331"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "max_norm_op = max_norm_constraint(weights, biases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2016-06-15T05:29:46.961Z"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 0.815, 0.884)\n",
      "[54.544594, 62.16592, 7.0781412, 3.4363286, 0.99740976, 0.073847778]\n",
      "(1, 0.86086667, 0.91799998)\n",
      "[56.009369, 65.056999, 6.656517, 3.472435, 1.6046538, 1.6776466]\n",
      "(2, 0.88129997, 0.93370003)\n",
      "[57.315178, 67.659279, 6.3940864, 3.5244188, 2.5057023, 2.649864]\n",
      "(3, 0.89335001, 0.93550003)\n",
      "[58.549725, 70.100227, 6.3261886, 3.588649, 3.3910596, 3.6299446]\n"
     ]
    }
   ],
   "source": [
    "test_accuracies = []\n",
    "train_accuracies = []\n",
    "norms = []\n",
    "with tf.Session() as sess:\n",
    "    # create a log writer. run 'tensorboard --logdir=./logs/{logfile}'\n",
    "    writer = tf.train.SummaryWriter(\"./logs/{0}\".format(logfile), sess.graph) # for 0.8\n",
    "    merged = tf.merge_all_summaries()\n",
    " \n",
    "    tf.initialize_all_variables().run()\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        trX, trY = alignmnist.next_epoch()\n",
    "      \n",
    "        for start, end in zip(range(0, len(trX), batch_size), range(batch_size, len(trX), batch_size)):\n",
    "            sess.run(train_op, feed_dict={x: trX[start:end], y: trY[start:end]})\n",
    "                \n",
    "        #sess.run(max_norm_op)\n",
    "        \n",
    "        summary, trn_acc, tst_acc = sess.run([merged, train_acc_op, test_acc_op], feed_dict={x: trX, y: trY, xt: teX_0, yt: teY_0})\n",
    "        writer.add_summary(summary, epoch)  \n",
    "    \n",
    "        \n",
    "        print(epoch, trn_acc, tst_acc)\n",
    "        train_accuracies.append(trn_acc)\n",
    "        test_accuracies.append(tst_acc)\n",
    "\n",
    "        nrms= [np.linalg.norm(weights['h1'].eval()),np.linalg.norm(weights['h2'].eval()), np.linalg.norm(weights['out'].eval()), np.linalg.norm(biases['b1'].eval()),np.linalg.norm(biases['b2'].eval()),np.linalg.norm(biases['out'].eval())]\n",
    "        print nrms\n",
    "        norms.append(nrms)\n",
    "        writer.flush()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:31:51.327745",
     "start_time": "2016-06-14T16:31:48.392102"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:31:51.529806",
     "start_time": "2016-06-14T16:31:51.329495"
    },
    "collapsed": false
   },
   "source": [
    "plt.plot(train_accuracies)\n",
    "plt.plot(test_accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:31:51.535330",
     "start_time": "2016-06-14T16:31:51.531173"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.max(test_accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number test errors is"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:32:07.217220",
     "start_time": "2016-06-14T16:32:07.212804"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "10000*(1.0-np.max(test_accuracies))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:32:40.544667",
     "start_time": "2016-06-14T16:32:40.541309"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "print norms[0]\n",
    "print norms[-1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:33:49.528886",
     "start_time": "2016-06-14T16:33:49.525098"
    }
   },
   "source": [
    "#### What is the best way to implement the cutoff ?\n",
    "\n",
    "by the norm of the weights at each layer ?\n",
    "\n",
    "see also:  http://keras.io/constraints/\n",
    "\n",
    "Does this just reflect the loss decreasing ?\n",
    "\n",
    "Can we simply bound the norm?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-14T16:57:05.122672",
     "start_time": "2016-06-14T16:57:04.910441"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(6):\n",
    "    plt.plot([nrm[i] for nrm in norms])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}