2-Layer-DK-AlignMNiST-100runs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-04-20T21:52:04.548207",
     "start_time": "2016-04-20T21:52:04.544771"
    }
   },
   "source": [
    "## 2 Layer MLP for Dark Knoweldge Experiments w/AlignMNIST\n",
    "\n",
    "75 epochs of RMSProp\n",
    "\n",
    "Eval extra data first, on same size network\n",
    "\n",
    "100 runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-17T09:38:45.623523",
     "start_time": "2016-06-17T09:38:44.320664"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-17T09:38:45.630691",
     "start_time": "2016-06-17T09:38:45.624935"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "%run augmentmnist.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-17T09:38:45.635447",
     "start_time": "2016-06-17T09:38:45.632192"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_epochs = 75\n",
    "batch_size = 125\n",
    "\n",
    "learning_rate = 0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-17T09:38:48.533125",
     "start_time": "2016-06-17T09:38:48.528434"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Network Parameters\n",
    "n_hidden_1 = 800 # 1st layer num features\n",
    "n_hidden_2 = 800 # 2nd layer num features\n",
    "n_input = 784 # MNIST data input (img shape: 28*28)\n",
    "n_classes = 10 # MNIST total classes (0-9 digits)\n",
    "\n",
    "std_0  = np.sqrt(6.0/(n_input+n_hidden_1))\n",
    "std_h1 = np.sqrt(6.0/(n_hidden_1+n_hidden_2))\n",
    "std_h2 = np.sqrt(6.0/(n_hidden_2+n_classes))\n",
    "\n",
    "logfile = \"2-layer-dlk-alignmnist\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-17T09:38:48.721466",
     "start_time": "2016-06-17T09:38:48.715778"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# tf Graph input\n",
    "x = tf.placeholder(\"float\", [None, n_input])\n",
    "y = tf.placeholder(\"float\", [None, n_classes])\n",
    "\n",
    "xt = tf.placeholder(\"float\", [None, n_input])\n",
    "yt = tf.placeholder(\"float\", [None, n_classes])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-17T09:38:48.929238",
     "start_time": "2016-06-17T09:38:48.923948"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def multilayer_perceptron(_X, _weights, _biases):\n",
    "    layer_1 = tf.nn.relu(tf.add(tf.matmul(_X, _weights['h1']), _biases['b1'])) \n",
    "    layer_2 = tf.nn.relu(tf.add(tf.matmul(layer_1, _weights['h2']), _biases['b2'])) \n",
    "    return tf.matmul(layer_2, weights['out']) + biases['out']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-17T09:38:49.647851",
     "start_time": "2016-06-17T09:38:49.613657"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Store layers weight & bias\n",
    "weights = {\n",
    "    'h1': tf.Variable(tf.random_normal([n_input, n_hidden_1], stddev=std_0)),\n",
    "    'h2': tf.Variable(tf.random_normal([n_hidden_1, n_hidden_2], stddev=std_h1)),\n",
    "    'out': tf.Variable(tf.random_normal([n_hidden_2, n_classes], stddev=std_h2))\n",
    "}\n",
    "biases = {\n",
    "    'b1': tf.Variable(tf.random_normal([n_hidden_1], stddev=0.1)),\n",
    "    'b2': tf.Variable(tf.random_normal([n_hidden_2], stddev=0.01)),\n",
    "    'out': tf.Variable(tf.random_normal([n_classes], stddev=0.001))\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-17T09:38:49.847915",
     "start_time": "2016-06-17T09:38:49.835526"
    },
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "mlp = multilayer_perceptron(x, weights, biases )\n",
    "mlp_test = multilayer_perceptron(xt, weights, biases )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-17T09:38:50.445168",
     "start_time": "2016-06-17T09:38:50.435323"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cost =  tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(mlp, y)) \n",
    "gs = tf.get_variable(\"global_step\",[],trainable=False,initializer=tf.constant_initializer(0))\n",
    "lr = tf.constant(learning_rate) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-05-04T21:57:37.534602",
     "start_time": "2016-05-04T21:57:37.532739"
    },
    "collapsed": true
   },
   "source": [
    "### Original MNIST Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-17T09:38:52.586345",
     "start_time": "2016-06-17T09:38:51.649236"
    },
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets(\".\", one_hot=True)\n",
    "trX_0, trY_0 = mnist.train.images, mnist.train.labels\n",
    "teX_0, teY_0 = mnist.test.images, mnist.test.labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2016-06-18T15:52:46.204Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "test_accuracies = []\n",
    "train_accuracies = []\n",
    "for i in range(25):\n",
    "    alignmnist = AlignMNIST()\n",
    "    train_op = tf.contrib.layers.optimize_loss(cost, global_step=gs, learning_rate=lr,optimizer=\"RMSProp\")\n",
    "\n",
    "    pred = tf.equal(tf.argmax(mlp, 1), tf.argmax(y, 1)) # Count correct predictions\n",
    "    train_acc_op = tf.reduce_mean(tf.cast(pred, \"float\"))  # Cast boolean to float to average\n",
    "\n",
    "    test_pred = tf.equal(tf.argmax(mlp_test, 1), tf.argmax(yt, 1)) # Count correct predictions\n",
    "    test_acc_op = tf.reduce_mean(tf.cast(test_pred, \"float\"))  # Cast boolean to float to average\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "        \n",
    "        tf.initialize_all_variables().run()\n",
    "\n",
    "        for epoch in range(num_epochs):\n",
    "            trX, trY = alignmnist.next_epoch()\n",
    "\n",
    "            for start, end in zip(range(0, len(trX), batch_size), range(batch_size, len(trX), batch_size)):\n",
    "                sess.run(train_op, feed_dict={x: trX[start:end], y: trY[start:end]})\n",
    "\n",
    "            trn_acc, tst_acc = sess.run([train_acc_op, test_acc_op], feed_dict={x: trX, y: trY, xt: teX_0, yt: teY_0})\n",
    "            \n",
    "            train_accuracies.append(trn_acc)\n",
    "            test_accuracies.append(tst_acc)\n",
    "     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-18T21:40:54.408566",
     "start_time": "2016-06-19T04:39:41.822Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-18T21:40:54.408837",
     "start_time": "2016-06-19T04:39:41.993Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.plot(train_accuracies)\n",
    "plt.plot(test_accuracies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-18T21:40:54.409330",
     "start_time": "2016-06-19T04:39:42.172Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "np.max(test_accuracies)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number test errors is"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2016-06-18T21:40:54.409788",
     "start_time": "2016-06-19T04:39:43.430Z"
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "10000*(1.0-np.max(test_accuracies))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### this sets the baseline improvement, with better data, from 146 to 103\n",
    "\n",
    "I expect aggressive dropout to improve from 103 to ~70\n",
    "\n",
    "and i have yet to figure out how to make this happen\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}