MNIST_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"; \n",
    "# Set the right GPU here (the default is \"0\")\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "import h5py\n",
    "\n",
    "from keras import backend as K\n",
    "from keras import initializers\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Flatten\n",
    "from keras.optimizers import Adam, SGD\n",
    "from keras.metrics import categorical_accuracy\n",
    "from keras.utils import to_categorical\n",
    "from keras.callbacks import Callback\n",
    "import keras.utils\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_input_data(filename):\n",
    "    input_file = h5py.File(filename, 'r')\n",
    "    x = np.array(input_file.get('images'))\n",
    "    y = np.array(input_file.get('labels'))\n",
    "    input_file.close()\n",
    "    return x, y\n",
    "\n",
    "#load data\n",
    "train_x, train_y = read_input_data('../Data/mnist_train.h5')\n",
    "val_x, val_y = read_input_data('../Data/mnist_val.h5')\n",
    "test_x, test_y = read_input_data('../Data/mnist_test.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(55000, 28, 28, 1)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data downsampling\n",
    "picked_ind_list = []\n",
    "for cat in np.unique(train_y):\n",
    "    cat_ind = np.where(train_y==cat)[0]\n",
    "    cat_pop = len(cat_ind)\n",
    "    # Taking only 1% of each catergory at random\n",
    "    picked_ind = np.random.choice(cat_ind, 20)\n",
    "    for ind in picked_ind:\n",
    "        picked_ind_list.append(ind)\n",
    "train_x = train_x[picked_ind_list, :]\n",
    "train_y = train_y[picked_ind_list]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(200, 28, 28, 1)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Unroll the data\n",
    "train_x = train_x.reshape(train_x.shape[0], -1)\n",
    "val_x = val_x.reshape(val_x.shape[0], -1)\n",
    "test_x = test_x.reshape(test_x.shape[0], -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scaling\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(train_x)\n",
    "train_x_sc, val_x_sc, test_x_sc = scaler.transform(train_x), scaler.transform(val_x), scaler.transform(test_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y_cat = to_categorical(train_y)\n",
    "val_y_cat = to_categorical(val_y)\n",
    "test_y_cat = to_categorical(test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test + validation data are used together to assess the untrained error\n",
    "train_val_x_sc = np.concatenate((train_x_sc, val_x_sc))\n",
    "train_val_y_cat = np.concatenate((train_y_cat, val_y_cat))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# detect the current working directory and print it\n",
    "path = os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Vanilla neural network with 2 hidden layers, aimed for classification task\n",
    "def model_MNIST(n_units1, n_units2, input_dim, output_dim, seed, learning_rate):\n",
    "    \n",
    "    model = Sequential()\n",
    "    model.add(Dense(int(n_units1), \n",
    "                    input_dim=input_dim, \n",
    "                    kernel_initializer=initializers.he_uniform(seed),\n",
    "                    activation='relu'\n",
    "                   )\n",
    "             )\n",
    "\n",
    "    model.add(Dense(int(n_units2),\n",
    "                    kernel_initializer=initializers.he_uniform(seed),\n",
    "                    activation='relu'\n",
    "                   )\n",
    "             )\n",
    "\n",
    "    model.add(Dense(int(output_dim), \n",
    "                    kernel_initializer=initializers.he_uniform(seed),\n",
    "                    activation='softmax'\n",
    "                   ))\n",
    "    # Compile the model\n",
    "    my_adam = Adam(lr=learning_rate)\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=my_adam, metrics=[categorical_accuracy])\n",
    "    return model\n",
    "\n",
    "class IgnoreBeginningSaveBest(Callback):\n",
    "            \"\"\"Save the best weights only after some number of epochs has been trained\n",
    "    \n",
    "            Arguments:\n",
    "            filepath -- where to save the resulting model\n",
    "            best -- the initial value of the training metric (0 for accuracy, Inf for error)\n",
    "            ignore_first_epochs -- how many first epochs to ignore before registering the best validation loss\n",
    "            \n",
    "            \"\"\"\n",
    "\n",
    "            def __init__(self, filepath, best, ignore_first_epochs=0):\n",
    "                super(IgnoreBeginningSaveBest, self).__init__()\n",
    "\n",
    "                self.filepath = filepath\n",
    "                self.ignore_first_epochs = ignore_first_epochs\n",
    "                self.best = best\n",
    "\n",
    "                # best_weights to store the weights at which the minimum loss occurs\n",
    "                self.best_weights = None\n",
    "\n",
    "            def on_train_begin(self, logs=None):\n",
    "                # The epoch the training starts at\n",
    "                self.best_epoch = 0\n",
    "\n",
    "            def on_epoch_end(self, epoch, logs=None):\n",
    "                current = logs.get('val_categorical_accuracy')\n",
    "                if epoch>self.ignore_first_epochs:\n",
    "                    # SWAP arguments HERE if working with ERROR\n",
    "                    # (because accuracy should be maximised, while the error is being minimised)\n",
    "                    if np.less(self.best, current):\n",
    "                        self.best = current\n",
    "                        # Record the best weights if the current result is better\n",
    "                        self.best_weights = self.model.get_weights()\n",
    "                        self.best_epoch = epoch\n",
    "                        # Save the best so far model\n",
    "                        self.model.save(self.filepath)\n",
    "                    \n",
    "            def on_train_end(self, logs=None):\n",
    "                print(\"The model will be based on the epoch #{}\".format(self.best_epoch))\n",
    "                print('Restoring model weights from the end of the best epoch.')\n",
    "                if self.best_weights is not None:\n",
    "                    self.model.set_weights(self.best_weights)\n",
    "                \n",
    "##\n",
    "def tot_parameters(n_units1, n_units2):\n",
    "    return 785*n_units1 + (n_units1+1)*n_units2 + (n_units2+1)*10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Define ranges and parameters\n",
    "n_epochs = 200\n",
    "seed_range = [i for i in range(2)]\n",
    "n_units1_range = [8,16,24,32,48,56,64,96,128,256,512,1024]\n",
    "n_units2_range = [8,16,24,32,48,56,64,96,128,256,512,1024]\n",
    "learning_rate_range = [0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03]\n",
    "\n",
    "# A list with statistics\n",
    "stats = []\n",
    "\n",
    "##\n",
    "for n_units1 in n_units1_range:\n",
    "    for n_units2 in n_units2_range:\n",
    "        for learning_rate in learning_rate_range:\n",
    "            best_acc = 0\n",
    "            for seed in seed_range:\n",
    "                print(\"Running the architecture: [{0}, {1}] Seed: {2}, LR: {3}\".format(n_units1, n_units2, seed, learning_rate))\n",
    "                # Create the directory\n",
    "                seed_dir = \"{0}/Outputs/LR/{1}\".format(os.path.split(path)[0], seed)\n",
    "                output_dir = \"{0}/arch{1}_{2}_lr{3}\".format(seed_dir, n_units1, n_units2, learning_rate)\n",
    "                if not os.path.exists(seed_dir):\n",
    "                    os.makedirs(seed_dir)\n",
    "                if not os.path.exists(output_dir):\n",
    "                    os.makedirs(output_dir)\n",
    "\n",
    "                # Grid search\n",
    "                best_score = -1 # Some small initial value\n",
    "                print(\"Evaluating untrained performance...\")\n",
    "                K.clear_session()  \n",
    "                model = model_MNIST(n_units1,\n",
    "                                    n_units2,\n",
    "                                    input_dim = train_x_sc.shape[1],\n",
    "                                    output_dim = train_y_cat.shape[1],\n",
    "                                    seed=seed,\n",
    "                                    learning_rate=learning_rate\n",
    "                                    )\n",
    "                # Computing untrained accuracy evaluated on a combination of training and validation sets\n",
    "                _, untrained_accuracy = model.evaluate(train_val_x_sc, \n",
    "                                             train_val_y_cat,\n",
    "                                             verbose=0)\n",
    "                print(\"Untrained accuracy is: {0:0.4f}\".format(untrained_accuracy))\n",
    "\n",
    "                print(\"Training the model...\")\n",
    "                ignorebeginning = IgnoreBeginningSaveBest(filepath=output_dir+str(\"/model.hdf5\"),\n",
    "                                                          best=0,\n",
    "                                                          ignore_first_epochs=50)\n",
    "\n",
    "                history = model.fit(train_x_sc,\n",
    "                                    train_y_cat,\n",
    "                                    batch_size = 50,\n",
    "                                    validation_data = (val_x_sc, val_y_cat),\n",
    "                                    epochs=n_epochs,\n",
    "                                    verbose = 0,\n",
    "                                    callbacks = [ignorebeginning]\n",
    "                                   )\n",
    "                training_curve = history.history['categorical_accuracy']\n",
    "                validation_curve = history.history['val_categorical_accuracy']\n",
    "                pd.DataFrame(training_curve).to_csv(output_dir+\"/Training_curve.csv\", index=False)\n",
    "                pd.DataFrame(validation_curve).to_csv(output_dir+\"/Validation_curve.csv\", index=False)\n",
    "                # Evaluate the best trained model on the test set\n",
    "                _, trained_accuracy = model.evaluate(test_x_sc, \n",
    "                                                     test_y_cat, \n",
    "                                                     verbose=0)\n",
    "                best_epoch = ignorebeginning.best_epoch\n",
    "                best_val_acc = ignorebeginning.best\n",
    "                stats.append([n_units1, n_units2, learning_rate, seed, untrained_accuracy, trained_accuracy, best_val_acc, best_epoch])\n",
    "                # Save after every architecure/LR combination\n",
    "                pd.DataFrame(stats).to_csv(\"{0}/Outputs/Stats_test.csv\".format(os.path.split(path)[0]), index=False)\n",
    "##"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}