VGG Transfer Learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n",
      "Using gpu device 1: Tesla K20c (CNMeM is disabled, cuDNN 4007)\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "from keras.applications import VGG16\n",
    "from keras.models import Model\n",
    "from keras.layers import Dense, Dropout, Input, Flatten\n",
    "from keras.regularizers import l2, activity_l2,l1\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.image import array_to_img, img_to_array, load_img\n",
    "from keras.applications.vgg16 import preprocess_input\n",
    "from PIL import Image\n",
    "from scipy import misc\n",
    "from keras.optimizers import SGD\n",
    "from os import listdir\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import scipy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getModel( output_dim ):\n",
    "    # output_dim: the number of classes (int)\n",
    "    # return: compiled model (keras.engine.training.Model)\n",
    "    \n",
    "    vgg_model = VGG16( weights='imagenet', include_top=True )\n",
    "    vgg_out = vgg_model.layers[-1].output \n",
    "#     vgg_out = Flatten()(vgg_out)\n",
    "    \n",
    "    vgg_out = Dropout(0.25)(vgg_out)\n",
    "    #Create new transfer learning model    \n",
    "    out = Dense( output_dim, activation=\"softmax\", W_regularizer = l2(0.2))( vgg_out )\n",
    "\n",
    "    tl_model = Model( input=vgg_model.input, output=out )\n",
    "    for layer in tl_model.layers[0:-1]:\n",
    "        layer.trainable = False            \n",
    "\n",
    "    tl_model.compile(loss= \"categorical_crossentropy\", optimizer=\"adagrad\", metrics=[\"acc\"])\n",
    "    print tl_model.summary()\n",
    "    return tl_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# define functions to laod images\n",
    "def loadBatchImages(path,s, nVal = 2):\n",
    "    # return array of images\n",
    "    catList = listdir(path)\n",
    "    loadedImagesTrain = []\n",
    "    loadedLabelsTrain = []\n",
    "    loadedImagesVal = []\n",
    "    loadedLabelsVal = []\n",
    "\n",
    "\n",
    "    for cat in catList[0:256]:\n",
    "        deepPath = path+cat+\"/\"\n",
    "        # if cat == \".DS_Store\": continue\n",
    "        imageList = listdir(deepPath)\n",
    "        indx = 0\n",
    "        for images in imageList[0:s + nVal]:                \n",
    "            img = load_img(deepPath + images)\n",
    "            img = img_to_array(img)\n",
    "            img = misc.imresize(img, (224,224))\n",
    "            img = scipy.misc.imrotate(img,180)\n",
    "            if indx < s:\n",
    "                loadedLabelsTrain.append(int(images[0:3])-1)\n",
    "                loadedImagesTrain.append(img)\n",
    "            else:\n",
    "                loadedLabelsVal.append(int(images[0:3])-1)\n",
    "                loadedImagesVal.append(img)\n",
    "            indx += 1\n",
    "            \n",
    "\n",
    "    return loadedImagesTrain, np_utils.to_categorical(loadedLabelsTrain), \\\n",
    "loadedImagesVal, np_utils.to_categorical(loadedLabelsVal) \n",
    "    \n",
    "def shuffledSet(a, b):\n",
    "    # shuffle the entire dataset\n",
    "    assert np.shape(a)[0] == np.shape(b)[0]\n",
    "    p = np.random.permutation(np.shape(a)[0])\n",
    "    return (a[p], b[p])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "path = \"/mnt/cube/VGG_/256_ObjectCategories/\"    \n",
    "samCat = 32 # number of samples per category\n",
    "\n",
    "data, labels, dataVal, labelsVal = loadBatchImages(path,samCat, nVal = 2)\n",
    "\n",
    "data = preprocess_input(np.float64(data))\n",
    "data = data.swapaxes(1, 3).swapaxes(2, 3)\n",
    "\n",
    "dataVal = preprocess_input(np.float64(dataVal))\n",
    "dataVal = dataVal.swapaxes(1, 3).swapaxes(2, 3)\n",
    "\n",
    "train = shuffledSet(np.asarray(data),labels)\n",
    "val = shuffledSet(np.asarray(dataVal),labelsVal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8192, 3, 224, 224) (512, 3, 224, 224)\n"
     ]
    }
   ],
   "source": [
    "plt.imshow(train[0][0][0])\n",
    "plt.show()\n",
    "print train[0].shape, val[0].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_3 (InputLayer)             (None, 3, 224, 224)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "block1_conv1 (Convolution2D)     (None, 64, 224, 224)  1792        input_3[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "block1_conv2 (Convolution2D)     (None, 64, 224, 224)  36928       block1_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block1_pool (MaxPooling2D)       (None, 64, 112, 112)  0           block1_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block2_conv1 (Convolution2D)     (None, 128, 112, 112) 73856       block1_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block2_conv2 (Convolution2D)     (None, 128, 112, 112) 147584      block2_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block2_pool (MaxPooling2D)       (None, 128, 56, 56)   0           block2_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block3_conv1 (Convolution2D)     (None, 256, 56, 56)   295168      block2_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block3_conv2 (Convolution2D)     (None, 256, 56, 56)   590080      block3_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block3_conv3 (Convolution2D)     (None, 256, 56, 56)   590080      block3_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block3_pool (MaxPooling2D)       (None, 256, 28, 28)   0           block3_conv3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block4_conv1 (Convolution2D)     (None, 512, 28, 28)   1180160     block3_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block4_conv2 (Convolution2D)     (None, 512, 28, 28)   2359808     block4_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block4_conv3 (Convolution2D)     (None, 512, 28, 28)   2359808     block4_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block4_pool (MaxPooling2D)       (None, 512, 14, 14)   0           block4_conv3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block5_conv1 (Convolution2D)     (None, 512, 14, 14)   2359808     block4_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block5_conv2 (Convolution2D)     (None, 512, 14, 14)   2359808     block5_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block5_conv3 (Convolution2D)     (None, 512, 14, 14)   2359808     block5_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block5_pool (MaxPooling2D)       (None, 512, 7, 7)     0           block5_conv3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "flatten (Flatten)                (None, 25088)         0           block5_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "fc1 (Dense)                      (None, 4096)          102764544   flatten[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "fc2 (Dense)                      (None, 4096)          16781312    fc1[0][0]                        \n",
      "____________________________________________________________________________________________________\n",
      "predictions (Dense)              (None, 1000)          4097000     fc2[0][0]                        \n",
      "____________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)              (None, 1000)          0           predictions[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dense_3 (Dense)                  (None, 256)           256256      dropout_3[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 138,613,800\n",
      "Trainable params: 256,256\n",
      "Non-trainable params: 138,357,544\n",
      "____________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "output_dim = 256\n",
    "tl_model = getModel(output_dim) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nb_epoch = 5\n",
    "\n",
    "history = tl_model.fit(train[0], train[1], batch_size = 64, nb_epoch = nb_epoch, validation_data = val, \n",
    "                       shuffle = True)\n",
    "\n",
    "keras.callbacks.EarlyStopping(monitor='val_loss', min_delta = 0, patience = 2, verbose = 0, mode='auto')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "plt.plot(history.history['loss'])\n",
    "plt.plot(history.history['val_loss'])\n",
    "plt.title('Model loss for %d samples/ int' % samCat)\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()\n",
    "\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('model accuracy for %d samples per category' % samCat)\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "input_2 (InputLayer)             (None, 3, 224, 224)   0                                            \n",
      "____________________________________________________________________________________________________\n",
      "block1_conv1 (Convolution2D)     (None, 64, 224, 224)  1792        input_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "block1_conv2 (Convolution2D)     (None, 64, 224, 224)  36928       block1_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block1_pool (MaxPooling2D)       (None, 64, 112, 112)  0           block1_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block2_conv1 (Convolution2D)     (None, 128, 112, 112) 73856       block1_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block2_conv2 (Convolution2D)     (None, 128, 112, 112) 147584      block2_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block2_pool (MaxPooling2D)       (None, 128, 56, 56)   0           block2_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block3_conv1 (Convolution2D)     (None, 256, 56, 56)   295168      block2_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block3_conv2 (Convolution2D)     (None, 256, 56, 56)   590080      block3_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block3_conv3 (Convolution2D)     (None, 256, 56, 56)   590080      block3_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block3_pool (MaxPooling2D)       (None, 256, 28, 28)   0           block3_conv3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block4_conv1 (Convolution2D)     (None, 512, 28, 28)   1180160     block3_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block4_conv2 (Convolution2D)     (None, 512, 28, 28)   2359808     block4_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block4_conv3 (Convolution2D)     (None, 512, 28, 28)   2359808     block4_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block4_pool (MaxPooling2D)       (None, 512, 14, 14)   0           block4_conv3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block5_conv1 (Convolution2D)     (None, 512, 14, 14)   2359808     block4_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "block5_conv2 (Convolution2D)     (None, 512, 14, 14)   2359808     block5_conv1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block5_conv3 (Convolution2D)     (None, 512, 14, 14)   2359808     block5_conv2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "block5_pool (MaxPooling2D)       (None, 512, 7, 7)     0           block5_conv3[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "flatten (Flatten)                (None, 25088)         0           block5_pool[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "fc1 (Dense)                      (None, 4096)          102764544   flatten[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "fc2 (Dense)                      (None, 4096)          16781312    fc1[0][0]                        \n",
      "____________________________________________________________________________________________________\n",
      "predictions (Dense)              (None, 1000)          4097000     fc2[0][0]                        \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 1000)          0           predictions[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 256)           256256      dropout_2[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 138,613,800\n",
      "Trainable params: 256,256\n",
      "Non-trainable params: 138,357,544\n",
      "____________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "tl_model.summary()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}