Cifar10-darknet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cifar 10 - Darknet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.conv_learner import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = Path(\"data/cifar10/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.makedirs(PATH, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats = (np.array([ 0.4914 ,  0.48216,  0.44653]), np.array([ 0.24703,  0.24349,  0.26159]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 256\n",
    "sz = 32"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfms = tfms_from_stats(stats, sz, aug_tfms=[RandomFlip()], pad=sz//8)  # fastai uses reflection padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = ImageClassifierData.from_paths(PATH, val_name='test', tfms=tfms, bs=bs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Architecture"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**The conv layer does *not* need a bias because it is directly followed by a BatchNorm layer.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ConvLayer(n_in, n_out, kernel_size=3, stride=1):\n",
    "    return nn.Sequential(\n",
    "        nn.Conv2d(n_in, n_out, kernel_size=kernel_size, bias=False, stride=stride, padding=kernel_size//2),\n",
    "        nn.BatchNorm2d(n_out, momentum=0.01),\n",
    "        nn.LeakyReLU(negative_slope=0.1, inplace=True))\n",
    "\n",
    "# the 'inplace' saves memory\n",
    "# works with dropout too, activation functions and \n",
    "# arithmetic operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Bottleneck layer:**\n",
    "The 'Wide Resnet' paper hints that something like this is better/faster becaue more computation in parallel:\n",
    "\n",
    "```\n",
    "self.conv1 = ConvLayer(n_in, n_in, kernel_size=1)\n",
    "self.conv2 = ConvLayer(n_in, n_in, kernel_size=3)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResidualLayer(nn.Module):\n",
    "    def __init__(self, n_in):  # n_in == n_out\n",
    "        super().__init__()\n",
    "        self.conv1 = ConvLayer(n_in, n_in//2, kernel_size=1)  # basically n//2 weighted averages\n",
    "        self.conv2 = ConvLayer(n_in//2, n_in, kernel_size=3)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return x.add(self.conv2(self.conv1(x)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`make_group_layer` does the following:\n",
    " \n",
    " 1. The ConvLayer doubles the number of filters (and if stride=2 reduces the hight and width by 2)\n",
    " 1. The ConvLayer is followed by `n_blocks` residual layers which don't change the dimensionality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DarkNet(nn.Module):\n",
    "    def make_group_layer(self, ch_in, n_blocks, stride=1):\n",
    "        return [ConvLayer(ch_in, ch_in * 2, stride=stride)] + [(ResidualLayer(ch_in * 2)) for i in range(n_blocks)]\n",
    "    \n",
    "    def __init__(self, n_blocks, n_classes, nf = 32):\n",
    "        super().__init__()\n",
    "        layers = [ConvLayer(3, nf, kernel_size=3, stride=1)]\n",
    "        \n",
    "        for i, nb in enumerate(n_blocks):\n",
    "            layers += self.make_group_layer(nf, nb, stride=2 - (i==1))  # cifar10 imgs are small, can't half grid size every layer\n",
    "            nf *= 2\n",
    "        layers += [nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(nf, n_classes)]\n",
    "        \n",
    "        self.layers = nn.Sequential(*layers)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.layers(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Important notice: When using `nn.AdaptiveAvgPool` instead of `nn.AvgPool` you are *not* tied to a specific input image size for any fully convolutional architecture like resnets or VGG!**\n",
    "\n",
    "**Also the nice way utilized here to create a model: Make a list of layers and then use `nn.Sequential(*layers)`.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = DarkNet([1, 2, 4, 6, 3], n_classes=10, nf=32) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = nn.DataParallel(model, device_ids=None)  # if you have several GPUs use device_ids=[1, 2, 3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 1.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = ConvLearner.from_model_data(model, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.crit = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn.metrics = [accuracy]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd = 1e-4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "edc1415f9dfc412ba47a68f2d53f61ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=30), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss   accuracy                   \n",
      "    0      2.181148   2.967308   0.1161    \n",
      "    1      1.793812   1.927621   0.2715                     \n",
      "    2      1.560586   1.824395   0.3395                     \n",
      "    3      1.404661   1.678231   0.4158                     \n",
      "    4      1.218635   1.307029   0.5204                     \n",
      "    5      1.075651   1.404319   0.512                      \n",
      "    6      0.943258   1.132617   0.5939                      \n",
      "    7      0.858313   1.017639   0.638                       \n",
      "    8      0.772648   1.209958   0.5769                      \n",
      "    9      0.730512   2.093357   0.4441                      \n",
      "    10     0.686704   1.054698   0.6551                      \n",
      "    11     0.6303     0.978388   0.6768                      \n",
      "    12     0.614091   0.8449     0.7186                      \n",
      "    13     0.581461   0.713726   0.757                       \n",
      "    14     0.546164   0.729345   0.7531                      \n",
      "    15     0.520433   0.942237   0.6976                      \n",
      "    16     0.499694   0.916055   0.7063                      \n",
      "    17     0.478876   0.872791   0.7248                      \n",
      "    18     0.448665   1.160576   0.6403                      \n",
      "    19     0.43121    0.787396   0.7366                      \n",
      "    20     0.407684   0.822959   0.7347                      \n",
      "    21     0.357649   0.548874   0.8114                      \n",
      "    22     0.306851   0.558311   0.8035                      \n",
      "    23     0.253047   0.40758    0.8626                      \n",
      "    24     0.200201   0.354597   0.8828                      \n",
      "    25     0.176933   0.330506   0.8909                      \n",
      "    26     0.153635   0.288811   0.9048                      \n",
      "    27     0.142314   0.290564   0.9065                      \n",
      "    28     0.125174   0.272764   0.9125                      \n",
      "    29     0.104408   0.266702   0.9129                      \n",
      "\n",
      "CPU times: user 1h 44min 24s, sys: 13min 26s, total: 1h 57min 51s\n",
      "Wall time: 19min 51s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.2667015944480896, 0.9129]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%time learn.fit(lr, 1, wds=wd, cycle_len=30, use_clr_beta=(20, 20, 0.95, 0.85))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**The model in the notebook `Resnet from scratch - CIFAR 10.ipynb` reached only an accuracy of 0.8369%!**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:deeplearning]",
   "language": "python",
   "name": "conda-env-deeplearning-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}