train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/root/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import argparse\n",
    "import os\n",
    "import numpy as np\n",
    "from preprocessing import parse_annotation\n",
    "import json\n",
    "from keras.utils import multi_gpu_model\n",
    "from frontend import YOLO\n",
    "\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"]=\"PCI_BUS_ID\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_path = \"test/config.json\"\n",
    "with open(config_path) as config_buffer:    \n",
    "    config = json.loads(config_buffer.read())\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = config['env']['gpu']\n",
    "gpus = max(1, len(config['env']['gpu'].split(\",\")))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seen labels:{'RBC': 3854}\n",
      "Given labels:['RBC']\n",
      "Overlap labels:{'RBC'}\n",
      "(13, 13)\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 416, 416, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "model_1 (Model)                 (None, 13, 13, 1024) 50547936    input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv_23 (Conv2D)                (None, 13, 13, 30)   30750       model_1[1][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "reshape_1 (Reshape)             (None, 13, 13, 5, 6) 0           conv_23[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 1, 1, 1, 3, 4 0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 13, 13, 5, 6) 0           reshape_1[0][0]                  \n",
      "                                                                 input_2[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 50,578,686\n",
      "Trainable params: 50,558,014\n",
      "Non-trainable params: 20,672\n",
      "__________________________________________________________________________________________________\n",
      "Epoch 1/200\n",
      "3/4 [=====================>........] - ETA: 6s - loss: 0.5136 - acc: 0.1895  \n",
      " ### loss: 0.25166132756405407, acc: 0.19167488482263353, val_loss: 0.22632036209106446, val_acc: 0.016449704021215438 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 28s 2s/step - loss: 0.2064 - acc: 0.1921 - val_loss: 0.2263 - val_acc: 0.0164\n",
      "Epoch 2/200\n",
      "3/4 [=====================>........] - ETA: 0s - loss: 0.1069 - acc: 0.1396 \n",
      " ### loss: 0.09027312654587957, acc: 0.17736275659667122, val_loss: 0.34567384123802186, val_acc: 0.02087278086692095 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 9s 527ms/step - loss: 0.0874 - acc: 0.1839 - val_loss: 0.3457 - val_acc: 0.0209\n",
      "Epoch 3/200\n",
      "3/4 [=====================>........] - ETA: 0s - loss: 0.0753 - acc: 0.1677 \n",
      " ### loss: 0.0688217356801033, acc: 0.1527613393134541, val_loss: 0.2834055721759796, val_acc: 0.04869822412729263 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 10s 531ms/step - loss: 0.0677 - acc: 0.1502 - val_loss: 0.2834 - val_acc: 0.0487\n",
      "Epoch 4/200\n",
      "3/4 [=====================>........] - ETA: 0s - loss: 0.0605 - acc: 0.1865 \n",
      " ### loss: 0.05553425889876154, acc: 0.1692595345278581, val_loss: 0.14041551649570466, val_acc: 0.1227810651063919 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 10s 535ms/step - loss: 0.0547 - acc: 0.1663 - val_loss: 0.1404 - val_acc: 0.1228\n",
      "Epoch 5/200\n",
      "3/4 [=====================>........] - ETA: 0s - loss: 0.0482 - acc: 0.1762 \n",
      " ### loss: 0.04303090481294526, acc: 0.17018408825000128, val_loss: 0.058691290020942685, val_acc: 0.18229289948940278 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 9s 510ms/step - loss: 0.0421 - acc: 0.1691 - val_loss: 0.0587 - val_acc: 0.1823\n",
      "Epoch 6/200\n",
      "3/4 [=====================>........] - ETA: 0s - loss: 0.0371 - acc: 0.1672 \n",
      " ### loss: 0.03316556651973062, acc: 0.16558185468117395, val_loss: 0.035132527351379395, val_acc: 0.17738165557384492 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 10s 539ms/step - loss: 0.0325 - acc: 0.1653 - val_loss: 0.0351 - val_acc: 0.1774\n",
      "Epoch 7/200\n",
      "3/4 [=====================>........] - ETA: 0s - loss: 0.0298 - acc: 0.1976 \n",
      " ### loss: 0.027055917100773916, acc: 0.18069115612241957, val_loss: 0.028810468316078187, val_acc: 0.16032544374465943 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 9s 505ms/step - loss: 0.0266 - acc: 0.1778 - val_loss: 0.0288 - val_acc: 0.1603\n",
      "Epoch 8/200\n",
      "3/4 [=====================>........] - ETA: 0s - loss: 0.0245 - acc: 0.1676 \n",
      " ### loss: 0.02339770717339383, acc: 0.16770217070976892, val_loss: 0.0236772283911705, val_acc: 0.1603550285100937 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 10s 530ms/step - loss: 0.0232 - acc: 0.1677 - val_loss: 0.0237 - val_acc: 0.1604\n",
      "Epoch 9/200\n",
      "3/4 [=====================>........] - ETA: 0s - loss: 0.0213 - acc: 0.1886 \n",
      " ### loss: 0.02110838486502568, acc: 0.18696992016500896, val_loss: 0.02093815617263317, val_acc: 0.19924556016921996 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 9s 507ms/step - loss: 0.0211 - acc: 0.1867 - val_loss: 0.0209 - val_acc: 0.1992\n",
      "Epoch 10/200\n",
      "3/4 [=====================>........] - ETA: 0s - loss: 0.0213 - acc: 0.2016 \n",
      " ### loss: 0.01966121503048473, acc: 0.18435650815566382, val_loss: 0.019591163098812103, val_acc: 0.17059171795845032 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 10s 529ms/step - loss: 0.0194 - acc: 0.1814 - val_loss: 0.0196 - val_acc: 0.1706\n",
      "Epoch 11/200\n",
      "3/4 [=====================>........] - ETA: 0s - loss: 0.0190 - acc: 0.1828 \n",
      " ### loss: 0.018641508287853666, acc: 0.18609467397133508, val_loss: 0.3237072877585888, val_acc: 0.15630177557468414 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 9s 527ms/step - loss: 0.0186 - acc: 0.1867 - val_loss: 0.3237 - val_acc: 0.1563\n",
      "Epoch 12/200\n",
      "3/4 [=====================>........] - ETA: 0s - loss: 0.4559 - acc: 0.1663 \n",
      " ### loss: 0.2754108069671525, acc: 0.18526051855749553, val_loss: 0.4067183375358582, val_acc: 0.24915680289268494 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 9s 506ms/step - loss: 0.2442 - acc: 0.1885 - val_loss: 0.4067 - val_acc: 0.2492\n",
      "Epoch 13/200\n",
      "3/4 [=====================>........] - ETA: 0s - loss: 0.1940 - acc: 0.1724 \n",
      " ### loss: 0.1844649480448829, acc: 0.1506328077779876, val_loss: 0.3259760081768036, val_acc: 0.24306212961673737 ### \n",
      ", \n",
      "18/4 [=======================================================================================================================================] - 9s 507ms/step - loss: 0.1828 - acc: 0.1469 - val_loss: 0.3260 - val_acc: 0.2431\n",
      "Epoch 00013: early stopping\n",
      "Loss:  <keras.callbacks.History object at 0x7f2efe888198>\n"
     ]
    }
   ],
   "source": [
    "\n",
    "###############################\n",
    "#   Parse the annotations \n",
    "###############################\n",
    "\n",
    "# parse annotations of the training set\n",
    "train_imgs, train_labels = parse_annotation(config['train']['train_annot_folder'], \n",
    "                                            config['train']['train_image_folder'], \n",
    "                                            config['model']['labels'], \".jpg\")\n",
    "\n",
    "# parse annotations of the validation set, if any, otherwise split the training set\n",
    "if os.path.exists(config['valid']['valid_annot_folder']):\n",
    "    valid_imgs, valid_labels = parse_annotation(config['valid']['valid_annot_folder'], \n",
    "                                                config['valid']['valid_image_folder'], \n",
    "                                                config['model']['labels'], \".jpg\")\n",
    "else:\n",
    "    train_valid_split = int(0.8*len(train_imgs))\n",
    "    np.random.shuffle(train_imgs)\n",
    "    \n",
    "    valid_imgs = train_imgs[train_valid_split:]\n",
    "    train_imgs = train_imgs[:train_valid_split]\n",
    "\n",
    "\n",
    "overlap_labels = set(config['model']['labels']).intersection(set(train_labels.keys()))\n",
    "\n",
    "print ('Seen labels:{}'.format(train_labels))\n",
    "print ('Given labels:{}'.format(config['model']['labels']))\n",
    "print ('Overlap labels:{}'.format(overlap_labels))    \n",
    "\n",
    "if len(overlap_labels) < len(config['model']['labels']):\n",
    "    print ('Some labels have no images! Please revise the list of labels in the config.json file!')\n",
    "else:\n",
    "    ###############################\n",
    "    #   Construct the model \n",
    "    ###############################\n",
    "\n",
    "    yolo = YOLO(architecture        = config['model']['architecture'],\n",
    "                input_size          = config['model']['input_size'], \n",
    "                labels              = config['model']['labels'], \n",
    "                max_box_per_image   = config['model']['max_box_per_image'],\n",
    "                anchors             = config['model']['anchors'],\n",
    "                gpus                = gpus)\n",
    "\n",
    "    ###############################\n",
    "    #   Load the pretrained weights (if any) \n",
    "    ############################### \n",
    "    \n",
    "    if os.path.exists(config['train']['pretrained_weights']):\n",
    "       print (\"Loading pre-trained weights in {}\".format(config['train']['pretrained_weights']))\n",
    "       yolo.load_weights(config['train']['pretrained_weights'])\n",
    "\n",
    "    ###############################\n",
    "    #   Start the training process \n",
    "    ###############################\n",
    "    #from IPython.core.debugger import Pdb; Pdb().set_trace()\n",
    "    yolo.train(train_imgs         = train_imgs,\n",
    "              valid_imgs         = valid_imgs,\n",
    "              train_times        = config['train']['train_times'],\n",
    "              valid_times        = config['valid']['valid_times'],\n",
    "              nb_epoch           = config['train']['nb_epoch'], \n",
    "              learning_rate      = config['train']['learning_rate'], \n",
    "              batch_size         = config['train']['batch_size'],\n",
    "              warmup_bs          = config['train']['warmup_batches'],\n",
    "              object_scale       = config['train']['object_scale'],\n",
    "              no_object_scale    = config['train']['no_object_scale'],\n",
    "              coord_scale        = config['train']['coord_scale'],\n",
    "              class_scale        = config['train']['class_scale'],\n",
    "              saved_weights_name = config['train']['saved_weights_name'],\n",
    "              debug              = config['train']['debug'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}