train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dependecies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Image \n",
    "import cv2\n",
    "from sklearn.preprocessing import normalize\n",
    "from albumentations import Compose, ShiftScaleRotate, RandomGamma\n",
    "\n",
    "# Machine Learning\n",
    "from tensorflow.keras.layers import *\n",
    "from tensorflow.keras.optimizers import SGD\n",
    "from tensorflow.keras.models import load_model, Model\n",
    "from tensorflow.keras.callbacks import Callback, ModelCheckpoint, TensorBoard\n",
    "\n",
    "import tensorflow.keras.backend as K\n",
    "\n",
    "# Helper libraries\n",
    "import os\n",
    "import time\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import confusion_matrix, accuracy_score, precision_recall_fscore_support, classification_report"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "SHAPE      = (256, 256, 1) # target image shape\n",
    "BATCH_SIZE = 24   # training batch size\n",
    "N_SPLITS   = 3    # k-fold splits\n",
    "EPOCHS     = 100  # training epochs\n",
    "SEED       = 1881 # Used for reproduction\n",
    "\n",
    "DIR             = 'data/OCT2017/' # DATA Location Directory\n",
    "MODEL_SAVE_PATH = \"models/\"       # K-Fold Model Path\n",
    "LOGS_PATH       = \"logs/\"         # Tensorboard Records Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.listdir(DIR)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OCT_TrainingDataGenerator:\n",
    "    \n",
    "    \"\"\"\n",
    "    input_shape --> TUPLE.wanted image size\n",
    "    batch_size  --> INT.yielding data size for every iteration\n",
    "    orders      --> LIST.which images will be used. max=len(all_images). it can be used for K-fold(CV).\n",
    "    DIR         --> STR.DIR directory must consists \"test\", \"train\" dirs.\n",
    "    augment     --> BOOL. \n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, input_shape, batch_size, orders, DIR, augment=True):\n",
    "        self.SHAPE       = input_shape\n",
    "        self.BATCH_SIZE  = batch_size\n",
    "        self.arr         = orders\n",
    "        self.aug         = augment\n",
    "        \n",
    "        self.DIR         = DIR\n",
    "        self.SUBDIRS     = ['train']\n",
    "        self.SUB_SUBDIRS = ['CNV', \"DME\", \"DRUSEN\", \"NORMAL\"] \n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.get_img_paths())\n",
    "    \n",
    "    def get_img_paths(self):\n",
    "        # Get all training image paths.\n",
    "        arr = []\n",
    "        for x in self.SUBDIRS:\n",
    "            for y in self.SUB_SUBDIRS:\n",
    "                for im in os.listdir(os.path.join(self.DIR,x,y)):\n",
    "                    if \".jpeg\" in im:\n",
    "                        arr.append(os.path.join(self.DIR,x,y,im))\n",
    "        return arr\n",
    "\n",
    "    def get_img(self, img_path):\n",
    "        img = cv2.imread(img_path)\n",
    "        return np.array(img)\n",
    "    \n",
    "    def augmenting(self, img):\n",
    "        if self.aug:\n",
    "            augment = Compose([ShiftScaleRotate(p=0.5, shift_limit=0.01, scale_limit=0., rotate_limit=10),\n",
    "                               RandomGamma(p=0.4)])  \n",
    "        else:\n",
    "            augment = Compose([])  \n",
    "\n",
    "        img = augment(image=img)['image']\n",
    "        return img\n",
    "    \n",
    "    def resize_and_normalize(self, img):\n",
    "        img = cv2.resize(img, SHAPE[:2])\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) # OpenCV loads image as BGR, now we should convert it to grayscale.\n",
    "        img = normalize(img) # normalize the image in 0-1 range for faster training.\n",
    "        return np.expand_dims(img, axis=2)\n",
    "    \n",
    "    def data_generator(self):\n",
    "        img_paths = np.array(self.get_img_paths())\n",
    "        np.random.shuffle(img_paths)\n",
    "        \n",
    "        while True:\n",
    "            x = np.empty((self.BATCH_SIZE,)+self.SHAPE, dtype=np.float16)\n",
    "            y = np.empty((self.BATCH_SIZE, 4), dtype=np.float16)\n",
    "            \n",
    "            batch = np.random.choice(self.arr, self.BATCH_SIZE)\n",
    "\n",
    "            for ix, id_ in enumerate(batch):\n",
    "                # x\n",
    "                img_path = img_paths[id_]\n",
    "                img = self.get_img(img_path)\n",
    "                augmented_img = self.augmenting(img)\n",
    "                img = self.resize_and_normalize(augmented_img)\n",
    "                  \n",
    "                # y \n",
    "                if 'CNV' in img_path:\n",
    "                    label = [1,0,0,0]\n",
    "                elif 'DME' in img_path:\n",
    "                    label = [0,1,0,0]\n",
    "                elif 'DRUSEN' in img_path:\n",
    "                    label = [0,0,1,0]\n",
    "                elif 'NORMAL' in img_path:\n",
    "                    label = [0,0,0,1]\n",
    "                    \n",
    "                # Store the values    \n",
    "                x[ix] = img\n",
    "                y[ix] = label\n",
    "\n",
    "            yield x,y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Testing data generator library\n",
    "dataset = OCT_TrainingDataGenerator(SHAPE, 1, len(range(10)), DIR=DIR, augment=True)\n",
    "\n",
    "for ix, data in enumerate(dataset.data_generator()):\n",
    "    x, y = data\n",
    "    print(x)\n",
    "    print(x.shape)\n",
    "    print(y)\n",
    "    print(y.shape)\n",
    "    \n",
    "    if ix==0:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# credit: https://stackoverflow.com/a/45305384\n",
    "\n",
    "def recall(y_true, y_pred):\n",
    "    \"\"\"Recall metric.\n",
    "\n",
    "    Only computes a batch-wise average of recall.\n",
    "\n",
    "    Computes the recall, a metric for multi-label classification of\n",
    "    how many relevant items are selected.\n",
    "    \"\"\"\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "    recall = true_positives / (possible_positives + K.epsilon())\n",
    "    return recall\n",
    "\n",
    "def precision(y_true, y_pred):\n",
    "    \"\"\"Precision metric.\n",
    "\n",
    "    Only computes a batch-wise average of precision.\n",
    "\n",
    "    Computes the precision, a metric for multi-label classification of\n",
    "    how many selected items are relevant.\n",
    "    \"\"\"\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "    precision = true_positives / (predicted_positives + K.epsilon())\n",
    "    return precision\n",
    "\n",
    "def f1(y_true, y_pred):\n",
    "    precisionx = precision(y_true, y_pred)\n",
    "    recallx = recall(y_true, y_pred)\n",
    "    return 2*((precisionx*recallx)/(precisionx+recallx+K.epsilon()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Rate Scheduler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# credit: https://gist.github.com/jeremyjordan/5a222e04bb78c242f5763ad40626c452\n",
    "\n",
    "class SGDRScheduler(Callback):\n",
    "    '''Cosine annealing learning rate scheduler with periodic restarts.\n",
    "    # Usage\n",
    "        ```python\n",
    "            schedule = SGDRScheduler(min_lr=1e-5,\n",
    "                                     max_lr=1e-2,\n",
    "                                     steps_per_epoch=np.ceil(epoch_size/batch_size),\n",
    "                                     lr_decay=0.9,\n",
    "                                     cycle_length=5,\n",
    "                                     mult_factor=1.5)\n",
    "            model.fit(X_train, Y_train, epochs=100, callbacks=[schedule])\n",
    "        ```\n",
    "    # Arguments\n",
    "        min_lr: The lower bound of the learning rate range for the experiment.\n",
    "        max_lr: The upper bound of the learning rate range for the experiment.\n",
    "        steps_per_epoch: Number of mini-batches in the dataset. Calculated as `np.ceil(epoch_size/batch_size)`. \n",
    "        lr_decay: Reduce the max_lr after the completion of each cycle.\n",
    "                  Ex. To reduce the max_lr by 20% after each cycle, set this value to 0.8.\n",
    "        cycle_length: Initial number of epochs in a cycle.\n",
    "        mult_factor: Scale epochs_to_restart after each full cycle completion.\n",
    "    # References\n",
    "        Blog post: jeremyjordan.me/nn-learning-rate\n",
    "        Original paper: http://arxiv.org/abs/1608.03983\n",
    "    '''\n",
    "    def __init__(self,\n",
    "                 min_lr,\n",
    "                 max_lr,\n",
    "                 steps_per_epoch,\n",
    "                 lr_decay=1,\n",
    "                 cycle_length=10,\n",
    "                 mult_factor=2):\n",
    "\n",
    "        self.min_lr = min_lr\n",
    "        self.max_lr = max_lr\n",
    "        self.lr_decay = lr_decay\n",
    "\n",
    "        self.batch_since_restart = 0\n",
    "        self.next_restart = cycle_length\n",
    "\n",
    "        self.steps_per_epoch = steps_per_epoch\n",
    "\n",
    "        self.cycle_length = cycle_length\n",
    "        self.mult_factor = mult_factor\n",
    "\n",
    "        self.history = {}\n",
    "\n",
    "    def clr(self):\n",
    "        '''Calculate the learning rate.'''\n",
    "        fraction_to_restart = self.batch_since_restart / (self.steps_per_epoch * self.cycle_length)\n",
    "        lr = self.min_lr + 0.5 * (self.max_lr - self.min_lr) * (1 + np.cos(fraction_to_restart * np.pi))\n",
    "        return lr\n",
    "\n",
    "    def on_train_begin(self, logs={}):\n",
    "        '''Initialize the learning rate to the minimum value at the start of training.'''\n",
    "        logs = logs or {}\n",
    "        K.set_value(self.model.optimizer.lr, self.max_lr)\n",
    "\n",
    "    def on_batch_end(self, batch, logs={}):\n",
    "        '''Record previous batch statistics and update the learning rate.'''\n",
    "        logs = logs or {}\n",
    "        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n",
    "        for k, v in logs.items():\n",
    "            self.history.setdefault(k, []).append(v)\n",
    "\n",
    "        self.batch_since_restart += 1\n",
    "        K.set_value(self.model.optimizer.lr, self.clr())\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        '''Check for end of current cycle, apply restarts when necessary.'''\n",
    "        if epoch + 1 == self.next_restart:\n",
    "            self.batch_since_restart = 0\n",
    "            self.cycle_length = np.ceil(self.cycle_length * self.mult_factor)\n",
    "            self.next_restart += self.cycle_length\n",
    "            self.max_lr *= self.lr_decay\n",
    "            self.best_weights = self.model.get_weights()\n",
    "\n",
    "    def on_train_end(self, logs={}):\n",
    "        '''Set weights to the values from the end of the most recent cycle for best performance.'''\n",
    "        self.model.set_weights(self.best_weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# credit: https://github.com/titu1994/keras-squeeze-excite-network/blob/master/se.py\n",
    "\n",
    "def squeeze_excite_block(input, ratio=16):\n",
    "    ''' Create a squeeze-excite block\n",
    "    Args:\n",
    "        input: input tensor\n",
    "        filters: number of output filters\n",
    "        k: width factor\n",
    "    Returns: a keras tensor\n",
    "    '''\n",
    "    init = input\n",
    "    channel_axis = 1 if K.image_data_format() == \"channels_first\" else -1\n",
    "    filters = init.shape[channel_axis]\n",
    "    se_shape = (1, 1, filters)\n",
    "\n",
    "    se = GlobalAveragePooling2D()(init)\n",
    "    se = Reshape(se_shape)(se)\n",
    "    se = Dense(filters // ratio, activation='relu', kernel_initializer='he_normal', use_bias=False)(se)\n",
    "    se = Dense(filters, activation='sigmoid', kernel_initializer='he_normal', use_bias=False)(se)\n",
    "\n",
    "    if K.image_data_format() == 'channels_first':\n",
    "        se = Permute((3, 1, 2))(se)\n",
    "\n",
    "    x = multiply([init, se])\n",
    "    return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model():\n",
    "    \n",
    "    dropRate = 0.3\n",
    "    \n",
    "    init = Input(SHAPE)\n",
    "    x = Conv2D(32, (3, 3), activation=None, padding='same')(init) \n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = Conv2D(32, (3, 3), activation=None, padding='same')(x) \n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x1 = MaxPooling2D((2,2))(x)\n",
    "    \n",
    "    x = Conv2D(64, (3, 3), activation=None, padding='same')(x1)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = squeeze_excite_block(x)\n",
    "    x = Conv2D(64, (5, 5), activation=None, padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = squeeze_excite_block(x)\n",
    "    x = Conv2D(64, (3, 3), activation=None, padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x2 = MaxPooling2D((2,2))(x)\n",
    "    \n",
    "    x = Conv2D(128, (3, 3), activation=None, padding='same')(x2)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = squeeze_excite_block(x)\n",
    "    x = Conv2D(128, (2, 2), activation=None, padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = squeeze_excite_block(x)\n",
    "    x = Conv2D(128, (3, 3), activation=None, padding='same')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x3 = MaxPooling2D((2,2))(x)\n",
    "    \n",
    "    ginp1 = UpSampling2D(size=(2, 2), interpolation='bilinear')(x1)\n",
    "    ginp2 = UpSampling2D(size=(4, 4), interpolation='bilinear')(x2)\n",
    "    ginp3 = UpSampling2D(size=(8, 8), interpolation='bilinear')(x3)\n",
    "    \n",
    "    concat = Concatenate()([ginp1, ginp2, ginp3]) \n",
    "    gap = GlobalAveragePooling2D()(concat)\n",
    "    \n",
    "    x = Dense(256, activation=None)(gap)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "    x = Dropout(dropRate)(x)\n",
    "    \n",
    "    x = Dense(256, activation=None)(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Activation('relu')(x)\n",
    "\n",
    "    x = Dense(4, activation='softmax')(x)\n",
    "    \n",
    "    model = Model(init, x)\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=SGD(lr=1e-3, momentum=0.9), metrics=['acc', precision, recall, f1])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = create_model()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "kf = KFold(n_splits=N_SPLITS, random_state=SEED, shuffle=True)\n",
    "\n",
    "dataset = OCT_TrainingDataGenerator(SHAPE, BATCH_SIZE, 0, DIR, augment=False)\n",
    "\n",
    "for ix, (train_index, test_index) in enumerate(kf.split(range(len(dataset)))):\n",
    "    print(\"** FOLD {} **\".format(ix))       \n",
    "    \n",
    "    tg = OCT_TrainingDataGenerator(SHAPE, BATCH_SIZE, train_index, DIR, augment=True)  # Create training data generator with augmentation is true.\n",
    "    vg = OCT_TrainingDataGenerator(SHAPE, BATCH_SIZE, test_index , DIR, augment=False) # Create validation data generator with augmentation is false.\n",
    "        \n",
    "    schedule = SGDRScheduler(min_lr=1e-6,\n",
    "                             max_lr=1e-3,\n",
    "                             steps_per_epoch=np.ceil(EPOCHS/BATCH_SIZE),\n",
    "                             lr_decay=0.8,\n",
    "                             cycle_length=10,\n",
    "                             mult_factor=1.)\n",
    "\n",
    "    model_ckpt = os.path.join(MODEL_SAVE_PATH,\"heysaw_fold_\"+str(ix)+\".h5\")\n",
    "    callbacks = [ModelCheckpoint(model_ckpt, monitor='val_loss', mode='min', verbose=1, save_best_only=True, save_weights_only=False),\n",
    "                 TensorBoard(log_dir=os.path.join(LOGS_PATH,'log_'+str(ix)), update_freq='epoch'), schedule] \n",
    "                                               \n",
    "    model.fit_generator(tg.data_generator(),\n",
    "                        steps_per_epoch=len(train_index)//BATCH_SIZE,\n",
    "                        epochs=EPOCHS,\n",
    "                        verbose=2,\n",
    "                        validation_data=vg.data_generator(),\n",
    "                        validation_steps=len(test_index)//BATCH_SIZE,\n",
    "                        callbacks=callbacks)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_test_data():\n",
    "    # Load test images' paths.\n",
    "    arr = []\n",
    "    for dir_ in ['test/']:\n",
    "        for y in ['CNV', \"DME\", \"DRUSEN\", \"NORMAL\"]:\n",
    "            for im in os.listdir(os.path.join(DIR,dir_,y)):\n",
    "                arr.append(os.path.join(DIR,dir_,y,im))\n",
    "    \n",
    "    # Get and Store test images and labels.\n",
    "    x = np.empty((len(arr),)+SHAPE, dtype=np.float16)\n",
    "    y = np.empty((len(arr), 4), dtype=np.float16)\n",
    "    \n",
    "    for ix, path in tqdm(enumerate(arr)):\n",
    "        img = np.array(cv2.imread(path))\n",
    "        img = cv2.resize(img, SHAPE[:2])\n",
    "        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "        img = normalize(img)\n",
    "        img = np.expand_dims(img, axis=2)\n",
    "\n",
    "        if 'CNV' in path:\n",
    "            label = [1,0,0,0]\n",
    "        elif 'DME' in path:\n",
    "            label = [0,1,0,0]\n",
    "        elif 'DRUSEN' in path:\n",
    "            label = [0,0,1,0]\n",
    "        elif 'NORMAL' in path:\n",
    "            label = [0,0,0,1]\n",
    "            \n",
    "        x[ix] = img\n",
    "        y[ix] = label\n",
    "        \n",
    "    return x, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, y = get_test_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Threshold predictions\n",
    "def threshold_arr(array):\n",
    "    \n",
    "    # Get all predictions from array\n",
    "    # Find \"max confident score location\" in predictions\n",
    "    # Create numpy zeros array\n",
    "    # Just make \"max confident score location\" is 1 in numpy zeros array (other's zero)\n",
    "    # Store thresholded prediction in new array\n",
    "    \n",
    "    new_arr = []\n",
    "    for ix, val in enumerate(array):\n",
    "        loc = np.array(val).argmax(axis=0)\n",
    "        k = list(np.zeros((len(val)), dtype=np.float16))\n",
    "        k[loc]=1\n",
    "        new_arr.append(k)\n",
    "        \n",
    "    return np.array(new_arr, dtype=np.float16)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Best Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load all models [3-fold(CV)]\n",
    "models = []\n",
    "scores = []\n",
    "for i in range(3):\n",
    "    print(\"** MODEL {} **\".format(i))\n",
    "    model = load_model(os.path.join(MODEL_SAVE_PATH,\"heysaw_fold_\"+str(i)+\".h5\"), custom_objects={'f1': f1, 'precision': precision, 'recall': recall})\n",
    "    score = model.evaluate(x, y, verbose=0)\n",
    "    print(score)\n",
    "    scores.append(score[0])\n",
    "    models.append(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Choose best scored model according to minimum loss\n",
    "best_model_index = int(np.array(scores).argmin())\n",
    "model = models[best_model_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Visualize Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm,\n",
    "                          target_names,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=None,\n",
    "                          normalize=True):\n",
    "    \"\"\"\n",
    "    given a sklearn confusion matrix (cm), make a nice plot\n",
    "\n",
    "    Arguments\n",
    "    ---------\n",
    "    cm:           confusion matrix from sklearn.metrics.confusion_matrix\n",
    "\n",
    "    target_names: given classification classes such as [0, 1, 2]\n",
    "                  the class names, for example: ['high', 'medium', 'low']\n",
    "\n",
    "    title:        the text to display at the top of the matrix\n",
    "\n",
    "    cmap:         the gradient of the values displayed from matplotlib.pyplot.cm\n",
    "                  see http://matplotlib.org/examples/color/colormaps_reference.html\n",
    "                  plt.get_cmap('jet') or plt.cm.Blues\n",
    "\n",
    "    normalize:    If False, plot the raw numbers\n",
    "                  If True, plot the proportions\n",
    "\n",
    "    Usage\n",
    "    -----\n",
    "    plot_confusion_matrix(cm           = cm,                  # confusion matrix created by\n",
    "                                                              # sklearn.metrics.confusion_matrix\n",
    "                          normalize    = True,                # show proportions\n",
    "                          target_names = y_labels_vals,       # list of names of the classes\n",
    "                          title        = best_estimator_name) # title of graph\n",
    "\n",
    "    Citiation\n",
    "    ---------\n",
    "    http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n",
    "\n",
    "    \"\"\"\n",
    "    import matplotlib.pyplot as plt\n",
    "    import numpy as np\n",
    "    import itertools\n",
    "\n",
    "    accuracy = np.trace(cm) / float(np.sum(cm))\n",
    "    misclass = 1 - accuracy\n",
    "\n",
    "    if cmap is None:\n",
    "        cmap = plt.get_cmap('Blues')\n",
    "\n",
    "    plt.figure(figsize=(8, 6))\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "\n",
    "    if target_names is not None:\n",
    "        tick_marks = np.arange(len(target_names))\n",
    "        plt.xticks(tick_marks, target_names, rotation=45)\n",
    "        plt.yticks(tick_marks, target_names)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "\n",
    "\n",
    "    thresh = cm.max() / 1.5 if normalize else cm.max() / 2\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        if normalize:\n",
    "            plt.text(j, i, \"{:0.4f}\".format(cm[i, j]),\n",
    "                     horizontalalignment=\"center\",\n",
    "                     color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "        else:\n",
    "            plt.text(j, i, \"{:,}\".format(cm[i, j]),\n",
    "                     horizontalalignment=\"center\",\n",
    "                     color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True Label')\n",
    "    plt.xlabel('Predicted Label\\naccuracy={:0.4f}'.format(accuracy))\n",
    "    plt.savefig(\"stuffs/confusion_matrix.png\", dpi=150)\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "y_preds = threshold_arr(model.predict(x, verbose=0))\n",
    "\n",
    "results = precision_recall_fscore_support(y, y_preds ,average='macro')\n",
    "acc = accuracy_score(y, y_preds)\n",
    "\n",
    "print(\"Accuracy: {}, F1_Score: {}, Precision: {}, Recall: {}\".format(acc, results[2], results[0], results[1]))\n",
    "print(\"\\n\")\n",
    "print(classification_report(y, y_preds))\n",
    "print(\"\\n\")\n",
    "cnf_matrix = confusion_matrix(y.argmax(axis=1), y_preds.argmax(axis=1))\n",
    "\n",
    "plot_confusion_matrix(cm           = cnf_matrix, \n",
    "                      normalize    = False,\n",
    "                      target_names = ['DME', 'CNV', 'DRUSEN', 'NORMAL'],\n",
    "                      title        = \"Confusion Matrix\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## Inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(5): \n",
    "    img = np.array(cv2.imread(DIR+\"test/NORMAL/NORMAL-2434258-1.jpeg\"))\n",
    "    img = cv2.resize(img, SHAPE[:2])\n",
    "    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "    img = normalize(img)\n",
    "    img = np.expand_dims(img, axis=2)\n",
    "    \n",
    "    start = time.time()\n",
    "    prediction = model.predict(np.expand_dims(img, axis=0), batch_size=1)\n",
    "    finish = time.time()\n",
    "    \n",
    "    print(threshold_arr(prediction))\n",
    "    print((finish-start)*1000,\"ms\")\n",
    "    print(\"***\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}