model3d.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/zhangsiyu/siyu/anaconda3/envs/py36/lib/python3.6/site-packages/sklearn/externals/joblib/__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0,1'\n",
    "\n",
    "import importlib\n",
    "import os\n",
    "import sys\n",
    "import datetime\n",
    "\n",
    "import keras.models as KM\n",
    "import keras.layers as KL\n",
    "from keras.optimizers import Adam\n",
    "from keras.utils import multi_gpu_model\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import util\n",
    "\n",
    "cur_dir = os.path.abspath('.')\n",
    "root_dir = os.path.dirname(cur_dir)\n",
    "sys.path.append(os.path.join(cur_dir, '3DUnetCNN'))\n",
    "\n",
    "import unet3d.my_generator\n",
    "import unet3d.my_augment\n",
    "from unet3d.model import unet_model_3d\n",
    "from unet3d.training import load_old_model, train_model\n",
    "from unet3d.metrics import dice_coefficient_loss, get_label_dice_coefficient_function, dice_coefficient"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "now = datetime.datetime.utcnow() + datetime.timedelta(hours=8)\n",
    "config = dict()\n",
    "config[\"pool_size\"] = (2, 2, 2)  # pool size for the max pooling operations\n",
    "# config[\"image_shape\"] = (144, 144, 144)  # This determines what shape the images will be cropped/resampled to.\n",
    "config[\"patch_shape\"] = (128, 128, 128)  # switch to None to train on the whole image\n",
    "config[\"labels\"] = (1, 5, 31, 32)  # the label numbers on the input image\n",
    "config[\"n_labels\"] = len(config[\"labels\"])\n",
    "# config[\"all_modalities\"] = [\"t1\", \"t1ce\", \"flair\", \"t2\"]\n",
    "# config[\"training_modalities\"] = config[\"all_modalities\"]  # change this if you want to only use some of the modalities\n",
    "# config[\"nb_channels\"] = len(config[\"training_modalities\"])\n",
    "config[\"nb_channels\"] = 1\n",
    "if \"patch_shape\" in config and config[\"patch_shape\"] is not None:\n",
    "    config[\"input_shape\"] = tuple([config[\"nb_channels\"]] + list(config[\"patch_shape\"]))\n",
    "else:\n",
    "    config[\"input_shape\"] = tuple([config[\"nb_channels\"]] + list(config[\"image_shape\"]))\n",
    "config[\"truth_channel\"] = config[\"nb_channels\"]\n",
    "config[\"deconvolution\"] = True  # if False, will use upsampling instead of deconvolution\n",
    "\n",
    "config[\"batch_size\"] = 1\n",
    "config[\"validation_batch_size\"] = 1\n",
    "config[\"n_epochs\"] = 300  # cutoff the training after this many epochs\n",
    "config[\"patience\"] = 10  # learning rate will be reduced after this many epochs if the validation loss is not improving\n",
    "config[\"early_stop\"] = 50  # training will be stopped after this many epochs without the validation loss improving\n",
    "config[\"initial_learning_rate\"] = 0.00005\n",
    "config[\"learning_rate_drop\"] = 0.5  # factor by which the learning rate will be reduced\n",
    "config[\"validation_split\"] = 0.1  # portion of the data that will be used for training\n",
    "config[\"flip\"] = True  # augments the data by randomly flipping an axis during\n",
    "config[\"permute\"] = False  # data shape must be a cube. Augments the data by permuting in various directions\n",
    "config[\"distort\"] = None  # switch to None if you want no distortion\n",
    "config[\"augment\"] = config[\"flip\"] or config[\"distort\"]\n",
    "config[\"validation_patch_overlap\"] = 0  # if > 0, during training, validation patches will be overlapping\n",
    "config[\"training_patch_start_offset\"] = (40, 30, 30)  # randomly offset the first patch index by up to this offset\n",
    "config[\"skip_blank\"] = True  # if True, then patches without any target will be skipped\n",
    "\n",
    "config[\"data_path\"] = os.path.abspath(\"./data/3d/\")\n",
    "config[\"model_file\"] = os.path.abspath(\"./model_unet3d_{}.h5\".format(now.strftime('%Y%m%d_%H%M%S')))\n",
    "# config[\"model_file\"] = os.path.abspath(\"./pre-trained/tumor_segmentation_model.h5\")\n",
    "config[\"training_file\"] = os.path.abspath(\"training_uids.pkl\")\n",
    "config[\"validation_file\"] = os.path.abspath(\"validation_uids.pkl\")\n",
    "config[\"overwrite\"] = True  # If True, will previous files. If False, will use previously written files."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spliting training and validation data ...\n",
      "Creating validation split...\n",
      "build training generator ...\n",
      "build validation generator ...\n",
      "calculating training steps ...\n",
      "counting number of patches from 1293 samples ...\n",
      "Number of training steps:  6812\n",
      "calculating validation steps ...\n",
      "counting number of patches from 143 samples ...\n",
      "Number of validation steps:  739\n"
     ]
    }
   ],
   "source": [
    "importlib.reload(unet3d.my_generator)\n",
    "importlib.reload(unet3d.my_augment)\n",
    "\n",
    "# get training and testing generators\n",
    "train_generator, validation_generator, n_train_steps, n_validation_steps = unet3d.my_generator.get_training_and_validation_generators(\n",
    "    config['data_path'],\n",
    "    batch_size=config[\"batch_size\"],\n",
    "    validation_split=config[\"validation_split\"],\n",
    "    overwrite=config['overwrite'],\n",
    "    validation_keys_file=config[\"validation_file\"],\n",
    "    training_keys_file=config[\"training_file\"],\n",
    "    n_labels=config[\"n_labels\"],\n",
    "    labels=config[\"labels\"],\n",
    "    patch_shape=config[\"patch_shape\"],\n",
    "    validation_batch_size=config[\"validation_batch_size\"],\n",
    "    validation_patch_overlap=config[\"validation_patch_overlap\"],\n",
    "    training_patch_start_offset=config[\"training_patch_start_offset\"],\n",
    "    permute=config[\"permute\"],\n",
    "    augment=config[\"augment\"],\n",
    "    skip_blank=config[\"skip_blank\"],\n",
    "    augment_flip=config[\"flip\"],\n",
    "    augment_distortion_factor=config[\"distort\"],\n",
    "#     calculate_steps=False,\n",
    "    )\n",
    "# n_train_steps, n_validation_steps = 1200, 120"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import pickle\n",
    "# with open(config[\"training_file\"], 'rb') as f:\n",
    "#     training_list = pickle.load(f)\n",
    "# with open(config[\"validation_file\"], 'rb') as f:\n",
    "#     validation_list = pickle.load(f)\n",
    "# print(training_list)\n",
    "# print(validation_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data = next(train_generator)\n",
    "# x, y = data\n",
    "# print(x.shape)\n",
    "# print(y.shape)\n",
    "\n",
    "# util.plot_ct_as_slide(x[0][0], start=0, end=None, interval=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# util.plot_ct_as_slide(y[0][0], start=0, end=None, interval=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if not config['overwrite'] and os.path.exists(config[\"model_file\"]):\n",
    "model = load_old_model('./pre-trained/tumor_segmentation_model.h5')\n",
    "for layer in model.layers:\n",
    "    layer.trainable = False\n",
    "#     if layer.name.startswith('new_') or layer.name.startswith('append_'):\n",
    "#         layer.trainable = True\n",
    "#     else:\n",
    "#         layer.trainable = False\n",
    "    \n",
    "# else:\n",
    "# # instantiate new model\n",
    "# model = unet_model_3d(\n",
    "#     input_shape=config[\"input_shape\"],\n",
    "#     pool_size=config[\"pool_size\"],\n",
    "#     n_labels=config[\"n_labels\"],\n",
    "#     initial_learning_rate=config[\"initial_learning_rate\"],\n",
    "#     deconvolution=config[\"deconvolution\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.summary(line_length=120)\n",
    "\n",
    "# # from keras.utils import plot_model\n",
    "# import keras.utils\n",
    "# importlib.reload(keras.utils)\n",
    "# keras.utils.plot_model(model, to_file='_model.png', show_shapes=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "________________________________________________________________________________________________________________________\n",
      "Layer (type)                                          Output Shape                                    Param #           \n",
      "========================================================================================================================\n",
      "new_input_1 (InputLayer)                              (None, 1, 128, 128, 128)                        0                 \n",
      "________________________________________________________________________________________________________________________\n",
      "new_conv3d_1 (Conv3D)                                 (None, 4, 128, 128, 128)                        112               \n",
      "________________________________________________________________________________________________________________________\n",
      "new_conv3d_2 (Conv3D)                                 (None, 4, 128, 128, 128)                        436               \n",
      "________________________________________________________________________________________________________________________\n",
      "new_maxpooling3d_1 (MaxPooling3D)                     (None, 4, 64, 64, 64)                           0                 \n",
      "________________________________________________________________________________________________________________________\n",
      "model_1 (Model)                                       (None, 64, 64, 64, 64)                          19071520          \n",
      "________________________________________________________________________________________________________________________\n",
      "append_upsampling_1 (Conv3DTranspose)                 (None, 16, 128, 128, 128)                       27664             \n",
      "________________________________________________________________________________________________________________________\n",
      "append_conv3d_1 (Conv3D)                              (None, 8, 128, 128, 128)                        3464              \n",
      "________________________________________________________________________________________________________________________\n",
      "append_conv3d_2 (Conv3D)                              (None, 4, 128, 128, 128)                        868               \n",
      "========================================================================================================================\n",
      "Total params: 19,104,064\n",
      "Trainable params: 32,544\n",
      "Non-trainable params: 19,071,520\n",
      "________________________________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "input_shape = (1,) + tuple(config['patch_shape'])\n",
    "new_input = KL.Input(input_shape, name='new_input_1')\n",
    "new_conv1 = KL.Conv3D(4, (3, 3, 3), padding='same', activation='relu', name='new_conv3d_1')\n",
    "new_conv2 = KL.Conv3D(4, (3, 3, 3), padding='same', activation='relu', name='new_conv3d_2')\n",
    "new_maxpool1 = KL.MaxPool3D(name='new_maxpooling3d_1')\n",
    "input_ = new_conv1(new_input)\n",
    "input_ = new_conv2(input_)\n",
    "input_ = new_maxpool1(input_)\n",
    "\n",
    "mid_model = KM.Model(inputs=model.input, outputs=model.layers[-3].output)\n",
    "out = mid_model(input_)\n",
    "\n",
    "append_upsample1 = KL.Conv3DTranspose(16, (3, 3, 3), strides=(2, 2, 2), padding='same', activation='relu', name='append_upsampling_1')\n",
    "append_conv1 = KL.Conv3D(8, (3, 3, 3), padding='same', activation='relu', name='append_conv3d_1')\n",
    "append_conv2 = KL.Conv3D(4, (3, 3, 3), padding='same', activation='relu', name='append_conv3d_2')\n",
    "out = append_upsample1(out)\n",
    "out = append_conv1(out)\n",
    "out = append_conv2(out)\n",
    "\n",
    "new_model = KM.Model(new_input, out)\n",
    "new_model.summary(line_length=120)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "________________________________________________________________________________________________________________________\n",
      "Layer (type)                           Output Shape               Param #       Connected to                            \n",
      "========================================================================================================================\n",
      "input_1 (InputLayer)                   (None, 4, 64, 64, 64)      0                                                     \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_1 (Conv3D)                      (None, 32, 64, 64, 64)     3488          input_1[0][0]                           \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_1 (Activation)              (None, 32, 64, 64, 64)     0             conv3d_1[0][0]                          \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_2 (Conv3D)                      (None, 64, 64, 64, 64)     55360         activation_1[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_2 (Activation)              (None, 64, 64, 64, 64)     0             conv3d_2[0][0]                          \n",
      "________________________________________________________________________________________________________________________\n",
      "max_pooling3d_1 (MaxPooling3D)         (None, 64, 32, 32, 32)     0             activation_2[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_3 (Conv3D)                      (None, 64, 32, 32, 32)     110656        max_pooling3d_1[0][0]                   \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_3 (Activation)              (None, 64, 32, 32, 32)     0             conv3d_3[0][0]                          \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_4 (Conv3D)                      (None, 128, 32, 32, 32)    221312        activation_3[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_4 (Activation)              (None, 128, 32, 32, 32)    0             conv3d_4[0][0]                          \n",
      "________________________________________________________________________________________________________________________\n",
      "max_pooling3d_2 (MaxPooling3D)         (None, 128, 16, 16, 16)    0             activation_4[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_5 (Conv3D)                      (None, 128, 16, 16, 16)    442496        max_pooling3d_2[0][0]                   \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_5 (Activation)              (None, 128, 16, 16, 16)    0             conv3d_5[0][0]                          \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_6 (Conv3D)                      (None, 256, 16, 16, 16)    884992        activation_5[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_6 (Activation)              (None, 256, 16, 16, 16)    0             conv3d_6[0][0]                          \n",
      "________________________________________________________________________________________________________________________\n",
      "max_pooling3d_3 (MaxPooling3D)         (None, 256, 8, 8, 8)       0             activation_6[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_7 (Conv3D)                      (None, 256, 8, 8, 8)       1769728       max_pooling3d_3[0][0]                   \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_7 (Activation)              (None, 256, 8, 8, 8)       0             conv3d_7[0][0]                          \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_8 (Conv3D)                      (None, 512, 8, 8, 8)       3539456       activation_7[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_8 (Activation)              (None, 512, 8, 8, 8)       0             conv3d_8[0][0]                          \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_transpose_1 (Conv3DTranspose)   (None, 512, 16, 16, 16)    2097664       activation_8[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)            (None, 768, 16, 16, 16)    0             conv3d_transpose_1[0][0]                \n",
      "                                                                                activation_6[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_9 (Conv3D)                      (None, 256, 16, 16, 16)    5308672       concatenate_1[0][0]                     \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_9 (Activation)              (None, 256, 16, 16, 16)    0             conv3d_9[0][0]                          \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_10 (Conv3D)                     (None, 256, 16, 16, 16)    1769728       activation_9[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_10 (Activation)             (None, 256, 16, 16, 16)    0             conv3d_10[0][0]                         \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_transpose_2 (Conv3DTranspose)   (None, 256, 32, 32, 32)    524544        activation_10[0][0]                     \n",
      "________________________________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)            (None, 384, 32, 32, 32)    0             conv3d_transpose_2[0][0]                \n",
      "                                                                                activation_4[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_11 (Conv3D)                     (None, 128, 32, 32, 32)    1327232       concatenate_2[0][0]                     \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_11 (Activation)             (None, 128, 32, 32, 32)    0             conv3d_11[0][0]                         \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_12 (Conv3D)                     (None, 128, 32, 32, 32)    442496        activation_11[0][0]                     \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_12 (Activation)             (None, 128, 32, 32, 32)    0             conv3d_12[0][0]                         \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_transpose_3 (Conv3DTranspose)   (None, 128, 64, 64, 64)    131200        activation_12[0][0]                     \n",
      "________________________________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)            (None, 192, 64, 64, 64)    0             conv3d_transpose_3[0][0]                \n",
      "                                                                                activation_2[0][0]                      \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_13 (Conv3D)                     (None, 64, 64, 64, 64)     331840        concatenate_3[0][0]                     \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_13 (Activation)             (None, 64, 64, 64, 64)     0             conv3d_13[0][0]                         \n",
      "________________________________________________________________________________________________________________________\n",
      "conv3d_14 (Conv3D)                     (None, 64, 64, 64, 64)     110656        activation_13[0][0]                     \n",
      "________________________________________________________________________________________________________________________\n",
      "activation_14 (Activation)             (None, 64, 64, 64, 64)     0             conv3d_14[0][0]                         \n",
      "========================================================================================================================\n",
      "Total params: 19,071,520\n",
      "Trainable params: 0\n",
      "Non-trainable params: 19,071,520\n",
      "________________________________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "mid_model.summary(line_length=120)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# new_model = multi_gpu_model(new_model, gpus=2)\n",
    "new_model.compile(optimizer=Adam(lr=config['initial_learning_rate']), loss=dice_coefficient_loss, metrics=[dice_coefficient])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading pre-trained model\n",
      "________________________________________________________________________________________________________________________\n",
      "Layer (type)                                          Output Shape                                    Param #           \n",
      "========================================================================================================================\n",
      "new_input_1 (InputLayer)                              (None, 1, 128, 128, 128)                        0                 \n",
      "________________________________________________________________________________________________________________________\n",
      "new_conv3d_1 (Conv3D)                                 (None, 4, 128, 128, 128)                        112               \n",
      "________________________________________________________________________________________________________________________\n",
      "new_conv3d_2 (Conv3D)                                 (None, 4, 128, 128, 128)                        436               \n",
      "________________________________________________________________________________________________________________________\n",
      "new_maxpooling3d_1 (MaxPooling3D)                     (None, 4, 64, 64, 64)                           0                 \n",
      "________________________________________________________________________________________________________________________\n",
      "model_1 (Model)                                       (None, 64, 64, 64, 64)                          19071520          \n",
      "________________________________________________________________________________________________________________________\n",
      "append_upsampling_1 (Conv3DTranspose)                 (None, 16, 128, 128, 128)                       27664             \n",
      "________________________________________________________________________________________________________________________\n",
      "append_conv3d_1 (Conv3D)                              (None, 8, 128, 128, 128)                        3464              \n",
      "________________________________________________________________________________________________________________________\n",
      "append_conv3d_2 (Conv3D)                              (None, 4, 128, 128, 128)                        868               \n",
      "========================================================================================================================\n",
      "Total params: 19,104,064\n",
      "Trainable params: 32,544\n",
      "Non-trainable params: 19,071,520\n",
      "________________________________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# new_model = load_old_model('./model_unet3d_20190721_202302.h5')\n",
    "# new_model = multi_gpu_model(new_model, gpus=2)\n",
    "# new_model.summary(line_length=120)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "starting from learning rate 5e-05 ...\n",
      "Epoch 1/300\n",
      "6812/6812 [==============================] - 10938s 2s/step - loss: -0.0306 - dice_coefficient: 0.0306 - val_loss: -0.0467 - val_dice_coefficient: 0.0467\n",
      "Epoch 2/300\n",
      "6812/6812 [==============================] - 11044s 2s/step - loss: -0.0442 - dice_coefficient: 0.0442 - val_loss: -0.0589 - val_dice_coefficient: 0.0589\n",
      "Epoch 3/300\n",
      "3086/6812 [============>.................] - ETA: 1:33:25 - loss: -0.0551 - dice_coefficient: 0.0551"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "IOPub message rate exceeded.\n",
      "The notebook server will temporarily stop sending output\n",
      "to the client in order to avoid crashing it.\n",
      "To change this limit, set the config variable\n",
      "`--NotebookApp.iopub_msg_rate_limit`.\n",
      "\n",
      "Current values:\n",
      "NotebookApp.iopub_msg_rate_limit=1000.0 (msgs/sec)\n",
      "NotebookApp.rate_limit_window=3.0 (secs)\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6812/6812 [==============================] - 10767s 2s/step - loss: -0.0536 - dice_coefficient: 0.0536 - val_loss: -0.0672 - val_dice_coefficient: 0.0672\n",
      "Epoch 4/300\n",
      "6812/6812 [==============================] - 10474s 2s/step - loss: -0.0597 - dice_coefficient: 0.0597 - val_loss: -0.0716 - val_dice_coefficient: 0.0716\n",
      "Epoch 5/300\n",
      "6812/6812 [==============================] - 10270s 2s/step - loss: -0.0631 - dice_coefficient: 0.0631 - val_loss: -0.0740 - val_dice_coefficient: 0.0740\n",
      "Epoch 6/300\n",
      "6812/6812 [==============================] - 10208s 1s/step - loss: -0.0709 - dice_coefficient: 0.0709 - val_loss: -0.0784 - val_dice_coefficient: 0.0784\n",
      "Epoch 7/300\n",
      "6812/6812 [==============================] - 10214s 1s/step - loss: -0.0744 - dice_coefficient: 0.0744 - val_loss: -0.0834 - val_dice_coefficient: 0.0834\n",
      "Epoch 8/300\n",
      "6812/6812 [==============================] - 10211s 1s/step - loss: -0.0763 - dice_coefficient: 0.0763 - val_loss: -0.0804 - val_dice_coefficient: 0.0804\n",
      "Epoch 9/300\n",
      "1469/6812 [=====>........................] - ETA: 2:08:20 - loss: -0.0762 - dice_coefficient: 0.0762"
     ]
    }
   ],
   "source": [
    "# run training\n",
    "train_model(\n",
    "    model=new_model,\n",
    "    model_file=config[\"model_file\"],\n",
    "    training_generator=train_generator,\n",
    "    validation_generator=validation_generator,\n",
    "    steps_per_epoch=n_train_steps,\n",
    "    validation_steps=n_validation_steps,\n",
    "    initial_learning_rate=config[\"initial_learning_rate\"],\n",
    "    learning_rate_drop=config[\"learning_rate_drop\"],\n",
    "    learning_rate_patience=config[\"patience\"],\n",
    "    early_stopping_patience=config[\"early_stop\"],\n",
    "    n_epochs=config[\"n_epochs\"])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}