evaluate_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import math\n",
    "import cv2\n",
    "import numpy as np\n",
    "import utils\n",
    "import tensorflow as tf\n",
    "import tensorflow.contrib.slim as slim\n",
    "from libs.networks.network_factory import get_network_byname\n",
    "from libs import build_rpn, build_fast_rcnn, build_fpn\n",
    "from data.io.read_tfrecord import eval_predict_input_fn\n",
    "from configs.config import Config\n",
    "from libs.visualize import display_instances\n",
    "from libs.label_dict import LABEl_NAME_MAP\n",
    "from tools.test import model_fn\n",
    "\n",
    "%matplotlib inline "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Define the evaluate function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_match(gt_boxes, gt_class_ids,\n",
    "               pred_boxes, pred_class_ids, pred_scores,\n",
    "               iou_threshold=0.5):\n",
    "    \"\"\"Compute Average Precision at a set IoU threshold (default 0.5).\n",
    "\n",
    "    Returns:\n",
    "    gt_match_dict: array for gt has been preded(1 be for has been pred, 0 for don't been pred) \n",
    "    pred_match_dict: array for pred is or not right(1 is right, 0 is wrong)\n",
    "    pred_scores_dict: array for has been sorted.(be correspend to pred_match)\n",
    "\n",
    "    \"\"\"\n",
    "    gt_match_dict = {}\n",
    "    pred_match_dict = {}\n",
    "    pred_scores_dict = {}\n",
    "    # Trim zero padding and sort predictions by score from high to low\n",
    "    # TODO: cleaner to do zero unpadding upstream\n",
    "    gt_boxes = utils.trim_zeros(gt_boxes)\n",
    "    pred_boxes = utils.trim_zeros(pred_boxes)\n",
    "    pred_scores = pred_scores[:pred_boxes.shape[0]]\n",
    "    indices = np.argsort(pred_scores)[::-1]\n",
    "    pred_boxes = pred_boxes[indices]\n",
    "    pred_class_ids = pred_class_ids[indices]\n",
    "    pred_scores = pred_scores[indices]\n",
    "    \n",
    "    # Compute IoU overlaps [pred_boxes, gt_boxes]\n",
    "    overlaps = utils.compute_overlaps(pred_boxes, gt_boxes)\n",
    "    \n",
    "    # Loop through ground truth boxes and find matching predictions\n",
    "    match_count = 0\n",
    "    pred_match = np.zeros([pred_boxes.shape[0]])\n",
    "    gt_match = np.zeros([gt_boxes.shape[0]])\n",
    "    for i in range(len(pred_boxes)):\n",
    "        # Find best matching ground truth box\n",
    "        sorted_ixs = np.argsort(overlaps[i])[::-1]\n",
    "        for j in sorted_ixs:\n",
    "            # If ground truth box is already matched, go to next one\n",
    "            if gt_match[j] == 1:\n",
    "                continue\n",
    "            # If we reach IoU smaller than the threshold, end the loop\n",
    "            iou = overlaps[i, j]\n",
    "            if iou < iou_threshold:\n",
    "                break\n",
    "            # Do we have a match?\n",
    "            if pred_class_ids[i] == gt_class_ids[j]:\n",
    "                match_count += 1\n",
    "                gt_match[j] = 1\n",
    "                pred_match[i] = 1\n",
    "                break\n",
    "    for class_id in range(1,12):\n",
    "        gt_match_dict[class_id] =  gt_match[gt_class_ids==class_id]\n",
    "        pred_match_dict[class_id] = pred_match[pred_class_ids==class_id]\n",
    "        pred_scores_dict[class_id] = pred_scores[pred_class_ids==class_id]\n",
    "    return gt_match_dict, pred_match_dict, pred_scores_dict\n",
    "\n",
    "def compute_class_mAP(single_record):\n",
    "    \"\"\"\n",
    "    single_record: the single result of model(it is a generator)\n",
    "    \n",
    "    all_class_mAP(array): all class AP.\n",
    "    \"\"\"\n",
    "    \n",
    "    all_gt_match_dict = {}\n",
    "    all_pred_match_dict = {}\n",
    "    all_pred_scores_dict = {}\n",
    "    for class_id in range(1,12):\n",
    "        all_gt_match_dict[class_id] = np.array([])\n",
    "        all_pred_match_dict[class_id]= np.array([])\n",
    "        all_pred_scores_dict[class_id]= np.array([])\n",
    "        \n",
    "    while True:\n",
    "        # Load\n",
    "        try:\n",
    "            np_single_sample = next(single_record)\n",
    "            predict_bbox = np.round(np_single_sample[\"predict_bbox\"]) \n",
    "            predict_class_id = np_single_sample[\"predict_class_id\"].astype(np.int32)\n",
    "            predict_scores = np_single_sample[\"predict_scores\"]\n",
    "            gt_bbox = np_single_sample[\"gt_box_labels\"][:, :4]\n",
    "            gt_class_id = np_single_sample[\"gt_box_labels\"][:, 4]\n",
    "            # Run object detection\n",
    "            # Compute AP\n",
    "            gt_match_dict, pred_match_dict, pred_scores_dict =\\\n",
    "                compute_match(gt_bbox, gt_class_id,\n",
    "                              predict_bbox, predict_class_id,\n",
    "                              predict_scores)\n",
    "            for class_id in range(1,12):\n",
    "                all_gt_match_dict[class_id] = np.concatenate([all_gt_match_dict[class_id],gt_match_dict[class_id]])\n",
    "                all_pred_match_dict[class_id]= np.concatenate([all_pred_match_dict[class_id],pred_match_dict[class_id]])\n",
    "                all_pred_scores_dict[class_id]= np.concatenate([all_pred_scores_dict[class_id],pred_scores_dict[class_id]])\n",
    "        except:\n",
    "            break\n",
    "            \n",
    "    all_class_mAP = np.zeros([11])\n",
    "    all_class_AR = np.zeros([11])\n",
    "    for class_id in range(1,12):\n",
    "        gt_match = all_gt_match_dict[class_id]\n",
    "        pred_match = all_pred_match_dict[class_id]\n",
    "        pred_score = all_pred_scores_dict[class_id]\n",
    "        indices = np.argsort(pred_score)[::-1]\n",
    "        pred_scores = pred_score[indices]\n",
    "        pred_match = pred_match[indices]\n",
    "        # Compute precision and recall at each prediction box step\n",
    "        precisions = np.cumsum(pred_match) / (np.arange(len(pred_match)) + 1)\n",
    "        recalls = np.cumsum(pred_match).astype(np.float32) / len(gt_match)\n",
    "\n",
    "        # Pad with start and end values to simplify the math\n",
    "        precisions = np.concatenate([[0], precisions, [0]])\n",
    "        recalls = np.concatenate([[0], recalls, [1]])\n",
    "\n",
    "        # Ensure precision values decrease but don't increase. This way, the\n",
    "        # precision value at each recall threshold is the maximum it can be\n",
    "        # for all following recall thresholds, as specified by the VOC paper.\n",
    "        for i in range(len(precisions) - 2, -1, -1):\n",
    "            precisions[i] = np.maximum(precisions[i], precisions[i + 1])\n",
    "\n",
    "        # Compute mean AP over recall range\n",
    "        indices = np.where(recalls[:-1] != recalls[1:])[0] + 1\n",
    "        mAP = np.sum((recalls[indices] - recalls[indices - 1]) *\n",
    "                     precisions[indices])\n",
    "        all_class_mAP[class_id-1] = mAP\n",
    "        all_class_AR[class_id-1] = recalls[-2]\n",
    "    return all_class_mAP, all_class_AR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define prediction generator "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using config: {'_model_dir': './logs', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': gpu_options {\n",
      "  allow_growth: true\n",
      "}\n",
      "allow_soft_placement: true\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x7f0b7643c160>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n"
     ]
    }
   ],
   "source": [
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"1\"\n",
    "class PConfig(Config):\n",
    "    PER_GPU_IMAGE = 1\n",
    "\n",
    "net_config = PConfig()\n",
    "session_config = tf.ConfigProto()\n",
    "session_config.gpu_options.allow_growth = True\n",
    "session_config.allow_soft_placement = True\n",
    "estimator_config = tf.estimator.RunConfig(model_dir=net_config.MODLE_DIR,\n",
    "                                          session_config=session_config)\n",
    "\n",
    "my_estimator = tf.estimator.Estimator(model_fn,\n",
    "                                      params={\"net_config\": net_config}, \n",
    "                                      config=estimator_config)\n",
    "single_record = my_estimator.predict(input_fn=lambda: \n",
    "                                     eval_predict_input_fn(net_config.DATA_DIR),\n",
    "                                     yield_single_examples=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/userfolder/anaconda3/lib/python3.6/site-packages/scipy/misc/pilutil.py:482: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
      "  if issubdtype(ts, int):\n",
      "/root/userfolder/anaconda3/lib/python3.6/site-packages/scipy/misc/pilutil.py:485: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  elif issubdtype(type(size), float):\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from ./logs/model.ckpt-113179\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n"
     ]
    }
   ],
   "source": [
    "all_mAP, all_AR = compute_class_mAP(single_record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ascus</th>\n",
       "      <td>0.273919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asch</th>\n",
       "      <td>0.067316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lsil</th>\n",
       "      <td>0.417430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hsil</th>\n",
       "      <td>0.400730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scc</th>\n",
       "      <td>0.217877</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>agc</th>\n",
       "      <td>0.545329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trichomonas</th>\n",
       "      <td>0.449804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>candida</th>\n",
       "      <td>0.655251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flora</th>\n",
       "      <td>0.635147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>herps</th>\n",
       "      <td>0.680915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>actinomyces</th>\n",
       "      <td>0.704771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAP</th>\n",
       "      <td>0.458954</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    0\n",
       "ascus        0.273919\n",
       "asch         0.067316\n",
       "lsil         0.417430\n",
       "hsil         0.400730\n",
       "scc          0.217877\n",
       "agc          0.545329\n",
       "trichomonas  0.449804\n",
       "candida      0.655251\n",
       "flora        0.635147\n",
       "herps        0.680915\n",
       "actinomyces  0.704771\n",
       "mAP          0.458954"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "class_names = LABEl_NAME_MAP\n",
    "all_mAP_add = np.concatenate([all_mAP,np.array([np.mean(all_mAP)])])\n",
    "allclass_mAP = pd.DataFrame(all_mAP_add,index=[class_names[i] for i in range(1,12)] + [\"mAP\"])\n",
    "display(allclass_mAP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ascus</th>\n",
       "      <td>0.556604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>asch</th>\n",
       "      <td>0.194030</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>lsil</th>\n",
       "      <td>0.539394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hsil</th>\n",
       "      <td>0.685530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>scc</th>\n",
       "      <td>0.412281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>agc</th>\n",
       "      <td>0.738132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>trichomonas</th>\n",
       "      <td>0.714885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>candida</th>\n",
       "      <td>0.769231</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>flora</th>\n",
       "      <td>0.791667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>herps</th>\n",
       "      <td>0.771429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>actinomyces</th>\n",
       "      <td>0.812500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mAR</th>\n",
       "      <td>0.635062</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    0\n",
       "ascus        0.556604\n",
       "asch         0.194030\n",
       "lsil         0.539394\n",
       "hsil         0.685530\n",
       "scc          0.412281\n",
       "agc          0.738132\n",
       "trichomonas  0.714885\n",
       "candida      0.769231\n",
       "flora        0.791667\n",
       "herps        0.771429\n",
       "actinomyces  0.812500\n",
       "mAR          0.635062"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "class_names = LABEl_NAME_MAP\n",
    "all_AR_add = np.concatenate([all_AR, np.array([np.mean(all_AR)])])\n",
    "allclass_mAR = pd.DataFrame(all_AR_add,index=[class_names[i] for i in range(1,12)] + [\"mAR\"])\n",
    "display(allclass_mAR)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}