experiments/sign_detector/test_sign_detector.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test and visualize sign detector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from PIL import Image\n",
    "from ast import literal_eval\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for auto-reloading external modules\n",
    "# see http://stackoverflow.com/questions/1907993/autoreload-of-modules-in-ipython\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "relative_path = '../../'\n",
    "# ensure that parent path is on the python path in order to have all packages available\n",
    "import sys, os\n",
    "parent_path = os.path.join(os.getcwd(), relative_path)\n",
    "parent_path = os.path.realpath(parent_path)  # os.path.abspath(...)\n",
    "sys.path.insert(0, parent_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tobias/.virtualenvs/pytorch/local/lib/python2.7/site-packages/cryptography/hazmat/primitives/constant_time.py:26: CryptographyDeprecationWarning: Support for your Python version is deprecated. The next version of cryptography will remove support. Please upgrade to a 2.7.x release that supports hmac.compare_digest as soon as possible.\n",
      "  utils.DeprecatedIn23,\n"
     ]
    }
   ],
   "source": [
    "from lib.datasets.cunei_dataset_segments import CuneiformSegments, get_segment_meta\n",
    "from lib.models.trained_model_loader import get_fpn_ssd_net\n",
    "from lib.detection.run_gen_ssd_detection import gen_ssd_detections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "collections = ['test']   # e.g. train test saa06\n",
    "only_annotated = True\n",
    "only_assigned = True\n",
    "\n",
    "# store detections for re-use\n",
    "save_detections = False\n",
    "\n",
    "# show detections\n",
    "show_detections = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_version = 'v191ft01' \n",
    "\n",
    "arch_type = 'mobile'  # resnet, mobile\n",
    "arch_opt = 1\n",
    "width_mult = 0.625  # 0.5 0.625 0.75\n",
    "\n",
    "crop_shape = [600, 600]\n",
    "tile_shape = [600, 600]\n",
    "\n",
    "num_classes = 240\n",
    "\n",
    "with_64 = False \n",
    "create_bg_class = False \n",
    "with_4_aspects = False  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='complconf'>Config Completeness</a>\n",
    "\n",
    "[Jump to results](#results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_min_score_thresh = 0.01  # 0.01 0.05\n",
    "test_nms_thresh = 0.5 \n",
    "\n",
    "eval_ovthresh = 0.5 # 0.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FPNSSD(\n",
      "  (fpn): MobileNetV2FPN(\n",
      "    (features): Sequential(\n",
      "      (0): Sequential(\n",
      "        (0): Conv2d(1, 20, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
      "        (1): BatchNorm2d(20, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): ReLU6(inplace)\n",
      "      )\n",
      "      (1): MobileBlock(\n",
      "        (mobile_block): Sequential(\n",
      "          (0): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(20, 20, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(20, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(20, 20, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=20, bias=False)\n",
      "              (4): BatchNorm2d(20, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(20, 10, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(10, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (2): MobileBlock(\n",
      "        (mobile_block): Sequential(\n",
      "          (0): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(10, 60, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(60, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(60, 60, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=60, bias=False)\n",
      "              (4): BatchNorm2d(60, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(60, 15, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(15, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "          (1): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(15, 90, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(90, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(90, 90, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=90, bias=False)\n",
      "              (4): BatchNorm2d(90, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(90, 15, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(15, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (3): MobileBlock(\n",
      "        (mobile_block): Sequential(\n",
      "          (0): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(15, 90, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(90, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(90, 90, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=90, bias=False)\n",
      "              (4): BatchNorm2d(90, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(90, 20, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(20, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "          (1): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(20, 120, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(120, 120, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=120, bias=False)\n",
      "              (4): BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(120, 20, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(20, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "          (2): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(20, 120, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(120, 120, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=120, bias=False)\n",
      "              (4): BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(120, 20, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(20, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (4): MobileBlock(\n",
      "        (mobile_block): Sequential(\n",
      "          (0): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(20, 120, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(120, 120, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), groups=120, bias=False)\n",
      "              (4): BatchNorm2d(120, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(120, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "          (1): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(240, 240, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=240, bias=False)\n",
      "              (4): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(240, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "          (2): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(240, 240, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=240, bias=False)\n",
      "              (4): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(240, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "          (3): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(240, 240, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=240, bias=False)\n",
      "              (4): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(240, 40, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (5): MobileBlock(\n",
      "        (mobile_block): Sequential(\n",
      "          (0): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(40, 240, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(240, 240, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=240, bias=False)\n",
      "              (4): BatchNorm2d(240, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(240, 60, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(60, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "          (1): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(60, 360, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(360, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(360, 360, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=360, bias=False)\n",
      "              (4): BatchNorm2d(360, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(360, 60, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(60, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "          (2): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(60, 360, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(360, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(360, 360, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=360, bias=False)\n",
      "              (4): BatchNorm2d(360, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(360, 60, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(60, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (6): MobileBlock(\n",
      "        (mobile_block): Sequential(\n",
      "          (0): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(60, 360, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(360, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(360, 360, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=360, bias=False)\n",
      "              (4): BatchNorm2d(360, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(360, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "          (1): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(100, 600, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(600, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(600, 600, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=600, bias=False)\n",
      "              (4): BatchNorm2d(600, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(600, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "          (2): InvertedResidual(\n",
      "            (conv): Sequential(\n",
      "              (0): Conv2d(100, 600, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (1): BatchNorm2d(600, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (2): ReLU6(inplace)\n",
      "              (3): Conv2d(600, 600, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), groups=600, bias=False)\n",
      "              (4): BatchNorm2d(600, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "              (5): ReLU6(inplace)\n",
      "              (6): Conv2d(600, 100, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "              (7): BatchNorm2d(100, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (7): Sequential(\n",
      "        (0): Conv2d(100, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
      "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "        (2): ReLU6(inplace)\n",
      "      )\n",
      "      (8): AvgPool2d(kernel_size=7, stride=1, padding=0)\n",
      "    )\n",
      "    (conv6): Conv2d(512, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))\n",
      "    (toplayer): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1))\n",
      "  )\n",
      "  (loc_head): Sequential(\n",
      "    (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (1): ReLU(inplace)\n",
      "    (2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (3): ReLU(inplace)\n",
      "    (4): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (5): ReLU(inplace)\n",
      "    (6): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (7): ReLU(inplace)\n",
      "    (8): Conv2d(256, 48, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  )\n",
      "  (cls_head): Sequential(\n",
      "    (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (1): ReLU(inplace)\n",
      "    (2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (3): ReLU(inplace)\n",
      "    (4): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (5): ReLU(inplace)\n",
      "    (6): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "    (7): ReLU(inplace)\n",
      "    (8): Conv2d(256, 2880, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "fpnssd_net = get_fpn_ssd_net(model_version, device, arch_type, with_64, arch_opt, width_mult, \n",
    "                             relative_path, num_classes, num_c=1, rnd_init_model=False)\n",
    "\n",
    "print(fpnssd_net)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(torch.Size([1, 15360, 4]), torch.Size([1, 15360, 240]))\n"
     ]
    }
   ],
   "source": [
    "### Test net\n",
    "loc_preds, cls_preds = fpnssd_net(torch.randn(1, 1, 1024, 1024).to(device))\n",
    "print(loc_preds.size(), cls_preds.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Prepare dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Setup dataset spanning 3 collections with 4465 annotations [67 segments, 67 indices]\n"
     ]
    }
   ],
   "source": [
    "dataset = CuneiformSegments(collections=collections, relative_path=relative_path, \n",
    "                            only_annotated=only_annotated, only_assigned=only_assigned, preload_segments=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "### Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train:   5%|▌         | 1/19 [00:00<00:15,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('P334921', 'Obv')\n",
      "mAP 0.7926 | global AP: 0.7473 | mAP (align): 0.8859\n",
      "total_tp: 22 | total_fp: 17 [46] | acc: 0.56\n",
      "('P334921', 'Rev')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  11%|█         | 2/19 [00:01<00:08,  1.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mAP 0.7143 | global AP: 0.7286 | mAP (align): 1.0\n",
      "total_tp: 7 | total_fp: 1 [9] | acc: 0.88\n",
      "('P334863', 'Obv')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/tobias/Dropbox/NeuralNets/caffe_workspace/pycaffe/cuneiform-sign-detection/lib/evaluations/sign_evaluator.py:184: RuntimeWarning: invalid value encountered in divide\n",
      "  return num_tp, num_fp, num_fp_global, num_tp / float(num_tp + num_fp)\n",
      "\r",
      "train:  16%|█▌        | 3/19 [00:01<00:06,  2.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mAP 0.0 | global AP: 0.0 | mAP (align): nan\n",
      "total_tp: 0 | total_fp: 0 [2] | acc: nan\n",
      "('P334831', 'Rev')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  21%|██        | 4/19 [00:01<00:05,  2.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mAP 0.8409 | global AP: 0.7323 | mAP (align): 0.881\n",
      "total_tp: 27 | total_fp: 38 [74] | acc: 0.42\n",
      "('P334831', 'Obv')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  26%|██▋       | 5/19 [00:01<00:05,  2.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mAP 0.9274 | global AP: 0.8946 | mAP (align): 0.9518\n",
      "total_tp: 59 | total_fp: 55 [97] | acc: 0.52\n",
      "('P334892', 'Rev')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  32%|███▏      | 6/19 [00:02<00:04,  2.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mAP 0.9236 | global AP: 0.8063 | mAP (align): 0.9236\n",
      "total_tp: 18 | total_fp: 10 [28] | acc: 0.64\n",
      "('P334892', 'Obv')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  37%|███▋      | 7/19 [00:02<00:04,  2.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mAP 0.9667 | global AP: 0.9474 | mAP (align): 0.9667\n",
      "total_tp: 18 | total_fp: 8 [27] | acc: 0.69\n",
      "('P336635', 'Obv')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  42%|████▏     | 8/19 [00:02<00:03,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mAP 0.9061 | global AP: 0.7393 | mAP (align): 0.9061\n",
      "total_tp: 13 | total_fp: 8 [35] | acc: 0.62\n",
      "('P334865', 'Obv')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  47%|████▋     | 9/19 [00:03<00:03,  2.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mAP 0.9157 | global AP: 0.887 | mAP (align): 0.9443\n",
      "total_tp: 40 | total_fp: 34 [52] | acc: 0.54\n",
      "('P334865', 'Rev')\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train:  58%|█████▊    | 11/19 [00:03<00:02,  3.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mAP 0.8139 | global AP: 0.8114 | mAP (align): 0.8879\n",
      "total_tp: 49 | total_fp: 47 [71] | acc: 0.51\n",
      "('P334842', 'Obv')\n",
      "mAP 0.0 | global AP: 0.0 | mAP (align): 0.0\n",
      "total_tp: 0 | total_fp: 0 [0] | acc: 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  63%|██████▎   | 12/19 [00:03<00:02,  3.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('P334848', 'Obv')\n",
      "mAP 0.9074 | global AP: 0.9346 | mAP (align): 0.9074\n",
      "total_tp: 22 | total_fp: 14 [25] | acc: 0.61\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  68%|██████▊   | 13/19 [00:04<00:01,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('P334848', 'Rev')\n",
      "mAP 0.9375 | global AP: 0.8014 | mAP (align): 1.0\n",
      "total_tp: 16 | total_fp: 4 [38] | acc: 0.8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  74%|███████▎  | 14/19 [00:04<00:01,  3.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('P334839', 'Obv')\n",
      "mAP 0.9333 | global AP: 0.8562 | mAP (align): 0.9956\n",
      "total_tp: 22 | total_fp: 17 [52] | acc: 0.56\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  79%|███████▉  | 15/19 [00:04<00:01,  3.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('P334896', 'Obv')\n",
      "mAP 0.9375 | global AP: 0.8586 | mAP (align): 0.9375\n",
      "total_tp: 26 | total_fp: 20 [36] | acc: 0.57\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  84%|████████▍ | 16/19 [00:04<00:00,  3.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('P334836', 'Rev')\n",
      "mAP 1.0 | global AP: 0.9705 | mAP (align): 1.0\n",
      "total_tp: 45 | total_fp: 24 [65] | acc: 0.65\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  89%|████████▉ | 17/19 [00:05<00:00,  3.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('P334894', 'Rev')\n",
      "mAP 1.0 | global AP: 0.8218 | mAP (align): 1.0\n",
      "total_tp: 15 | total_fp: 9 [39] | acc: 0.62\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train:  95%|█████████▍| 18/19 [00:05<00:00,  3.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('P334894', 'Obv')\n",
      "mAP 0.8727 | global AP: 0.8457 | mAP (align): 0.8727\n",
      "total_tp: 41 | total_fp: 39 [62] | acc: 0.51\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "train: 100%|██████████| 19/19 [00:06<00:00,  3.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('P336178', 'Obv')\n",
      "mAP 0.9375 | global AP: 0.8319 | mAP (align): 0.9375\n",
      "total_tp: 31 | total_fp: 20 [34] | acc: 0.61\n",
      "train | v191ft01\n",
      "RESULTS ON FULL COLLECTION :\n",
      "mAP 0.7739 | global AP: 0.7816 | mAP (align): 0.7958\n",
      "total_tp: 471 | total_fp: 690 [792] | prec: 0.406\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# filter collection dataset - OPTIONAL\n",
    "didx_list = range(len(dataset))\n",
    "didx_list = didx_list[:19]  #19\n",
    "\n",
    "### Generate ssd detections\n",
    "(list_seg_ap, \n",
    " list_seg_name_with_anno) = gen_ssd_detections(didx_list, dataset, collections[0], relative_path, \n",
    "                                               model_version, fpnssd_net, with_64, create_bg_class, device,\n",
    "                                               test_min_score_thresh, test_nms_thresh, eval_ovthresh,\n",
    "                                               save_detections, show_detections, with_4_aspects=with_4_aspects, \n",
    "                                               verbose_mode=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a id='results'>Results</a>\n",
    "\n",
    "[Jump to completeness config](#complconf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump to Results](#results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}