verify_gts.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 基本信息\n",
    "在训练集或者验证集中标注了目标信息，可以计算\n",
    "\n",
    "1、目标的类别\n",
    "\n",
    "2、目标的尺寸位置(y1,x1,y2,x2)\n",
    "\n",
    "3、目标的面积\n",
    "\n",
    "4、宽高比\n",
    "\n",
    "5、目标的多边形面积\n",
    "\n",
    "6、面积比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Mask RCNN\n",
    "# sys.path.append(ROOT_DIR)  # To find local version of the library\n",
    "import os, json\n",
    "import numpy as np\n",
    "import skimage.draw\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "from mrcnn.config import Config\n",
    "from mrcnn import model as modellib, utils\n",
    "from mrcnn import visualize\n",
    "\n",
    "# Path to data set\n",
    "DATASET = \"D:\\Projects\\Mask_RCNN\\my_mask_rcnn\\dataset_1119\\\\\"\n",
    "\n",
    "class MyConfig(Config):\n",
    "    \"\"\"Configuration for training on the toy  dataset.\n",
    "    Derives from the base Config class and overrides some values.\n",
    "    \"\"\"\n",
    "    # Give the configuration a recognizable name\n",
    "    NAME = \"mask\"\n",
    "\n",
    "    # We use a GPU with 12GB memory, which can fit two images.\n",
    "    # Adjust down if you use a smaller GPU.\n",
    "    IMAGES_PER_GPU = 1\n",
    "\n",
    "    # Number of classes (including background)\n",
    "    NUM_CLASSES = 1 + 7  # Background + my\n",
    "\n",
    "    # Number of training steps per epoch\n",
    "    STEPS_PER_EPOCH = 100\n",
    "\n",
    "    # Skip detections with < 90% confidence\n",
    "    DETECTION_MIN_CONFIDENCE = 0.9\n",
    "    BACKBONE = \"resnet101\"\n",
    "    IMAGE_RESIZE_MODE = \"square\"\n",
    "    IMAGE_MIN_DIM = 800  \n",
    "    IMAGE_MAX_DIM = 1024\n",
    "    IMAGE_MIN_SCALE = 0\n",
    "\n",
    "    # BACKBONE = \"resnet50\"\n",
    "    # BACKBONE_STRIDES = [4, 8, 16, 32, 64]\n",
    "   # # BACKBONE_STRIDES = [2, 4, 8, 16, 32]\n",
    "    # RPN_ANCHOR_SCALES = (10, 32, 64, 128, 256)\n",
    "    # RPN_ANCHOR_STRIDE = 2\n",
    "    # RPN_NMS_THRESHOLD = 0.9\n",
    "    # RPN_TRAIN_ANCHORS_PER_IMAGE = 512\n",
    "    # TRAIN_ROIS_PER_IMAGE = 512\n",
    "\n",
    "############################################################\n",
    "#  Dataset\n",
    "############################################################\n",
    "\n",
    "class MyDataset(utils.Dataset):\n",
    "\n",
    "    def print_size(self, poly):\n",
    "        for p in poly:\n",
    "            a = np.array(p['all_points_y'])\n",
    "            height = a.max() - a.min()\n",
    "            a = np.array(p['all_points_x'])\n",
    "            width = a.max() - a.min()\n",
    "            self.areas.append(height * width)\n",
    "            #if height * width < 4096:\n",
    "            #    print(width, height)\n",
    "\n",
    "    def load_my(self, dataset_dir, subset, class_dict):\n",
    "        \"\"\"Load a subset of the My dataset.\n",
    "        dataset_dir: Root directory of the dataset.\n",
    "        subset: Subset to load: train or val\n",
    "        \"\"\"\n",
    "        self.areas = []\n",
    "        # Add classes. We have only one class to add.\n",
    "        for (k, v) in class_dict.items():\n",
    "            self.add_class(\"my\", v, k)\n",
    "\n",
    "        # Train or validation dataset?\n",
    "        assert subset in [\"train\", \"val\"]\n",
    "        dataset_dir = os.path.join(dataset_dir, subset)\n",
    "\n",
    "        # Load annotations\n",
    "        # VGG Image Annotator (up to version 1.6) saves each image in the form:\n",
    "        # { 'filename': '28503151_5b5b7ec140_b.jpg',\n",
    "        #   'regions': {\n",
    "        #       '0': {\n",
    "        #           'region_attributes': {},\n",
    "        #           'shape_attributes': {\n",
    "        #               'all_points_x': [...],\n",
    "        #               'all_points_y': [...],\n",
    "        #               'name': 'polygon'}},\n",
    "        #       ... more regions ...\n",
    "        #   },\n",
    "        #   'size': 100202\n",
    "        # }\n",
    "        # We mostly care about the x and y coordinates of each region\n",
    "        # Note: In VIA 2.0, regions was changed from a dict to a list.\n",
    "        annotations = json.load(open(os.path.join(dataset_dir, \"via_region_data.json\")))\n",
    "        annotations = list(annotations.values())  # don't need the dict keys\n",
    "\n",
    "        # The VIA tool saves images in the JSON even if they don't have any\n",
    "        # annotations. Skip unannotated images.\n",
    "        annotations = [a for a in annotations if a['regions']]\n",
    "\n",
    "        # Add images\n",
    "        for a in annotations:\n",
    "            # Get the x, y coordinaets of points of the polygons that make up\n",
    "            # the outline of each object instance. These are stores in the\n",
    "            # shape_attributes (see json format above)\n",
    "            # The if condition is needed to support VIA versions 1.x and 2.x.\n",
    "            # print(a['regions'])\n",
    "            # print(a['filename'])\n",
    "            if type(a['regions']) is dict:\n",
    "                polygons = [r['shape_attributes'] for r in a['regions'].values()]\n",
    "            else:\n",
    "                if a['regions']:\n",
    "                    class_ids = []\n",
    "                    polygons = []\n",
    "                    for r in a['regions']:\n",
    "                        polygons.append(r['shape_attributes'])\n",
    "                        class_type = r['region_attributes']['type']\n",
    "                        class_ids.append(class_dict[class_type])\n",
    "                        \n",
    "                    self.print_size(polygons)\n",
    "                    # print(class_ids)\n",
    "                        \n",
    "\n",
    "            # load_mask() needs the image size to convert polygons to masks.\n",
    "            # Unfortunately, VIA doesn't include it in JSON, so we must read\n",
    "            # the image. This is only managable since the dataset is tiny.\n",
    "                    image_path = os.path.join(dataset_dir, a['filename'])\n",
    "                    image = skimage.io.imread(image_path)\n",
    "                    height, width = image.shape[:2]\n",
    "\n",
    "                    self.add_image(\n",
    "                        \"my\",\n",
    "                        image_id=a['filename'],  # use file name as a unique image id\n",
    "                        path=image_path,\n",
    "                        width=width, height=height,\n",
    "                        polygons=polygons,\n",
    "                        class_ids=class_ids)\n",
    "        self.areas.sort()\n",
    "        print(np.unique(np.round(np.sqrt(self.areas))))\n",
    "\n",
    "    def load_mask(self, image_id):\n",
    "        \"\"\"Generate instance masks for an image.\n",
    "       Returns:\n",
    "        masks: A bool array of shape [height, width, instance count] with\n",
    "            one mask per instance.\n",
    "        class_ids: a 1D array of class IDs of the instance masks.\n",
    "        \"\"\"\n",
    "        # If not a my dataset image, delegate to parent class.\n",
    "        image_info = self.image_info[image_id]\n",
    "        if image_info[\"source\"] != \"my\":\n",
    "            return super(self.__class__, self).load_mask(image_id)\n",
    "\n",
    "        # Convert polygons to a bitmap mask of shape\n",
    "        # [height, width, instance_count]\n",
    "        info = self.image_info[image_id]\n",
    "        mask = np.zeros([info[\"height\"], info[\"width\"], len(info[\"polygons\"])],\n",
    "                        dtype=np.uint8)\n",
    "        for i, p in enumerate(info[\"polygons\"]):\n",
    "            # Get indexes of pixels inside the polygon and set them to 1\n",
    "            rr, cc = skimage.draw.polygon(p['all_points_y'], p['all_points_x'])\n",
    "            mask[rr, cc, i] = 1\n",
    "        \n",
    "        class_ids = np.array(info['class_ids'])\n",
    "        # Return mask, and array of class IDs of each instance. Since we have\n",
    "        # one class ID only, we return an array of 1s\n",
    "        return mask.astype(np.bool), class_ids.astype(np.int32)\n",
    "\n",
    "    def image_reference(self, image_id):\n",
    "        \"\"\"Return the path of the image.\"\"\"\n",
    "        info = self.image_info[image_id]\n",
    "        if info[\"source\"] == \"my\":\n",
    "            return info[\"path\"]\n",
    "        else:\n",
    "            super(self.__class__, self).image_reference(image_id)\n",
    "\n",
    "def toSquareBox(bbox):\n",
    "    \"\"\"bbox:[y1, x1, y2, x2]\n",
    "    将它按照宽高比转换为正方形\n",
    "    并调整左上和右下的坐标\n",
    "    \n",
    "    正方形的坐标 [y1, x1, y2, x2]\n",
    "    \"\"\"\n",
    "    box_height = bbox[2] - bbox[0]\n",
    "    box_width = bbox[3] - bbox[1]\n",
    "    wh_ratio = box_width / box_height\n",
    "    box_size = box_width / math.sqrt(wh_ratio)\n",
    "    y1 = int(bbox[0] + box_height / 2 - box_size / 2)\n",
    "    y2 = int(y1 + box_size)\n",
    "    x1 = int(bbox[1] + box_width / 2 - box_size / 2)\n",
    "    x2 = int(x1 + box_size)\n",
    "    \n",
    "    return wh_ratio, box_size, box_height * box_width, [y1, x1, y2, x2]\n",
    "\n",
    "target = {\n",
    "    'file' : '',\n",
    "    'gts' : [],\n",
    "    'objs' : []\n",
    "}\n",
    "obj = {\n",
    "    'class_id' : 1,\n",
    "    'bbox' : [],\n",
    "    'bbox_area' : 123,\n",
    "    'wh_ratio' : 0.34,\n",
    "    'mask_area' : 123,\n",
    "    'mask_ratio' : 0.1,\n",
    "    'score' : 1\n",
    "}\n",
    "config = MyConfig()\n",
    "class_dict = {}\n",
    "label_file = open(DATASET+'label.txt')\n",
    "label_lines = label_file.readlines()\n",
    "label_id = 1\n",
    "for label_line in label_lines:\n",
    "    label_line = label_line.replace('\\n', '')\n",
    "    class_dict[label_line] = label_id\n",
    "    label_id = label_id + 1\n",
    "# Validation dataset\n",
    "dataset_val = MyDataset()\n",
    "dataset_val.load_my(DATASET, \"val\", class_dict)\n",
    "dataset_val.prepare()\n",
    "gt_targets = []\n",
    "for image_id in dataset_val.image_ids:\n",
    "    image, image_meta, gt_class_id, gt_box, gt_mask = modellib.load_image_gt(dataset_val, config, image_id, use_mini_mask=False)\n",
    "    print(dataset_val.image_reference(image_id))\n",
    "    scores = np.ones(len(gt_class_id))\n",
    "    one_ids = [x for x in gt_class_id if x == 7]\n",
    "    print(gt_class_id)\n",
    "    id_pos = np.where(gt_class_id == 7)\n",
    "    print(id_pos[0])\n",
    "    gt_class_id_select = gt_class_id[id_pos[0]]\n",
    "    # print(gt_class_id.shape[0])\n",
    "    gt_box_select = gt_box[id_pos[0]]\n",
    "    # print(gt_box.shape[0])\n",
    "    gt_mask_select = gt_mask[:, :, id_pos[0]]\n",
    "    # print(gt_mask.shape)\n",
    "    visualize.display_instances(image, gt_box_select, gt_mask_select, gt_class_id_select, \n",
    "                            dataset_val.class_names, scores,\n",
    "                            title=\"GroundTrue\")\n",
    "    target = {\n",
    "        'file' : dataset_val.image_reference(image_id),\n",
    "        'gts' : [],\n",
    "        'objs' : []\n",
    "    }\n",
    "    square_boxes = []\n",
    "    for i in range(0,len(gt_class_id)):\n",
    "        wh_ratio, box_size, box_area, square_box = toSquareBox(gt_box[i])\n",
    "        mask_area = np.sum(gt_mask[:,:,i]==True)\n",
    "        mask_ratio = mask_area / box_area\n",
    "        obj = {}\n",
    "        obj['class_id'] = gt_class_id[i]\n",
    "        obj['bbox'] = gt_box[i]\n",
    "        obj['bbox_area'] = box_area\n",
    "        obj['wh_ratio'] = wh_ratio\n",
    "        obj['mask_area'] = mask_area\n",
    "        obj['mask_ratio'] = mask_ratio\n",
    "        obj['size'] = box_size\n",
    "        square_boxes.append(square_box)\n",
    "        target['gts'].append(obj)\n",
    "        # print(gt_class_id[i], gt_box[i], box_area, box_ratio, mask_area, mask_ratio)\n",
    "    gt_targets.append(target)\n",
    "    print(square_boxes)\n",
    "    square_boxes = np.array(square_boxes)\n",
    "    square_boxes = square_boxes[id_pos[0]]\n",
    "    visualize.display_instances(image, square_boxes, gt_mask_select, gt_class_id_select, \n",
    "                            dataset_val.class_names, scores,\n",
    "                            title=\"GroundTrue with SquareBox\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 标注目标的正方形尺寸\n",
    "\n",
    "上面已知标注的目标尺寸与宽高比，可以计算目标的正方形尺寸\n",
    "\n",
    "$$Size = \\lceil \\frac {width}{ \\sqrt {wh\\_ratio}} \\rceil$$\n",
    "\n",
    "尺寸$Size$向上取整。\n",
    "\n",
    "宽高比$wh\\_ratio$设定为小数点后一位，进位后舍弃后面的小数。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "# print(gt_targets)\n",
    "for target in gt_targets:\n",
    "    target['gt_wh_ratios'] = []\n",
    "    target['gt_sizes'] = []\n",
    "    for i in range(0, 8):\n",
    "        #target['obj_sizes'] = [obj['size'] for obj in target['objs'] if obj['class_id'] == 7]\n",
    "        #target['obj_wh_ratios'] = [obj['wh_ratio'] for obj in target['objs'] if obj['class_id'] == 7]\n",
    "        target['gt_wh_ratios'].append([gt['wh_ratio'] for gt in target['gts'] if gt['class_id'] == i])#gt_wh_ratios\n",
    "        target['gt_sizes'].append([gt['size'] for gt in target['gts'] if gt['class_id'] == i])#gt_wh_ratiosgt_sizes\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 尺寸、宽高比分布\n",
    "\n",
    "横坐标为尺寸，纵坐标为宽高比，把所有标注目标绘制坐标图\n",
    "\n",
    "主要查看目标的分布情况，猜测目标有7类，又在同一场景，不会有多少点"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.cla()\n",
    "plt.clf()\n",
    "plt.close()\n",
    "for i in range(0, 8):\n",
    "    for target in gt_targets:\n",
    "        # plt.title(target['file'])\n",
    "        #print('标注目标尺寸：', np.sort(target['obj_sizes']))\n",
    "        #print('标注目标宽高比：', np.sort(target['obj_wh_ratios']))\n",
    "        plt.plot(target['gt_sizes'][i], target['gt_wh_ratios'][i], 'ro')\n",
    "        #plt.plot(target['obj_sizes'], target['obj_wh_ratios'], 'go')\n",
    "        plt.axis([0, 512, 0, 2])\n",
    "    plt.title(i)\n",
    "    plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 目标检测数据\n",
    "\n",
    "对数据集，如验证集（val）、测试集（test）等都可以，每张图片检测后把结果收集。\n",
    "\n",
    "检测结果：类别、ROIS、目标框面积、宽高比、掩码面积、面积比、得分"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 目标框\n",
    "\n",
    "计算GT的bbox与检测的bbox的IoU。\n",
    "\n",
    "2个目标怎么匹配？IoU最大的那个"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 覆盖测试\n",
    "\n",
    "人工不可能把所有目标的尺寸和宽高比逐个设置。现实做法是设置几个代表性的尺寸，如32、64、128。代表性的宽高比，如0.5、1、2\n",
    "\n",
    "代表性数据不可能把所有目标都覆盖（IOU）\n",
    "\n",
    "根据上述坐标图的数据分布情况，设置代表性尺寸、宽高比，在每个目标上标识最大的交并比（IOU）。\n",
    "\n",
    "最大交并比（IOU）：理想值就是目标本身，但是以理想值为代表值，必然造成其它目标的交并比（IOU）下降。\n",
    "\n",
    "三维坐标图：尺寸、宽高比、IOU\n",
    "\n",
    "尺寸：锚框的正方形尺寸，应该从GT目标中挑选，把尺寸差值小于阈值的目标归为一组，将平均值作为锚框尺寸。目前网络设计，最多只有5个尺寸设置\n",
    "\n",
    "宽高比：锚框的，也应该从GT目标中挑选。IoU：对所有目标的平均IoU，\n",
    "\n",
    "理想状态是IOU值足够高（0~1），足以用于目标的过滤。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 从小到大排序GT目标框尺寸\n",
    "\n",
    "# 查找GT目标框尺寸的分组位置，即组内尺寸接近"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 代表性的尺寸、宽高比挑选（实验性）\n",
    "\n",
    "人工设置代表值达到IOU理想状态也不现实。。。\n",
    "\n",
    "怎么让计算机挑选代表值？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}