update wholebody models and codes (#820)

* Update metrics.py * Update train.py * Update metrics.py * Update metrics.py * update datasets * update utils * update train.py & validate.py * reupload * delete * delete useless files... * delete useless files... * when frei rotate 180 * simple demo * single image demo * finetune * small fix * Add files via upload * small update * Update demo.py * Update demo.py * Add files via upload * Add files via upload * Add files via upload * Update writer.py * Update metrics.py * Update metrics.py * Update coco_det.py * Update coco_wholebody_det.py * Update custom.py * Update halpe_136_det.py * Update halpe_26_det.py * Update file_detector.py * Update writer.py * Update pPose_nms.py * Update criterion.py * Update custom.py * Update criterion.py * Update criterion.py * Update and rename demo.py to demo_api.py * Update halpe_136_det.py * Update coco_wholebody_det.py * Update metrics.py * Update simple_transform.py * Update custom.py * Update custom.py * Update custom.py * Update simple_transform.py * Update demo_api.py * weighted loss for wholebody * Update simple_transform.py * Update simple_transform.py * Update coco_wholebody.py * fix a bug when flip-test is off * update fro combined loss * update train.py for combined loss * update for combined loss * update for combined loss * update for combined loss * update for combined loss * update for combined loss * small update * support combined loss * dcn config for halpe * small fix * small fix * small fix * small fix * update model zoo * update * update model zoo * fix * support halpecocotools * vis thres * update model zoo * ppose_nms * fix * fix * fix a bug * update models * small fix Co-authored-by: Fred Fang <[email protected]>
MVIG-SJTU · Jan 22, 2022 · 5002003 · 5002003
1 parent bcfbc99
commit 5002003
Show file tree

Hide file tree

Showing 50 changed files with 3,297 additions and 432 deletions.
diff --git a/.gitignore b/.gitignore
@@ -114,4 +114,5 @@ data/
 tmp/
 exp/json
 tmp_*/
-example/res/
+example/res/
+data/
diff --git a/alphapose/datasets/__init__.py b/alphapose/datasets/__init__.py
@@ -3,8 +3,24 @@
 from .custom import CustomDataset
 from .mscoco import Mscoco
 from .mpii import Mpii
+from .coco_wholebody import coco_wholebody
+from .coco_wholebody_det import coco_wholebody_det
 from .halpe_26 import Halpe_26
 from .halpe_136 import Halpe_136
 from .halpe_136_det import  Halpe_136_det
 from .halpe_26_det import  Halpe_26_det
-__all__ = ['CustomDataset', 'Halpe_136', 'Halpe_26_det', 'Halpe_136_det', 'Halpe_26', 'Mscoco', 'Mscoco_det', 'Mpii', 'ConcatDataset', 'coco_wholebody', 'coco_wholebody_det']
+from .halpe_coco_wholebody_26 import Halpe_coco_wholebody_26
+from .halpe_coco_wholebody_26_det import Halpe_coco_wholebody_26_det
+from .halpe_coco_wholebody_136 import Halpe_coco_wholebody_136
+from .halpe_coco_wholebody_136_det import Halpe_coco_wholebody_136_det
+from .halpe_68_noface import Halpe_68_noface
+from .halpe_68_noface_det import Halpe_68_noface_det
+from .single_hand import SingleHand
+from .single_hand_det import SingleHand_det
+
+__all__ = ['CustomDataset', 'ConcatDataset', 'Mpii', 'Mscoco', 'Mscoco_det', \
+		   'Halpe_26', 'Halpe_26_det', 'Halpe_136', 'Halpe_136_det', \
+		   'Halpe_coco_wholebody_26', 'Halpe_coco_wholebody_26_det', \
+		   'Halpe_coco_wholebody_136', 'Halpe_coco_wholebody_136_det', \
+		   'Halpe_68_noface', 'Halpe_68_noface_det', 'SingleHand', 'SingleHand_det', \
+		   'coco_wholebody', 'coco_wholebody_det']
diff --git a/alphapose/datasets/coco_det.py b/alphapose/datasets/coco_det.py
@@ -68,10 +68,10 @@ def __getitem__(self, index):
             img_id = int(img_id)
         else:
             img_id = det_res['image_id']
-        img_path = './data/coco/val2017/%012d.jpg' % img_id
+        img_path = os.path.join(self._root, 'val2017/%012d.jpg' % img_id)
 
         # Load image
-        image = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB) #scipy.misc.imread(img_path, mode='RGB')
+        image = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB) # scipy.misc.imread(img_path, mode='RGB') is deprecated
 
         imght, imgwidth = image.shape[0], image.shape[1]
         x1, y1, w, h = det_res['bbox']

diff --git a/alphapose/datasets/coco_wholebody.py b/alphapose/datasets/coco_wholebody.py
@@ -3,7 +3,7 @@
 # Written by Haoyi Zhu and Hao-Shu Fang
 # -----------------------------------------------------
 
-"""Halpe Full-Body(136 points) Human keypoint dataset."""
+"""Coco WholeBody (133 points) Human keypoint dataset."""
 import os
 
 import numpy as np
@@ -17,7 +17,7 @@
 
 @DATASET.register_module
 class coco_wholebody(CustomDataset):
-    """ Halpe Full-Body(136 points) Person dataset.
+    """ Coco WholeBody (133 points) Person dataset.
 
     Parameters
     ----------
@@ -32,17 +32,18 @@ class coco_wholebody(CustomDataset):
     CLASSES = ['person']
     EVAL_JOINTS = list(range(133))
     num_joints = 133
-    CustomDataset.lower_body_ids = (11, 12, 13, 14, 15, 16, 17, 21-3, 22-3, 23-3, 24-3, 25-3)
+    CustomDataset.lower_body_ids = (11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22)
     """Joint pairs which defines the pairs of joint to be swapped
         when the image is flipped horizontally."""
-    joint_pairs =  [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14], [15, 16], #17 body keypoints
-        [20-3, 23-3], [21-3, 24-3], [22-3, 25-3], [26-3, 42-3], [27-3, 41-3], [28-3, 40-3], [29-3, 39-3], [30-3, 38-3], 
-        [31-3, 37-3], [32-3, 36-3], [33-3, 35-3], [43-3, 52-3], [44-3, 51-3], [45-3, 50-3], [46-3, 49-3], [47-3, 48-3], 
-        [62-3, 71-3], [63-3, 70-3], [64-3, 69-3], [65-3, 68-3], [66-3, 73-3], [67-3, 72-3], [57-3, 61-3], [58-3, 60-3],
-        [74-3, 80-3], [75-3, 79-3], [76-3, 78-3], [87-3, 89-3], [93-3, 91-3], [86-3, 90-3], [85-3, 81-3], [84-3, 82-3],
-        [94-3, 115-3], [95-3, 116-3], [96-3, 117-3], [97-3, 118-3], [98-3, 119-3], [99-3, 120-3], [100-3, 121-3],
-        [101-3, 122-3], [102-3, 123-3], [103-3, 124-3], [104-3, 125-3], [105-3, 126-3], [106-3, 127-3], [107-3, 128-3],
-        [108-3, 129-3], [109-3, 130-3], [110-3, 131-3], [111-3, 132-3], [112-3, 133-3], [113-3, 134-3], [114-3, 135-3]]
+    joint_pairs =  [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14], [15, 16], 
+                    [17, 20], [18, 21], [19, 22], [23, 39], [24, 38], [25, 37], [26, 36], 
+                    [27, 35], [28, 34], [29, 33], [30, 32], [40, 49], [41, 48], [42, 47], 
+                    [43, 46], [44, 45], [59, 68], [60, 67], [61, 66], [62, 65], [63, 70], 
+                    [64, 69], [54, 58], [55, 57], [71, 77], [72, 76], [73, 75], [84, 86], 
+                    [90, 88], [83, 87], [82, 78], [81, 79], [91, 112], [92, 113], [93, 114], 
+                    [94, 115], [95, 116], [96, 117], [97, 118], [98, 119], [99, 120], 
+                    [100, 121], [101, 122], [102, 123], [103, 124], [104, 125], [105, 126], 
+                    [106, 127], [107, 128], [108, 129], [109, 130], [110, 131], [111, 132]]
 
 
     def _load_jsons(self):
@@ -62,7 +63,7 @@ def _load_jsons(self):
         image_ids = sorted(_coco.getImgIds())
         for entry in _coco.loadImgs(image_ids):
             dirname, filename = entry['coco_url'].split('/')[-2:]
-            abs_path = os.path.join('/DATA1/Benchmark/coco', dirname, filename)
+            abs_path = os.path.join(self._root, dirname, filename)
             if not os.path.exists(abs_path):
                 raise IOError('Image: {} not exists.'.format(abs_path))
             label = self._check_load_keypoints(_coco, entry)
@@ -84,11 +85,11 @@ def _check_load_keypoints(self, coco, entry):
         height = entry['height']
 
         for obj in objs:
-            #obj['keypoints'].extend([0,0,0, 0,0,0, 0,0,0])
-            obj['keypoints'].extend(obj['foot_kpts'])
-            obj['keypoints'].extend(obj['face_kpts'])
-            obj['keypoints'].extend(obj['lefthand_kpts'])
-            obj['keypoints'].extend(obj['righthand_kpts'])
+            if 'foot_kpts' in obj and 'face_kpts' in obj and 'lefthand_kpts' in obj and 'righthand_kpts' in obj:
+                obj['keypoints'].extend(obj['foot_kpts'])
+                obj['keypoints'].extend(obj['face_kpts'])
+                obj['keypoints'].extend(obj['lefthand_kpts'])
+                obj['keypoints'].extend(obj['righthand_kpts'])
             contiguous_cid = self.json_id_to_contiguous[obj['category_id']]
             if contiguous_cid >= self.num_class:
                 # not class of interest
@@ -98,8 +99,7 @@ def _check_load_keypoints(self, coco, entry):
             # convert from (x, y, w, h) to (xmin, ymin, xmax, ymax) and clip bound
             xmin, ymin, xmax, ymax = bbox_clip_xyxy(bbox_xywh_to_xyxy(obj['bbox']), width, height)
             # require non-zero box area
-            #if obj['area'] <= 0 or xmax <= xmin or ymax <= ymin:
-            if (xmax-xmin)*(ymax-ymin) <= 0 or xmax <= xmin or ymax <= ymin:
+            if (xmax - xmin) * (ymax - ymin) <= 0 or xmax <= xmin or ymax <= ymin:
                 continue
             if 'num_keypoints' in obj and obj['num_keypoints'] == 0:
                 continue
@@ -108,14 +108,11 @@ def _check_load_keypoints(self, coco, entry):
             for i in range(self.num_joints):
                 joints_3d[i, 0, 0] = obj['keypoints'][i * 3 + 0]
                 joints_3d[i, 1, 0] = obj['keypoints'][i * 3 + 1]
-                # joints_3d[i, 2, 0] = 0
                 if obj['keypoints'][i * 3 + 2] >= 0.35:
                     visible = 1
                 else:
                     visible = 0
-                #visible = min(1, visible)
                 joints_3d[i, :2, 1] = visible
-                # joints_3d[i, 2, 1] = 0
 
             if np.sum(joints_3d[:, 0, 1]) < 1:
                 # no visible keypoint

diff --git a/alphapose/datasets/coco_wholebody_det.py b/alphapose/datasets/coco_wholebody_det.py
@@ -3,7 +3,7 @@
 # Written by Haoyi Zhu
 # -----------------------------------------------------
 
-"""Haple_136 Human Detection Box dataset."""
+"""Coco WholeBody Human Detection Box dataset."""
 import json
 import os
 
@@ -19,7 +19,7 @@
 
 @DATASET.register_module
 class coco_wholebody_det(data.Dataset):
-    """ Halpe_136 human detection box dataset.
+    """ Coco WholeBody human detection box dataset.
 
     """
     EVAL_JOINTS = list(range(133))
@@ -68,10 +68,10 @@ def __getitem__(self, index):
             img_id = int(img_id)
         else:
             img_id = det_res['image_id']
-        img_path = '/DATA1/Benchmark/coco/val2017/%012d.jpg' % img_id
+        img_path = os.path.join(self._root, self._img_prefix, '%012d.jpg' % img_id)
 
         # Load image
-        image = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB) #scipy.misc.imread(img_path, mode='RGB')
+        image = cv2.cvtColor(cv2.imread(img_path), cv2.COLOR_BGR2RGB) # scipy.misc.imread(img_path, mode='RGB') is deprecated
 
         imght, imgwidth = image.shape[1], image.shape[2]
         x1, y1, w, h = det_res['bbox']
@@ -92,7 +92,7 @@ def write_coco_json(self, det_file):
         dets = []
         for entry in tqdm(_coco.loadImgs(image_ids)):
             abs_path = os.path.join(
-                '/DATA1/Benchmark/coco', self._img_prefix, entry['file_name'])
+                self._root, self._img_prefix, entry['file_name'])
             det = det_model.detect_one_img(abs_path)
             if det:
                 dets += det
@@ -103,11 +103,12 @@ def write_coco_json(self, det_file):
     def joint_pairs(self):
         """Joint pairs which defines the pairs of joint to be swapped
         when the image is flipped horizontally."""
-        return [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14], [15, 16], #17 body keypoints
-        [20-3, 23-3], [21-3, 24-3], [22-3, 25-3], [26-3, 42-3], [27-3, 41-3], [28-3, 40-3], [29-3, 39-3], [30-3, 38-3], 
-        [31-3, 37-3], [32-3, 36-3], [33-3, 35-3], [43-3, 52-3], [44-3, 51-3], [45-3, 50-3], [46-3, 49-3], [47-3, 48-3], 
-        [62-3, 71-3], [63-3, 70-3], [64-3, 69-3], [65-3, 68-3], [66-3, 73-3], [67-3, 72-3], [57-3, 61-3], [58-3, 60-3],
-        [74-3, 80-3], [75-3, 79-3], [76-3, 78-3], [87-3, 89-3], [93-3, 91-3], [86-3, 90-3], [85-3, 81-3], [84-3, 82-3],
-        [94-3, 115-3], [95-3, 116-3], [96-3, 117-3], [97-3, 118-3], [98-3, 119-3], [99-3, 120-3], [100-3, 121-3],
-        [101-3, 122-3], [102-3, 123-3], [103-3, 124-3], [104-3, 125-3], [105-3, 126-3], [106-3, 127-3], [107-3, 128-3],
-        [108-3, 129-3], [109-3, 130-3], [110-3, 131-3], [111-3, 132-3], [112-3, 133-3], [113-3, 134-3], [114-3, 135-3]]
+        return [[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14], [15, 16], 
+                [17, 20], [18, 21], [19, 22], [23, 39], [24, 38], [25, 37], [26, 36], 
+                [27, 35], [28, 34], [29, 33], [30, 32], [40, 49], [41, 48], [42, 47], 
+                [43, 46], [44, 45], [59, 68], [60, 67], [61, 66], [62, 65], [63, 70], 
+                [64, 69], [54, 58], [55, 57], [71, 77], [72, 76], [73, 75], [84, 86], 
+                [90, 88], [83, 87], [82, 78], [81, 79], [91, 112], [92, 113], [93, 114], 
+                [94, 115], [95, 116], [96, 117], [97, 118], [98, 119], [99, 120], 
+                [100, 121], [101, 122], [102, 123], [103, 124], [104, 125], [105, 126], 
+                [106, 127], [107, 128], [108, 129], [109, 130], [110, 131], [111, 132]]