support python2 and get more reasonable mAP

.gitignore

@@ -1,3 +1,4 @@
 *.pyc
 *.so
 *.a
+.idea/

lib/datasets/imagenet.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 # --------------------------------------------------------
 # Fast R-CNN
 # Copyright (c) 2015 Microsoft
@@ -13,7 +14,7 @@
 import numpy as np
 import scipy.sparse
 import scipy.io as sio
-import cPickle
+import pickle
 import subprocess
 import pdb
 
@@ -150,15 +151,15 @@ def gt_roidb(self):
         cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
         if os.path.exists(cache_file):
             with open(cache_file, 'rb') as fid:
-                roidb = cPickle.load(fid)
-            print '{} gt roidb loaded from {}'.format(self.name, cache_file)
+                roidb = pickle.load(fid)
+            print('{} gt roidb loaded from {}'.format(self.name, cache_file))
             return roidb
 
         gt_roidb = [self._load_imagenet_annotation(index)
                     for index in self.image_index]
         with open(cache_file, 'wb') as fid:
-            cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL)
-        print 'wrote gt roidb to {}'.format(cache_file)
+            pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
+        print('wrote gt roidb to {}'.format(cache_file))
 
         return gt_roidb
 

lib/datasets/pascal_voc.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+from __future__ import absolute_import
 # --------------------------------------------------------
 # Fast R-CNN
 # Copyright (c) 2015 Microsoft
@@ -12,22 +14,25 @@
 import numpy as np
 import scipy.sparse
 import subprocess
-import cPickle
 import math
 import glob
 import uuid
 import scipy.io as sio
 import xml.etree.ElementTree as ET
-
+import pickle
 from .imdb import imdb
 from .imdb import ROOT_DIR
-import ds_utils
+from . import ds_utils
 from .voc_eval import voc_eval
 
 # TODO: make fast_rcnn irrelevant
 # >>>> obsolete, because it depends on sth outside of this project
 from model.utils.config import cfg
 
+try:
+    xrange          # Python 2
+except NameError:
+    xrange = range  # Python 3
 
 # <<<< obsolete
 
@@ -119,15 +124,15 @@ def gt_roidb(self):
         cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
         if os.path.exists(cache_file):
             with open(cache_file, 'rb') as fid:
-                roidb = cPickle.load(fid)
-            print '{} gt roidb loaded from {}'.format(self.name, cache_file)
+                roidb = pickle.load(fid)
+            print('{} gt roidb loaded from {}'.format(self.name, cache_file))
             return roidb
 
         gt_roidb = [self._load_pascal_annotation(index)
                     for index in self.image_index]
         with open(cache_file, 'wb') as fid:
-            cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL)
-        print 'wrote gt roidb to {}'.format(cache_file)
+            pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
+        print('wrote gt roidb to {}'.format(cache_file))
 
         return gt_roidb
 
@@ -143,8 +148,8 @@ def selective_search_roidb(self):
 
         if os.path.exists(cache_file):
             with open(cache_file, 'rb') as fid:
-                roidb = cPickle.load(fid)
-            print '{} ss roidb loaded from {}'.format(self.name, cache_file)
+                roidb = pickle.load(fid)
+            print('{} ss roidb loaded from {}'.format(self.name, cache_file))
             return roidb
 
         if int(self._year) == 2007 or self._image_set != 'test':
@@ -154,8 +159,8 @@ def selective_search_roidb(self):
         else:
             roidb = self._load_selective_search_roidb(None)
         with open(cache_file, 'wb') as fid:
-            cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL)
-        print 'wrote ss roidb to {}'.format(cache_file)
+            pickle.dump(roidb, fid, pickle.HIGHEST_PROTOCOL)
+        print('wrote ss roidb to {}'.format(cache_file))
 
         return roidb
 
@@ -171,11 +176,11 @@ def rpn_roidb(self):
 
     def _load_rpn_roidb(self, gt_roidb):
         filename = self.config['rpn_file']
-        print 'loading {}'.format(filename)
+        print('loading {}'.format(filename))
         assert os.path.exists(filename), \
             'rpn data not found at: {}'.format(filename)
         with open(filename, 'rb') as f:
-            box_list = cPickle.load(f)
+            box_list = pickle.load(f)
         return self.create_roidb_from_box_list(box_list, gt_roidb)
 
     def _load_selective_search_roidb(self, gt_roidb):
@@ -268,7 +273,7 @@ def _write_voc_results_file(self, all_boxes):
         for cls_ind, cls in enumerate(self.classes):
             if cls == '__background__':
                 continue
-            print 'Writing {} VOC results file'.format(cls)
+            print('Writing {} VOC results file'.format(cls))
             filename = self._get_voc_results_file_template().format(cls)
             with open(filename, 'wt') as f:
                 for im_ind, index in enumerate(self.image_index):
@@ -298,7 +303,7 @@ def _do_python_eval(self, output_dir='output'):
         aps = []
         # The PASCAL VOC metric changed in 2010
         use_07_metric = True if int(self._year) < 2010 else False
-        print 'VOC07 metric? ' + ('Yes' if use_07_metric else 'No')
+        print('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
         if not os.path.isdir(output_dir):
             os.mkdir(output_dir)
         for i, cls in enumerate(self._classes):
@@ -310,8 +315,8 @@ def _do_python_eval(self, output_dir='output'):
                 use_07_metric=use_07_metric)
             aps += [ap]
             print('AP for {} = {:.4f}'.format(cls, ap))
-            with open(os.path.join(output_dir, cls + '_pr.pkl'), 'w') as f:
-                cPickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
+            with open(os.path.join(output_dir, cls + '_pr.pkl'), 'wb') as f:
+                pickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
         print('Mean AP = {:.4f}'.format(np.mean(aps)))
         print('~~~~~~~~')
         print('Results:')
@@ -328,9 +333,9 @@ def _do_python_eval(self, output_dir='output'):
         print('--------------------------------------------------------------')
 
     def _do_matlab_eval(self, output_dir='output'):
-        print '-----------------------------------------------------'
-        print 'Computing results with the official MATLAB eval code.'
-        print '-----------------------------------------------------'
+        print('-----------------------------------------------------')
+        print('Computing results with the official MATLAB eval code.')
+        print('-----------------------------------------------------')
         path = os.path.join(cfg.ROOT_DIR, 'lib', 'datasets',
                             'VOCdevkit-matlab-wrapper')
         cmd = 'cd {} && '.format(path)

lib/datasets/vg.py

@@ -11,11 +11,10 @@
 import xml.etree.ElementTree as ET
 import numpy as np
 import scipy.sparse
-import cPickle
 import gzip
 import PIL
 import json
-from vg_eval import vg_eval
+from datasets.vg_eval import vg_eval
 from model.utils.config import cfg
 import pickle
 import pdb
@@ -121,11 +120,11 @@ def _image_split_path(self):
         if self._image_set == "minitrain":
           return os.path.join(self._data_path, 'train.txt')
         if self._image_set == "smalltrain":
-          return os.path.join(self._data_path, 'train.txt')          
+          return os.path.join(self._data_path, 'train.txt')
         if self._image_set == "minival":
           return os.path.join(self._data_path, 'val.txt')
         if self._image_set == "smallval":
-          return os.path.join(self._data_path, 'val.txt')          
+          return os.path.join(self._data_path, 'val.txt')
         else:
           return os.path.join(self._data_path, self._image_set+'.txt')
 
@@ -141,7 +140,7 @@ def _load_image_set_index(self):
           if self._image_set == "minitrain":
             metadata = metadata[:1000]
           elif self._image_set == "smalltrain":
-            metadata = metadata[:20000]            
+            metadata = metadata[:20000]
           elif self._image_set == "minival":
             metadata = metadata[:100]
           elif self._image_set == "smallval":
@@ -172,21 +171,21 @@ def gt_roidb(self):
         Return the database of ground-truth regions of interest.
 
         This function loads/saves from/to a cache file to speed up future calls.
-        """        
+        """
         cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')
         if os.path.exists(cache_file):
             fid = gzip.open(cache_file,'rb')
-            roidb = cPickle.load(fid)
+            roidb = pickle.load(fid)
             fid.close()
-            print '{} gt roidb loaded from {}'.format(self.name, cache_file)
+            print('{} gt roidb loaded from {}'.format(self.name, cache_file))
             return roidb
 
         gt_roidb = [self._load_vg_annotation(index)
                     for index in self.image_index]
         fid = gzip.open(cache_file,'wb')
-        cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL)
+        pickle.dump(gt_roidb, fid, pickle.HIGHEST_PROTOCOL)
         fid.close()
-        print 'wrote gt roidb to {}'.format(cache_file)
+        print('wrote gt roidb to {}'.format(cache_file))
         return gt_roidb
 
     def _get_size(self, index):
@@ -227,7 +226,7 @@ def _load_vg_annotation(self, index):
                 y2 = min(height-1,float(bbox.find('ymax').text))
                 # If bboxes are not positive, just give whole image coords (there are a few examples)
                 if x2 < x1 or y2 < y1:
-                    print 'Failed bbox in %s, object %s' % (filename, obj_name)
+                    print('Failed bbox in %s, object %s' % (filename, obj_name))
                     x1 = 0
                     y1 = 0
                     x2 = width-1
@@ -312,7 +311,7 @@ def _write_voc_results_file(self, classes, all_boxes, output_dir):
         for cls_ind, cls in enumerate(classes):
             if cls == '__background__':
                 continue
-            print 'Writing "{}" vg results file'.format(cls)
+            print('Writing "{}" vg results file'.format(cls))
             filename = self._get_vg_results_file_template(output_dir).format(cls)
             with open(filename, 'wt') as f:
                 for im_ind, index in enumerate(self.image_index):
@@ -334,7 +333,7 @@ def _do_python_eval(self, output_dir, pickle=True, eval_attributes = False):
         thresh = []
         # The PASCAL VOC metric changed in 2010
         use_07_metric = False
-        print 'VOC07 metric? ' + ('Yes' if use_07_metric else 'No')
+        print('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
         if not os.path.isdir(output_dir):
             os.mkdir(output_dir)
         # Load ground truth
@@ -362,7 +361,7 @@ def _do_python_eval(self, output_dir, pickle=True, eval_attributes = False):
             print('AP for {} = {:.4f} (npos={:,})'.format(cls, ap, npos))
             if pickle:
                 with open(os.path.join(output_dir, cls + '_pr.pkl'), 'w') as f:
-                    cPickle.dump({'rec': rec, 'prec': prec, 'ap': ap,
+                    pickle.dump({'rec': rec, 'prec': prec, 'ap': ap,
                         'scores': scores, 'npos':npos}, f)
 
         # Set thresh to mean for classes with poor results

lib/datasets/vg_eval.py

@@ -6,21 +6,19 @@
 
 import xml.etree.ElementTree as ET
 import os
-import cPickle
 import numpy as np
-from voc_eval import voc_ap
+from datasets.voc_eval import voc_ap
 
 
-
-def vg_eval( detpath,
-             gt_roidb,
-             image_index,
-             classindex,
-             ovthresh=0.5,
-             use_07_metric=False,
-             eval_attributes=False):
+def vg_eval(detpath,
+            gt_roidb,
+            image_index,
+            classindex,
+            ovthresh=0.5,
+            use_07_metric=False,
+            eval_attributes=False):
     """rec, prec, ap, sorted_scores, npos = voc_eval(
-                                detpath, 
+                                detpath,
                                 gt_roidb,
                                 image_index,
                                 classindex,
@@ -40,27 +38,27 @@ def vg_eval( detpath,
     # extract gt objects for this class
     class_recs = {}
     npos = 0
-    for item,imagename in zip(gt_roidb,image_index):
+    for item, imagename in zip(gt_roidb, image_index):
         if eval_attributes:
             bbox = item['boxes'][np.where(np.any(item['gt_attributes'].toarray() == classindex, axis=1))[0], :]
         else:
             bbox = item['boxes'][np.where(item['gt_classes'] == classindex)[0], :]
         difficult = np.zeros((bbox.shape[0],)).astype(np.bool)
         det = [False] * bbox.shape[0]
-        npos = npos + sum(~difficult)        
+        npos = npos + sum(~difficult)
         class_recs[str(imagename)] = {'bbox': bbox,
-                                 'difficult': difficult,
-                                 'det': det}
+                                      'difficult': difficult,
+                                      'det': det}
     if npos == 0:
         # No ground truth examples
-        return 0,0,0,0,npos
+        return 0, 0, 0, 0, npos
 
     # read dets
     with open(detpath, 'r') as f:
         lines = f.readlines()
     if len(lines) == 0:
         # No detection examples
-        return 0,0,0,0,npos
+        return 0, 0, 0, 0, npos
 
     splitlines = [x.strip().split(' ') for x in lines]
     image_ids = [x[0] for x in splitlines]
@@ -121,5 +119,5 @@ def vg_eval( detpath,
     # ground truth
     prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
     ap = voc_ap(rec, prec, use_07_metric)
-    
+
     return rec, prec, ap, sorted_scores, npos

lib/datasets/voc_eval.py

@@ -117,7 +117,7 @@ def voc_eval(detpath,
           i + 1, len(imagenames)))
     # save
     print('Saving cached annotations to {:s}'.format(cachefile))
-    with open(cachefile, 'w') as f:
+    with open(cachefile, 'wb') as f:
       pickle.dump(recs, f)
   else:
     # load