adding player plugins.

pupil_src/capture/world.py

@@ -37,7 +37,7 @@
 import calibration_routines
 import recorder
 from show_calibration import Show_Calibration
-from display_gaze import Display_Gaze
+from display_recent_gaze import Display_Recent_Gaze
 from pupil_server import Pupil_Server
 from marker_detector import Marker_Detector
 
@@ -268,7 +268,7 @@ def map_pupil(vector):
     open_calibration(bar.calibration_type.value,bar.calibration_type)
 
     #load gaze_display plugin
-    g.plugins.append(Display_Gaze(g_pool))
+    g.plugins.append(Display_Recent_Gaze(g_pool))
 
     # Event loop
     while not g_pool.quit.value:

pupil_src/player/main.py

@@ -55,10 +55,13 @@
 from player_methods import correlate_gaze,patch_meta_info,is_pupil_rec_dir
 from gl_utils import basic_gl_setup, adjust_gl_view, draw_gl_texture, clear_gl_screen, draw_gl_point_norm,draw_gl_texture
 # Plug-ins
+from vis_circle import Vis_Circle
+from vis_polyline import Vis_Polyline
 from display_gaze import Display_Gaze
+from vis_light_points import Vis_Light_Points
 from seek_bar import Seek_Bar
 from export_launcher import Export_Launcher
-
+from scan_path import Scan_Path
 import logging
 logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)
@@ -67,7 +70,7 @@
 fh.setLevel(logging.DEBUG)
 # create console handler with a higher log level
 ch = logging.StreamHandler()
-ch.setLevel(logging.WARNING)
+ch.setLevel(logging.DEBUG)
 # create formatter and add it to the handlers
 formatter = logging.Formatter('Player: %(asctime)s - %(name)s - %(levelname)s - %(message)s')
 fh.setFormatter(formatter)
@@ -220,8 +223,12 @@ def save(var_name,var):
     g.plugins = []
     g.play = False
     g.new_seek = True
-    g.plugins.append(Display_Gaze(g))
+    # g.plugins.append(Display_Gaze(g))
     g.plugins.append(Seek_Bar(g,capture=cap))
+    g.plugins.append(Scan_Path(g))
+    # g.plugins.append(Vis_Light_Points(g))
+    g.plugins.append(Vis_Polyline(g))
+    g.plugins.append(Vis_Circle(g))
 
     # helpers called by the main atb bar
     def update_fps():
@@ -279,6 +286,7 @@ def set_play(value):
 
     g.plugins.append(Export_Launcher(g,data_dir=data_folder,frame_count=len(timestamps)))
 
+    g.plugins.sort(key=lambda x: x.order)
 
     while not glfwWindowShouldClose(main_window):
 

pupil_src/player/player_methods.py

@@ -11,7 +11,7 @@
 
 
 import os
-
+import cv2
 #logging
 import logging
 logger = logging.getLogger(__name__)
@@ -142,4 +142,19 @@ def convert_gaze_pos(gaze_list,capture_version):
     return gaze_list
 
 
+def transparent_cirlce(img,center,radius,color,thickness):
+    try:
+        center = tuple(map(int,center))
+        if thickness > 0:
+            pad = radius + 2 + thickness
+        else:
+            pad = radius + 3
+        roi = slice(center[1]-pad,center[1]+pad),slice(center[0]-pad,center[0]+pad)
+        overlay = img[roi].copy()
+        cv2.circle(overlay,(pad,pad), radius=radius, color=color[:3], thickness=thickness, lineType=cv2.cv.CV_AA)
+        opacity = color[-1]/255.
+        cv2.addWeighted(overlay, opacity, img[roi], 1. - opacity, 0, img[roi])
+    except:
+        logger.debug("transparent_cirlce would have been partially outise of img. Did not draw it.")
+
 

pupil_src/player/scan_path.py

@@ -0,0 +1,101 @@
+'''
+(*)~----------------------------------------------------------------------------------
+ Pupil - eye tracking platform
+ Copyright (C) 2012-2013  Moritz Kassner & William Patera
+
+ Distributed under the terms of the CC BY-NC-SA License.
+ License details are in the file license.txt, distributed as part of this software.
+----------------------------------------------------------------------------------~(*)
+'''
+
+import cv2
+from plugin import Plugin
+import numpy as np
+import atb
+from methods import denormalize,normalize
+import logging
+logger = logging.getLogger(__name__)
+
+class Scan_Path(Plugin):
+    """docstring
+    using this plugin will extend the recent_pupil_positions by x extra dots from previous frames.
+    lock recent gaze points onto pixels.
+    """
+
+    def __init__(self, g_pool):
+        super(Scan_Path, self).__init__()
+        self.g_pool = g_pool
+
+        #let the plugin work after most other plugins.
+        self.order = .6
+
+
+        #user settings
+        self.scan_path_timeframe = 3.
+
+        #algorithm working data
+        self.prev_frame_idx = -1
+        self.past_pupil_positions = []
+        self.prev_gray = None
+
+
+    def update(self,frame,recent_pupil_positions,events):
+        img = frame.img
+        img_shape = img.shape[:-1][::-1] # width,height
+
+        succeeding_frame = frame.index-self.prev_frame_idx == 1
+        gray_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+
+
+        #vars for calcOpticalFlowPyrLK
+        lk_params = dict( winSize  = (90, 90),
+                  maxLevel = 2,
+                  criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 20, 0.03))
+
+
+        updated_past_gaze = []
+
+        #lets update past gaze using optical flow: this is like sticking the gaze points onto the pixels of the img.
+        if self.past_pupil_positions and succeeding_frame:
+            past_screen_gaze = np.array([denormalize(ng['norm_gaze'] ,img_shape,flip_y=True) for ng in self.past_pupil_positions],dtype=np.float32)
+            new_pts, status, err = cv2.calcOpticalFlowPyrLK(self.prev_gray, gray_img,past_screen_gaze,minEigThreshold=0.01,**lk_params)
+
+            for gaze,new_gaze_pt,s,e in zip(self.past_pupil_positions,new_pts,status,err):
+                if s:
+                    # print "norm,updated",gaze['norm_gaze'], normalize(new_gaze_pt,img_shape[:-1],flip_y=True)
+                    gaze['norm_gaze'] = normalize(new_gaze_pt,img_shape,flip_y=True)
+                    updated_past_gaze.append(gaze)
+                    # logger.debug("updated gaze")
+
+                else:
+                    # logger.debug("dropping gaze")
+                    # Since we will replace self.past_pupil_positions later,
+                    # not appedning tu updated_past_gaze is like deliting this data point.
+                    pass
+
+
+        # print "new_gaze", len(recent_pupil_positions)
+        # print "from before", len(updated_past_gaze)
+
+        # trim of gaze that is too old
+        if recent_pupil_positions:
+            now = recent_pupil_positions[0]['timestamp']
+            cutof = now-self.scan_path_timeframe
+            updated_past_gaze = [g for g in updated_past_gaze if g['timestamp']>cutof]
+
+
+        #inject the scan path gaze points into recent_pupil_positions
+        recent_pupil_positions[:] = updated_past_gaze + recent_pupil_positions
+        recent_pupil_positions.sort(key=lambda x: x['timestamp']) #this may be redundant...
+
+
+        #update info for next frame.
+        self.prev_gray = gray_img
+        self.prev_frame_idx = frame.index
+        self.past_pupil_positions = recent_pupil_positions
+
+
+    def gl_display(self):
+        pass
+
+

pupil_src/player/vis_circle.py

@@ -0,0 +1,38 @@
+'''
+(*)~----------------------------------------------------------------------------------
+ Pupil - eye tracking platform
+ Copyright (C) 2012-2013  Moritz Kassner & William Patera
+
+ Distributed under the terms of the CC BY-NC-SA License.
+ License details are in the file license.txt, distributed as part of this software.
+----------------------------------------------------------------------------------~(*)
+'''
+
+from gl_utils import draw_gl_points_norm
+from player_methods import transparent_cirlce
+from plugin import Plugin
+import numpy as np
+
+import cv2
+
+from methods import denormalize
+
+class Vis_Circle(Plugin):
+    """docstring for DisplayGaze"""
+    def __init__(self, g_pool):
+        super(Vis_Circle, self).__init__()
+        self.g_pool = g_pool
+        self.order = .9
+        self.prev_frame_idx = -1
+        self.radius = 20
+
+    def update(self,frame,recent_pupil_positions,events):
+        if self.prev_frame_idx != frame.index:
+            pts = [denormalize(pt['norm_gaze'],frame.img.shape[:-1][::-1],flip_y=True) for pt in recent_pupil_positions if pt['norm_gaze'] is not None]
+            for pt in pts:
+                transparent_cirlce(frame.img, tuple(map(int,pt)), radius=self.radius, color=(0,255,0,100), thickness=3)
+            self.prev_frame_idx = frame.index
+
+
+    def gl_display(self):
+        pass

pupil_src/player/vis_light_points.py

@@ -0,0 +1,70 @@
+'''
+(*)~----------------------------------------------------------------------------------
+ Pupil - eye tracking platform
+ Copyright (C) 2012-2013  Moritz Kassner & William Patera
+
+ Distributed under the terms of the CC BY-NC-SA License.
+ License details are in the file license.txt, distributed as part of this software.
+----------------------------------------------------------------------------------~(*)
+'''
+
+import cv2
+from plugin import Plugin
+import numpy as np
+import atb
+from methods import denormalize
+import logging
+logger = logging.getLogger(__name__)
+
+class Vis_Light_Points(Plugin):
+    """docstring
+    show gaze dots at light dots on numpy.
+
+    """
+    #let the plugin work after most other plugins.
+
+    def __init__(self, g_pool):
+        super(Vis_Light_Points, self).__init__()
+        self.g_pool = g_pool
+
+        self.order = .8
+
+        self.prev_frame_idx = -1
+
+    def update(self,frame,recent_pupil_positions,events):
+
+        #since we edit the img inplace we should not do it in pause mode...
+        if self.prev_frame_idx != frame.index:
+            img = frame.img
+            img_shape = img.shape[:-1][::-1]#width,height
+            norm_gaze = [ng['norm_gaze'] for ng in recent_pupil_positions if ng['norm_gaze'] is not None]
+            screen_gaze = [denormalize(ng,img_shape,flip_y=True) for ng in norm_gaze]
+
+
+            overlay = np.ones(img.shape[:-1],dtype=img.dtype)
+
+            # draw recent gaze postions as black dots on an overlay image.
+            for gaze_point in screen_gaze:
+                try:
+                    overlay[int(gaze_point[1]),int(gaze_point[0])] = 0
+                except:
+                    pass
+
+            out = cv2.distanceTransform(overlay,cv2.cv.CV_DIST_L2, 5)
+
+            # fix for opencv binding incositency
+            if type(out)==tuple:
+                out = out[0]
+
+            overlay =  1/(out/20+1)
+
+            img *= cv2.cvtColor(overlay,cv2.COLOR_GRAY2RGB)
+
+            self.prev_frame_idx = frame.index
+
+
+
+    def gl_display(self):
+        pass
+
+

pupil_src/player/vis_polyline.py

@@ -0,0 +1,38 @@
+'''
+(*)~----------------------------------------------------------------------------------
+ Pupil - eye tracking platform
+ Copyright (C) 2012-2013  Moritz Kassner & William Patera
+
+ Distributed under the terms of the CC BY-NC-SA License.
+ License details are in the file license.txt, distributed as part of this software.
+----------------------------------------------------------------------------------~(*)
+'''
+
+from gl_utils import draw_gl_points_norm
+from plugin import Plugin
+import numpy as np
+
+import cv2
+
+from methods import denormalize
+
+class Vis_Polyline(Plugin):
+    """docstring for DisplayGaze"""
+    def __init__(self, g_pool):
+        super(Vis_Polyline, self).__init__()
+        self.g_pool = g_pool
+        self.order = .9
+        self.prev_frame_idx = -1
+
+    def update(self,frame,recent_pupil_positions,events):
+        if self.prev_frame_idx != frame.index:
+            pts = [denormalize(pt['norm_gaze'],frame.img.shape[:-1][::-1],flip_y=True) for pt in recent_pupil_positions if pt['norm_gaze'] is not None]
+            if pts:
+                pts = np.array([pts],dtype=np.int32)
+                cv2.polylines(frame.img, pts, isClosed=False, color=(0,255,0), thickness=1, lineType=cv2.cv.CV_AA)
+
+            self.prev_frame_idx = frame.index
+
+
+    def gl_display(self):
+        pass

pupil_src/shared_modules/display_gaze.py → pupil_src/shared_modules/display_recent_gaze.py

@@ -14,11 +14,12 @@
 
 from methods import denormalize
 
-class Display_Gaze(Plugin):
+class Display_Recent_Gaze(Plugin):
     """docstring for DisplayGaze"""
     def __init__(self, g_pool,atb_pos=None):
-        super(Display_Gaze, self).__init__()
+        super(Display_Recent_Gaze, self).__init__()
         self.g_pool = g_pool
+        self.order = .8
         self.atb_pos = atb_pos
         self.pupil_display_list = []
 
@@ -28,6 +29,7 @@ def update(self,frame,recent_pupil_positions,events):
                 self.pupil_display_list.append(pt['norm_gaze'])
         self.pupil_display_list[:-3] = []
 
+
     def gl_display(self):
         draw_gl_points_norm(self.pupil_display_list,size=35,color=(1.,.2,.4,.6))