diff --git a/pupil_src/player/display_gaze.py b/pupil_src/player/display_gaze.py new file mode 100644 index 0000000000..60846f1214 --- /dev/null +++ b/pupil_src/player/display_gaze.py @@ -0,0 +1,31 @@ +''' +(*)~---------------------------------------------------------------------------------- + Pupil - eye tracking platform + Copyright (C) 2012-2013 Moritz Kassner & William Patera + + Distributed under the terms of the CC BY-NC-SA License. + License details are in the file license.txt, distributed as part of this software. +----------------------------------------------------------------------------------~(*) +''' + +from gl_utils import draw_gl_points_norm +from plugin import Plugin +import numpy as np + +from methods import denormalize + +class Display_Gaze(Plugin): + """docstring for DisplayGaze""" + def __init__(self, g_pool,atb_pos=None): + super(Display_Gaze, self).__init__() + self.g_pool = g_pool + self.order = .8 + self.atb_pos = atb_pos + self.pupil_display_list = [] + + def update(self,frame,recent_pupil_positions,events): + self.pupil_display_list = [pt['norm_gaze'] for pt in recent_pupil_positions if pt['norm_gaze'] is not None] + + def gl_display(self): + draw_gl_points_norm(self.pupil_display_list,size=35,color=(1.,.2,.4,.6)) + diff --git a/pupil_src/player/export_launcher.py b/pupil_src/player/export_launcher.py index e310384610..1fe18717e1 100644 --- a/pupil_src/player/export_launcher.py +++ b/pupil_src/player/export_launcher.py @@ -61,13 +61,16 @@ def __init__(self, g_pool,data_dir,frame_count): self.start_frame = c_int(0) self.end_frame = c_int(frame_count) + + def init_gui(self): + atb_label = "Export Recording" - atb_pos = 10,220 + atb_pos = 320,10 self._bar = atb.Bar(name =self.__class__.__name__, label=atb_label, help="export vizualization video", color=(50, 50, 50), alpha=100, - text='light', position=atb_pos,refresh=.1, size=(300, 300)) + text='light', position=atb_pos,refresh=.1, size=(300, 100)) self.update_bar() diff --git a/pupil_src/player/exporter.py b/pupil_src/player/exporter.py index 8b9d08f991..7f4991ff24 100644 --- a/pupil_src/player/exporter.py +++ b/pupil_src/player/exporter.py @@ -135,16 +135,9 @@ def export(should_terminate,frames_to_export,current_frame, data_dir,start_frame for p in plugins: p.update(frame,current_pupil_positions,events) - # render visual feedback from loaded plugins - for p in plugins: - p.img_display(frame) - - - # right now we dont have plugins so let just hardcode a dot here: - for gp in current_pupil_positions: - x_screen, y_screen = denormalize(gp['norm_gaze'], (width, height),flip_y=True) - cv2.circle(frame.img, (int(x_screen),int(y_screen)), 30, (60, 20, 220), 2, cv2.cv.CV_AA) - + # # render visual feedback from loaded plugins + # for p in plugins: + # p.gl_display(frame) writer.write(frame.img) current_frame.value +=1 diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index 2b274684c4..06f69c295d 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -62,6 +62,22 @@ from seek_bar import Seek_Bar from export_launcher import Export_Launcher from scan_path import Scan_Path + +name_by_index = ( 'Vis_Circle', + 'Vis_Polyline', + 'Scan_Path', + 'Vis_Light_Points') + +plugin_by_index = ( Vis_Circle, + Vis_Polyline, + Scan_Path, + Vis_Light_Points) + +index_by_name = dict(zip(name_by_index,range(len(name_by_index)))) +plugin_by_name = dict(zip(name_by_index,plugin_by_index)) +additive_plugins = (Vis_Circle,Vis_Polyline) + + import logging logger = logging.getLogger() logger.setLevel(logging.DEBUG) @@ -256,11 +272,35 @@ def get_play(): def set_play(value): g.play = value + + def open_plugin(selection,data): + if plugin_by_index[selection] not in additive_plugins: + for p in g.plugins: + if isinstance(p,plugin_by_index[selection]): + return + + g.plugins = [p for p in g.plugins if p.alive] + logger.debug('Open Plungin: %s'%name_by_index[selection]) + new_plugin = plugin_by_index[selection](g) + g.plugins.append(new_plugin) + g.plugins.sort(key=lambda p: p.order) + + if hasattr(new_plugin,'init_gui'): + new_plugin.init_gui() + # save the value for atb bar + data.value=selection + + def get_from_data(data): + """ + helper for atb getter and setter use + """ + return data.value + atb.init() # add main controls ATB bar bar = atb.Bar(name = "Controls", label="Controls", help="Scene controls", color=(50, 50, 50), alpha=100,valueswidth=150, - text='light', position=(10, 10),refresh=.1, size=(300, 200)) + text='light', position=(10, 10),refresh=.1, size=(300, 160)) bar.next_atb_pos = (10,220) bar.fps = c_float(0.0) bar.timestamp = time() @@ -274,6 +314,11 @@ def set_play(value): bar.add_var("display size", vtype=window_size_enum,setter=set_window_size,getter=get_from_data,data=bar.window_size) bar.add_var("play",vtype=c_bool,getter=get_play,setter=set_play,key="space") bar.add_var("next frame",getter=cap.get_frame_index) + + bar.plugin_to_load = c_int(0) + plugin_type_enum = atb.enum("Plug In",index_by_name) + bar.add_var("plugin",setter=open_plugin,getter=get_from_data,data=bar.plugin_to_load, vtype=plugin_type_enum) + bar.add_var("version of recording",bar.recording_version, readonly=True, help="version of the capture software used to make this recording") bar.add_var("version of player",bar.version, readonly=True, help="version of the Pupil Player") bar.add_button("exit", on_close,data=main_window,key="esc") @@ -288,24 +333,30 @@ def set_play(value): g.plugins.sort(key=lambda x: x.order) + for p in g.plugins: + if hasattr(p,'init_gui'): + p.init_gui() + while not glfwWindowShouldClose(main_window): update_fps() #grab new frame if g.play or g.new_seek: - new_frame = cap.get_frame() + test_frame = cap.get_frame() #end of video logic: pause at last frame. - if not new_frame: + if not test_frame: g.play=False else: - frame = new_frame + new_frame = test_frame if g.new_seek: - display_time = frame.timestamp + display_time = new_frame.timestamp g.new_seek = False + frame = new_frame.copy() + #new positons and events current_pupil_positions = positions_by_frame[frame.index] events = None diff --git a/pupil_src/player/player_methods.py b/pupil_src/player/player_methods.py index 49d37a9007..d423700a29 100644 --- a/pupil_src/player/player_methods.py +++ b/pupil_src/player/player_methods.py @@ -143,13 +143,14 @@ def convert_gaze_pos(gaze_list,capture_version): def transparent_cirlce(img,center,radius,color,thickness): + center = tuple(map(int,center)) + if thickness > 0: + pad = radius + 2 + thickness + else: + pad = radius + 3 + roi = slice(center[1]-pad,center[1]+pad),slice(center[0]-pad,center[0]+pad) + try: - center = tuple(map(int,center)) - if thickness > 0: - pad = radius + 2 + thickness - else: - pad = radius + 3 - roi = slice(center[1]-pad,center[1]+pad),slice(center[0]-pad,center[0]+pad) overlay = img[roi].copy() cv2.circle(overlay,(pad,pad), radius=radius, color=color[:3], thickness=thickness, lineType=cv2.cv.CV_AA) opacity = color[-1]/255. diff --git a/pupil_src/player/scan_path.py b/pupil_src/player/scan_path.py index 1951bac912..43f103d406 100644 --- a/pupil_src/player/scan_path.py +++ b/pupil_src/player/scan_path.py @@ -12,6 +12,7 @@ from plugin import Plugin import numpy as np import atb +from ctypes import c_float from methods import denormalize,normalize import logging logger = logging.getLogger(__name__) @@ -39,7 +40,12 @@ def __init__(self, g_pool): self.prev_gray = None + #gui + self.time = c_float(3.) + def update(self,frame,recent_pupil_positions,events): + + self.scan_path_timeframe = self.time.value img = frame.img img_shape = img.shape[:-1][::-1] # width,height @@ -49,7 +55,7 @@ def update(self,frame,recent_pupil_positions,events): #vars for calcOpticalFlowPyrLK lk_params = dict( winSize = (90, 90), - maxLevel = 2, + maxLevel = 3, criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 20, 0.03)) @@ -58,7 +64,7 @@ def update(self,frame,recent_pupil_positions,events): #lets update past gaze using optical flow: this is like sticking the gaze points onto the pixels of the img. if self.past_pupil_positions and succeeding_frame: past_screen_gaze = np.array([denormalize(ng['norm_gaze'] ,img_shape,flip_y=True) for ng in self.past_pupil_positions],dtype=np.float32) - new_pts, status, err = cv2.calcOpticalFlowPyrLK(self.prev_gray, gray_img,past_screen_gaze,minEigThreshold=0.01,**lk_params) + new_pts, status, err = cv2.calcOpticalFlowPyrLK(self.prev_gray, gray_img,past_screen_gaze,minEigThreshold=0.005,**lk_params) for gaze,new_gaze_pt,s,e in zip(self.past_pupil_positions,new_pts,status,err): if s: @@ -95,7 +101,33 @@ def update(self,frame,recent_pupil_positions,events): self.past_pupil_positions = recent_pupil_positions + def init_gui(self,pos=None): + pos = 10,380 + import atb + from time import time + + atb_label = "Scan Path" + self._bar = atb.Bar(name =self.__class__.__name__+str(time()), label=atb_label, + help="polyline", color=(50, 50, 50), alpha=100, + text='light', position=pos,refresh=.1, size=(300, 70)) + + self._bar.add_var('duration,sec',self.time,min=0) + + + self._bar.add_button('remove',self.unset_alive) + + def unset_alive(self): + self.alive = False + + def gl_display(self): pass + def cleanup(self): + """ called when the plugin gets terminated. + This happends either voluntary or forced. + if you have an atb bar or glfw window destroy it here. + """ + self._bar.destroy() + diff --git a/pupil_src/player/vis_circle.py b/pupil_src/player/vis_circle.py index 6b174e5a43..07be551145 100644 --- a/pupil_src/player/vis_circle.py +++ b/pupil_src/player/vis_circle.py @@ -12,7 +12,7 @@ from player_methods import transparent_cirlce from plugin import Plugin import numpy as np - +from ctypes import c_int,c_float,c_bool import cv2 from methods import denormalize @@ -24,15 +24,57 @@ def __init__(self, g_pool): self.g_pool = g_pool self.order = .9 self.prev_frame_idx = -1 - self.radius = 20 + + + self.radius = c_int(20) + self.color = (c_float*4)(1.,.2,.4,.5) + self.thickness = c_int(1) + self.full = c_bool(0) + def update(self,frame,recent_pupil_positions,events): - if self.prev_frame_idx != frame.index: - pts = [denormalize(pt['norm_gaze'],frame.img.shape[:-1][::-1],flip_y=True) for pt in recent_pupil_positions if pt['norm_gaze'] is not None] - for pt in pts: - transparent_cirlce(frame.img, tuple(map(int,pt)), radius=self.radius, color=(0,255,0,100), thickness=3) - self.prev_frame_idx = frame.index + color = map(lambda x:int(x*255),self.color) + color = color[:3][::-1]+color[-1:] + if self.full.value: + thickness= -1 + else: + thickness = self.thickness.value + + radius = self.radius.value + pts = [denormalize(pt['norm_gaze'],frame.img.shape[:-1][::-1],flip_y=True) for pt in recent_pupil_positions if pt['norm_gaze'] is not None] + for pt in pts: + transparent_cirlce(frame.img, pt, radius=radius, color=color, thickness=thickness) + self.prev_frame_idx = frame.index + + def init_gui(self,pos=None): + pos = 10,200 + import atb + from time import time + atb_label = "Gaze Circle" + self._bar = atb.Bar(name =self.__class__.__name__+str(time()), label=atb_label, + help="circle", color=(50, 50, 50), alpha=100, + text='light', position=pos,refresh=.1, size=(300, 100)) + + self._bar.add_var('radius',self.radius) + self._bar.add_var('thickness',self.thickness) + self._bar.add_var('full',self.full) + self._bar.add_var('color',self.color) + + + self._bar.add_button('remove',self.unset_alive) + + def unset_alive(self): + self.alive = False def gl_display(self): pass + + + + def cleanup(self): + """ called when the plugin gets terminated. + This happends either voluntary or forced. + if you have an atb bar or glfw window destroy it here. + """ + self._bar.destroy() \ No newline at end of file diff --git a/pupil_src/player/vis_light_points.py b/pupil_src/player/vis_light_points.py index 4a73dfb6c5..f461b285cf 100644 --- a/pupil_src/player/vis_light_points.py +++ b/pupil_src/player/vis_light_points.py @@ -12,6 +12,7 @@ from plugin import Plugin import numpy as np import atb +from ctypes import c_int from methods import denormalize import logging logger = logging.getLogger(__name__) @@ -34,37 +35,55 @@ def __init__(self, g_pool): def update(self,frame,recent_pupil_positions,events): #since we edit the img inplace we should not do it in pause mode... - if self.prev_frame_idx != frame.index: - img = frame.img - img_shape = img.shape[:-1][::-1]#width,height - norm_gaze = [ng['norm_gaze'] for ng in recent_pupil_positions if ng['norm_gaze'] is not None] - screen_gaze = [denormalize(ng,img_shape,flip_y=True) for ng in norm_gaze] + img = frame.img + img_shape = img.shape[:-1][::-1]#width,height + norm_gaze = [ng['norm_gaze'] for ng in recent_pupil_positions if ng['norm_gaze'] is not None] + screen_gaze = [denormalize(ng,img_shape,flip_y=True) for ng in norm_gaze] - overlay = np.ones(img.shape[:-1],dtype=img.dtype) + overlay = np.ones(img.shape[:-1],dtype=img.dtype) - # draw recent gaze postions as black dots on an overlay image. - for gaze_point in screen_gaze: - try: - overlay[int(gaze_point[1]),int(gaze_point[0])] = 0 - except: - pass + # draw recent gaze postions as black dots on an overlay image. + for gaze_point in screen_gaze: + try: + overlay[int(gaze_point[1]),int(gaze_point[0])] = 0 + except: + pass - out = cv2.distanceTransform(overlay,cv2.cv.CV_DIST_L2, 5) + out = cv2.distanceTransform(overlay,cv2.cv.CV_DIST_L2, 5) - # fix for opencv binding incositency - if type(out)==tuple: - out = out[0] + # fix for opencv binding incositency + if type(out)==tuple: + out = out[0] - overlay = 1/(out/20+1) + overlay = 1/(out/20+1) - img *= cv2.cvtColor(overlay,cv2.COLOR_GRAY2RGB) + img *= cv2.cvtColor(overlay,cv2.COLOR_GRAY2RGB) - self.prev_frame_idx = frame.index + def init_gui(self,pos=None): + pos = 10,450 + import atb + from time import time + atb_label = "Light Points" + self._bar = atb.Bar(name =self.__class__.__name__+str(time()), label=atb_label, + help="circle", color=(50, 50, 50), alpha=100, + text='light', position=pos,refresh=.1, size=(300, 100)) + + self._bar.add_button('remove',self.unset_alive) + + def unset_alive(self): + self.alive = False def gl_display(self): pass + + def cleanup(self): + """ called when the plugin gets terminated. + This happends either voluntary or forced. + if you have an atb bar or glfw window destroy it here. + """ + self._bar.destroy() diff --git a/pupil_src/player/vis_polyline.py b/pupil_src/player/vis_polyline.py index 036e02749e..24e46150cf 100644 --- a/pupil_src/player/vis_polyline.py +++ b/pupil_src/player/vis_polyline.py @@ -11,6 +11,7 @@ from gl_utils import draw_gl_points_norm from plugin import Plugin import numpy as np +from ctypes import c_int,c_float,c_bool import cv2 @@ -23,16 +24,51 @@ def __init__(self, g_pool): self.g_pool = g_pool self.order = .9 self.prev_frame_idx = -1 + self.color = (c_float*3)(1.,.2,.4) + self.thickness = c_int(1) def update(self,frame,recent_pupil_positions,events): - if self.prev_frame_idx != frame.index: - pts = [denormalize(pt['norm_gaze'],frame.img.shape[:-1][::-1],flip_y=True) for pt in recent_pupil_positions if pt['norm_gaze'] is not None] - if pts: - pts = np.array([pts],dtype=np.int32) - cv2.polylines(frame.img, pts, isClosed=False, color=(0,255,0), thickness=1, lineType=cv2.cv.CV_AA) + color = map(lambda x:int(x*255),self.color) + color = color[::-1] - self.prev_frame_idx = frame.index + thickness = self.thickness.value + + pts = [denormalize(pt['norm_gaze'],frame.img.shape[:-1][::-1],flip_y=True) for pt in recent_pupil_positions if pt['norm_gaze'] is not None] + if pts: + pts = np.array([pts],dtype=np.int32) + cv2.polylines(frame.img, pts, isClosed=False, color=color, thickness=thickness, lineType=cv2.cv.CV_AA) + + self.prev_frame_idx = frame.index + + + + def init_gui(self,pos=None): + pos = 10,310 + import atb + atb_label = "Gaze Polyline" + from time import time + self._bar = atb.Bar(name = self.__class__.__name__+str(time()), label=atb_label, + help="polyline", color=(50, 50, 50), alpha=100, + text='light', position=pos,refresh=.1, size=(300, 70)) + + self._bar.add_var('thickness',self.thickness,min=1) + self._bar.add_var('color',self.color) + + + self._bar.add_button('remove',self.unset_alive) + + def unset_alive(self): + self.alive = False def gl_display(self): pass + + + + def cleanup(self): + """ called when the plugin gets terminated. + This happends either voluntary or forced. + if you have an atb bar or glfw window destroy it here. + """ + self._bar.destroy() \ No newline at end of file diff --git a/pupil_src/shared_modules/uvc_capture/__init__.py b/pupil_src/shared_modules/uvc_capture/__init__.py index 80e3231cef..974e87930c 100644 --- a/pupil_src/shared_modules/uvc_capture/__init__.py +++ b/pupil_src/shared_modules/uvc_capture/__init__.py @@ -52,6 +52,8 @@ def __init__(self, timestamp,img,index=None,compressed_img=None, compressed_pix_ self.compressed_img = compressed_img self.compressed_pix_fmt = compressed_pix_fmt + def copy(self): + return Frame(self.timestamp,self.img.copy(),self.index) class FileCapture(): """