diff --git a/pupil_src/capture/eye.py b/pupil_src/capture/eye.py index 82e129f9b9..aed356ef05 100644 --- a/pupil_src/capture/eye.py +++ b/pupil_src/capture/eye.py @@ -10,7 +10,7 @@ import os from time import time, sleep -import shelve +from file_methods import Persistent_Dict import logging from ctypes import c_int,c_bool,c_float import numpy as np @@ -119,7 +119,7 @@ def get_from_data(data): # load session persistent settings - session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_eye'),protocol=2) + session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye') ) def load(var_name,default): return session_settings.get(var_name,default) def save(var_name,var): diff --git a/pupil_src/capture/pupil_detectors/canny_detector.py b/pupil_src/capture/pupil_detectors/canny_detector.py index 45ca693018..164b94dd66 100644 --- a/pupil_src/capture/pupil_detectors/canny_detector.py +++ b/pupil_src/capture/pupil_detectors/canny_detector.py @@ -19,7 +19,7 @@ import cv2 from time import sleep -import shelve +from file_methods import Persistent_Dict import numpy as np from methods import * import atb @@ -47,7 +47,7 @@ def __init__(self,g_pool): super(Canny_Detector, self).__init__() # load session persistent settings - self.session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_detector'),protocol=2) + self.session_settings =Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_detector') ) # coase pupil filter params self.coarse_detection = c_bool(self.load('coarse_detection',True)) diff --git a/pupil_src/capture/recorder.py b/pupil_src/capture/recorder.py index 064df0613f..e2520397ea 100644 --- a/pupil_src/capture/recorder.py +++ b/pupil_src/capture/recorder.py @@ -91,13 +91,15 @@ def get_rec_time_str(self): return strftime("%H:%M:%S", rec_time) def update(self,frame,recent_pupil_positons,events): - self.frame_count += 1 + # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)) for p in recent_pupil_positons: if p['norm_pupil'] is not None: gaze_pt = p['norm_gaze'][0],p['norm_gaze'][1],p['norm_pupil'][0],p['norm_pupil'][1],p['timestamp'],p['confidence'] self.gaze_list.append(gaze_pt) self.timestamps.append(frame.timestamp) self.writer.write(frame.img) + self.frame_count += 1 + def stop_and_destruct(self): #explicit release of VideoWriter @@ -109,6 +111,7 @@ def stop_and_destruct(self): self.eye_tx.send(None) except: logger.warning("Could not stop eye-recording. Please report this bug!") + gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy") np.save(gaze_list_path,np.asarray(self.gaze_list)) @@ -116,25 +119,18 @@ def stop_and_destruct(self): np.save(timestamps_path,np.array(self.timestamps)) try: - surface_definitions_file = glob(os.path.join(self.g_pool.user_dir,"surface_definitions*"))[0].rsplit(os.path.sep,1)[-1] - copy2(os.path.join(self.g_pool.user_dir,surface_definitions_file),os.path.join(self.rec_path,surface_definitions_file)) + copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions")) except: logger.info("No surface_definitions data found. You may want this if you do marker tracking.") try: - cal_pt_cloud = np.load(os.path.join(self.g_pool.user_dir,"cal_pt_cloud.npy")) - cal_pt_cloud_path = os.path.join(self.rec_path, "cal_pt_cloud.npy") - np.save(cal_pt_cloud_path, cal_pt_cloud) + copy2(os.path.join(self.g_pool.user_dir,"cal_pt_cloud.npy"),os.path.join(self.rec_path,"cal_pt_cloud.npy")) except: logger.warning("No calibration data found. Please calibrate first.") try: - camera_matrix = np.load(os.path.join(self.g_pool.user_dir,"camera_matrix.npy")) - dist_coefs = np.load(os.path.join(self.g_pool.user_dir,"dist_coefs.npy")) - cam_path = os.path.join(self.rec_path, "camera_matrix.npy") - dist_path = os.path.join(self.rec_path, "dist_coefs.npy") - np.save(cam_path, camera_matrix) - np.save(dist_path, dist_coefs) + copy2(os.path.join(self.g_pool.user_dir,"camera_matrix.npy"),os.path.join(self.rec_path,"camera_matrix.npy")) + copy2(os.path.join(self.g_pool.user_dir,"dist_coefs.npy"),os.path.join(self.rec_path,"dist_coefs.npy")) except: logger.info("No camera intrinsics found.") diff --git a/pupil_src/capture/world.py b/pupil_src/capture/world.py index 43e41945ed..f9b48c3d32 100644 --- a/pupil_src/capture/world.py +++ b/pupil_src/capture/world.py @@ -19,7 +19,7 @@ import os, sys from time import time -import shelve +from file_methods import Persistent_Dict import logging from ctypes import c_int,c_bool,c_float,create_string_buffer import numpy as np @@ -64,6 +64,8 @@ def on_resize(window,w, h): atb.TwWindowSize(*map(int,fb_size)) adjust_gl_view(w,h,window) glfwMakeContextCurrent(active_window) + for p in g_pool.plugins: + p.on_window_resize(window,w,h) def on_iconify(window,iconfied): if not isinstance(cap,FakeCapture): @@ -104,37 +106,18 @@ def on_close(window): # load session persistent settings - session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_world'),protocol=2) + session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world')) def load(var_name,default): return session_settings.get(var_name,default) def save(var_name,var): session_settings[var_name] = var - # load last calibration data - try: - pt_cloud = np.load(os.path.join(g_pool.user_dir,'cal_pt_cloud.npy')) - logger.info("Using calibration found in %s" %g_pool.user_dir) - map_pupil = calibrate.get_map_from_cloud(pt_cloud,(width,height)) - except: - logger.info("No calibration found.") - def map_pupil(vector): - """ 1 to 1 mapping - """ - return vector - # any object we attach to the g_pool object now will only be visible to this process! - # vars should be declared here to make them visible to the reader. - g_pool.plugins = [] - g_pool.map_pupil = map_pupil - g_pool.update_textures = c_bool(1) # Initialize capture cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase) - if isinstance(cap,FakeCapture): - g_pool.update_textures.value = False - # Get an image from the grabber try: frame = cap.get_frame() @@ -142,10 +125,30 @@ def map_pupil(vector): logger.error("Could not retrieve image from capture") cap.close() return - height,width = frame.img.shape[:2] + + # load last calibration data + try: + pt_cloud = np.load(os.path.join(g_pool.user_dir,'cal_pt_cloud.npy')) + logger.debug("Using calibration found in %s" %g_pool.user_dir) + map_pupil = calibrate.get_map_from_cloud(pt_cloud,(width,height)) + except : + logger.debug("No calibration found.") + def map_pupil(vector): + """ 1 to 1 mapping """ + return vector + + + # any object we attach to the g_pool object *from now on* will only be visible to this process! + # vars should be declared here to make them visible to the code reader. + g_pool.plugins = [] + g_pool.map_pupil = map_pupil + g_pool.update_textures = c_bool(1) + if isinstance(cap,FakeCapture): + g_pool.update_textures.value = False g_pool.capture = cap + # helpers called by the main atb bar def update_fps(): old_time, bar.timestamp = bar.timestamp, time() @@ -336,7 +339,6 @@ def reset_timebase(): #check if a plugin need to be destroyed g_pool.plugins = [p for p in g_pool.plugins if p.alive] - # render camera image glfwMakeContextCurrent(world_window) diff --git a/pupil_src/player/export_launcher.py b/pupil_src/player/export_launcher.py index cd95adfd1b..f2933068ec 100644 --- a/pupil_src/player/export_launcher.py +++ b/pupil_src/player/export_launcher.py @@ -74,8 +74,6 @@ def __init__(self, g_pool,data_dir,frame_count): default_path = "world_viz.avi" self.rec_name = create_string_buffer(default_path,512) - self.start_frame = c_int(0) - self.end_frame = c_int(frame_count) def init_gui(self): @@ -96,8 +94,8 @@ def update_bar(self): self._bar.clear() self._bar.add_var('export name',self.rec_name, help="Supply export video recording name. The export will be in the recording dir. If you give a path the export will end up there instead.") - self._bar.add_var('start frame',self.start_frame,help="Supply start frame no. Negative numbers will count from the end. The behaves like python list indexing") - self._bar.add_var('end frame',self.end_frame,help="Supply end frame no. Negative numbers will count from the end. The behaves like python list indexing") + self._bar.add_var('start frame',vtype=c_int,getter=self.g_pool.trim_marks.atb_get_in_mark,setter= self.g_pool.trim_marks.atb_set_in_mark, help="Supply start frame no. Negative numbers will count from the end. The behaves like python list indexing") + self._bar.add_var('end frame',vtype=c_int,getter=self.g_pool.trim_marks.atb_get_out_mark,setter= self.g_pool.trim_marks.atb_set_out_mark,help="Supply end frame no. Negative numbers will count from the end. The behaves like python list indexing") self._bar.add_button('new export',self.add_export) for job,i in zip(self.exports,range(len(self.exports)))[::-1]: @@ -134,8 +132,8 @@ def add_export(self): current_frame = RawValue(c_int,0) data_dir = self.data_dir - start_frame= self.start_frame.value - end_frame= self.end_frame.value + start_frame= self.g_pool.trim_marks.in_mark + end_frame= self.g_pool.trim_marks.out_mark+1 #end_frame is exclusive plugins = [] # Here we make clones of every plugin that supports it. diff --git a/pupil_src/player/exporter.py b/pupil_src/player/exporter.py index d9ac65f3de..3f658d92b0 100644 --- a/pupil_src/player/exporter.py +++ b/pupil_src/player/exporter.py @@ -146,7 +146,7 @@ def export(should_terminate,frames_to_export,current_frame, data_dir,start_frame p = plugin_by_name[name](g,**args) plugins.append(p) except: - logger.warning("Plugin '%s' failed to load." %name) + logger.warning("Plugin '%s' could not be loaded in exporter." %name) while frames_to_export.value - current_frame.value > 0: diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index 1063d49f1a..2f1ddc0544 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -46,7 +46,7 @@ import logging #set up root logger before other imports logger = logging.getLogger() -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.WARNING) #since we are not using OS.fork on MacOS we need to do a few extra things to log our exports correctly. if platform.system() == 'Darwin': if __name__ == '__main__': #clear log if main @@ -72,7 +72,7 @@ logging.getLogger("OpenGL").addHandler(logging.NullHandler()) logger = logging.getLogger(__name__) -import shelve +from file_methods import Persistent_Dict from time import time,sleep from ctypes import c_int,c_bool,c_float,create_string_buffer import numpy as np @@ -81,7 +81,7 @@ from glfw import * import atb -from uvc_capture import autoCreateCapture,EndofVideoFileError,FakeCapture +from uvc_capture import autoCreateCapture,EndofVideoFileError,FileSeekError,FakeCapture # helpers/utils from methods import normalize, denormalize,Temp @@ -105,14 +105,15 @@ from display_gaze import Display_Gaze from vis_light_points import Vis_Light_Points from seek_bar import Seek_Bar +from trim_marks import Trim_Marks from export_launcher import Export_Launcher from scan_path import Scan_Path -from marker_detector import Marker_Detector +from offline_marker_detector import Offline_Marker_Detector from pupil_server import Pupil_Server from filter_fixations import Filter_Fixations from manual_gaze_correction import Manual_Gaze_Correction -plugin_by_index = (Vis_Circle,Vis_Cross, Vis_Polyline, Vis_Light_Points,Scan_Path,Filter_Fixations,Manual_Gaze_Correction,Marker_Detector,Pupil_Server) +plugin_by_index = (Vis_Circle,Vis_Cross, Vis_Polyline, Vis_Light_Points,Scan_Path,Filter_Fixations,Manual_Gaze_Correction,Offline_Marker_Detector,Pupil_Server) name_by_index = [p.__name__ for p in plugin_by_index] index_by_name = dict(zip(name_by_index,range(len(name_by_index)))) plugin_by_name = dict(zip(name_by_index,plugin_by_index)) @@ -130,6 +131,8 @@ def on_resize(window,w, h): fb_size = denormalize(norm_size,glfwGetFramebufferSize(window)) atb.TwWindowSize(*map(int,fb_size)) glfwMakeContextCurrent(active_window) + for p in g.plugins: + p.on_window_resize(window,w,h) def on_key(window, key, scancode, action, mods): if not atb.TwEventKeyboardGLFW(key,action): @@ -160,14 +163,14 @@ def on_scroll(window,x,y): def on_close(window): glfwSetWindowShouldClose(main_window,True) - logger.info('Process closing from window') + logger.debug('Process closing from window') try: rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: - rec_dir = "/Users/mkassner/Downloads/1-4/000/" + rec_dir = '/home/mkassner/Desktop/003' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: @@ -210,7 +213,7 @@ def on_close(window): # load session persistent settings - session_settings = shelve.open(os.path.join(user_dir,"user_settings"),protocol=2) + session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings")) def load(var_name,default): return session_settings.get(var_name,default) def save(var_name,var): @@ -242,7 +245,7 @@ def save(var_name,var): glfwSetScrollCallback(main_window,on_scroll) - # create container for globally scoped vars (within world) + # create container for globally scoped varfs (within world) g = Temp() g.plugins = [] g.play = False @@ -250,6 +253,8 @@ def save(var_name,var): g.user_dir = user_dir g.rec_dir = rec_dir g.app = 'player' + g.timestamps = timestamps + g.positions_by_frame = positions_by_frame @@ -280,11 +285,17 @@ def set_play(value): g.play = value def next_frame(): - cap.seek_to_frame(cap.get_frame_index()) + try: + cap.seek_to_frame(cap.get_frame_index()) + except FileSeekError: + pass g.new_seek = True def prev_frame(): - cap.seek_to_frame(cap.get_frame_index()-2) + try: + cap.seek_to_frame(cap.get_frame_index()-2) + except FileSeekError: + pass g.new_seek = True @@ -331,7 +342,7 @@ def get_from_data(data): bar.add_var("play",vtype=c_bool,getter=get_play,setter=set_play,key="space") bar.add_button('step next',next_frame,key='right') bar.add_button('step prev',prev_frame,key='left') - bar.add_var("frame index",getter=lambda:cap.get_frame_index()-1 ) + bar.add_var("frame index",vtype=c_int,getter=lambda:cap.get_frame_index()-1 ) bar.plugin_to_load = c_int(0) plugin_type_enum = atb.enum("Plug In",index_by_name) @@ -349,6 +360,8 @@ def get_from_data(data): #we always load these plugins g.plugins.append(Export_Launcher(g,data_dir=rec_dir,frame_count=len(timestamps))) g.plugins.append(Seek_Bar(g,capture=cap)) + g.trim_marks = Trim_Marks(g,capture=cap) + g.plugins.append(g.trim_marks) #these are loaded based on user settings for initializer in load('plugins',[]): @@ -397,8 +410,7 @@ def get_from_data(data): g.new_seek = False frame = new_frame.copy() - - #new positons and events we make a deepcopy just like the image should be a copy. + #new positons and events we make a deepcopy just like the image is a copy. current_pupil_positions = deepcopy(positions_by_frame[frame.index]) events = [] @@ -462,7 +474,7 @@ def get_from_data(data): if __name__ == '__main__': freeze_support() - if 1: + if 0: main() else: import cProfile,subprocess,os diff --git a/pupil_src/player/player_methods.py b/pupil_src/player/player_methods.py index d94b7f1eeb..09256255ed 100644 --- a/pupil_src/player/player_methods.py +++ b/pupil_src/player/player_methods.py @@ -30,10 +30,15 @@ def correlate_gaze(gaze_list,timestamps): gaze_list = list(gaze_list) timestamps = list(timestamps) - positions_by_frame = [[] for i in timestamps] + frame_idx = 0 - data_point = gaze_list.pop(0) + try: + data_point = gaze_list.pop(0) + except: + logger.warning("No gaze positons in this recording.") + return positions_by_frame + gaze_timestamp = data_point[4] while gaze_list: @@ -43,7 +48,7 @@ def correlate_gaze(gaze_list,timestamps): except IndexError: break if gaze_timestamp <= t_between_frames: - positions_by_frame[frame_idx].append({'norm_gaze':(data_point[0],data_point[1]),'norm_pupil': (data_point[2],data_point[3]), 'timestamp':gaze_timestamp}) + positions_by_frame[frame_idx].append({'norm_gaze':(data_point[0],data_point[1]),'norm_pupil': (data_point[2],data_point[3]), 'timestamp':data_point[4],'confidence':data_point[5]}) data_point = gaze_list.pop(0) gaze_timestamp = data_point[4] else: diff --git a/pupil_src/player/seek_bar.py b/pupil_src/player/seek_bar.py index 9089596c29..a79b831889 100644 --- a/pupil_src/player/seek_bar.py +++ b/pupil_src/player/seek_bar.py @@ -14,9 +14,7 @@ from glfw import glfwGetWindowSize,glfwGetCurrentContext,glfwGetCursorPos,GLFW_RELEASE,GLFW_PRESS from plugin import Plugin -import numpy as np -from methods import denormalize import logging logger = logging.getLogger(__name__) @@ -24,7 +22,6 @@ class Seek_Bar(Plugin): """docstring for Seek_Bar seek bar displays a bar at the bottom of the screen when you hover close to it. it will show the current positon and allow you to drag to any postion in the video file. - """ def __init__(self, g_pool,capture): super(Seek_Bar, self).__init__() @@ -33,23 +30,34 @@ def __init__(self, g_pool,capture): self.current_frame_index = self.cap.get_frame_index() self.frame_count = self.cap.get_frame_count() - self.norm_seek_pos = self.current_frame_index/float(self.frame_count) self.drag_mode = False self.was_playing = True #display layout - self.padding = 20. + self.padding = 20. #in sceen pixel + + + def init_gui(self): + self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext())) + + def on_window_resize(self,window,w,h): + self.window_size = w,h + self.h_pad = self.padding * self.frame_count/float(w) + self.v_pad = self.padding * 1./h def update(self,frame,recent_pupil_positions,events): self.current_frame_index = frame.index - self.norm_seek_pos = self.current_frame_index/float(self.frame_count) if self.drag_mode: - pos = glfwGetCursorPos(glfwGetCurrentContext()) - norm_seek_pos, _ = self.screen_to_seek_bar(pos) - norm_seek_pos = min(1,max(0,norm_seek_pos)) - if abs(norm_seek_pos-self.norm_seek_pos) >=.01: - seek_pos = int(norm_seek_pos*self.frame_count) - self.cap.seek_to_frame(seek_pos) + x,y = glfwGetCursorPos(glfwGetCurrentContext()) + x,_ = self.screen_to_seek_bar((x,y)) + seek_pos = min(self.frame_count,max(0,x)) + if abs(seek_pos-self.current_frame_index) >=.002*self.frame_count: + seek_pos = int(min(seek_pos,self.frame_count-5)) #the last frames can be problematic to seek to + try: + self.cap.seek_to_frame(seek_pos) + self.current_frame_index = seek_pos + except: + pass self.g_pool.new_seek = True @@ -60,7 +68,7 @@ def on_click(self,img_pos,button,action): pos = glfwGetCursorPos(glfwGetCurrentContext()) #drag the seek point if action == GLFW_PRESS: - screen_seek_pos = self.seek_bar_to_screen((self.norm_seek_pos,0)) + screen_seek_pos = self.seek_bar_to_screen((self.current_frame_index,0)) dist = abs(pos[0]-screen_seek_pos[0])+abs(pos[1]-screen_seek_pos[1]) if dist < 20: self.drag_mode=True @@ -69,20 +77,22 @@ def on_click(self,img_pos,button,action): elif action == GLFW_RELEASE: if self.drag_mode: - norm_seek_pos, _ = self.screen_to_seek_bar(pos) - norm_seek_pos = min(1,max(0,norm_seek_pos)) - seek_pos = int(norm_seek_pos*self.frame_count) - self.cap.seek_to_frame(seek_pos) + x, _ = self.screen_to_seek_bar(pos) + x = int(min(self.frame_count-5,max(0,x))) + try: + self.cap.seek_to_frame(x) + except: + pass self.g_pool.new_seek = True self.drag_mode=False self.g_pool.play = self.was_playing def seek_bar_to_screen(self,pos): - width,height = glfwGetWindowSize(glfwGetCurrentContext()) + width,height = self.window_size x,y=pos y = 1-y - x = x*(width-2*self.padding)+self.padding + x = (x/float(self.frame_count))*(width-self.padding*2) +self.padding y = y*(height-2*self.padding)+self.padding return x,y @@ -90,7 +100,7 @@ def seek_bar_to_screen(self,pos): def screen_to_seek_bar(self,pos): width,height = glfwGetWindowSize(glfwGetCurrentContext()) x,y=pos - x = (x-self.padding)/(width-2*self.padding) + x = (x-self.padding)/(width-2*self.padding)*self.frame_count y = (y-self.padding)/(height-2*self.padding) return x,1-y @@ -99,10 +109,7 @@ def gl_display(self): glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() - width,height = glfwGetWindowSize(glfwGetCurrentContext()) - h_pad = self.padding/width - v_pad = self.padding/height - gluOrtho2D(-h_pad, 1+h_pad, -v_pad, 1+v_pad) # gl coord convention + gluOrtho2D(-self.h_pad, (self.frame_count)+self.h_pad, -self.v_pad, 1+self.v_pad) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() @@ -114,14 +121,12 @@ def gl_display(self): color1 = (.25,.8,.8,.5) color2 = (.25,.8,.8,1.) - draw_gl_polyline( [(0,0),(self.norm_seek_pos,0)],color=color1) - draw_gl_polyline( [(self.norm_seek_pos,0),(1,0)],color=(.5,.5,.5,.5)) - draw_gl_point((self.norm_seek_pos,0),color=color1,size=40) - draw_gl_point((self.norm_seek_pos,0),color=color2,size=10) + draw_gl_polyline( [(0,0),(self.current_frame_index,0)],color=color1) + draw_gl_polyline( [(self.current_frame_index,0),(self.frame_count,0)],color=(.5,.5,.5,.5)) + draw_gl_point((self.current_frame_index,0),color=color1,size=40) + draw_gl_point((self.current_frame_index,0),color=color2,size=10) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) - glPopMatrix() - - + glPopMatrix() \ No newline at end of file diff --git a/pupil_src/player/trim_marks.py b/pupil_src/player/trim_marks.py new file mode 100644 index 0000000000..c357318435 --- /dev/null +++ b/pupil_src/player/trim_marks.py @@ -0,0 +1,156 @@ +''' +(*)~---------------------------------------------------------------------------------- + Pupil - eye tracking platform + Copyright (C) 2012-2014 Pupil Labs + + Distributed under the terms of the CC BY-NC-SA License. + License details are in the file license.txt, distributed as part of this software. +----------------------------------------------------------------------------------~(*) +''' + +from gl_utils import draw_gl_polyline,draw_gl_point +from OpenGL.GL import * +from OpenGL.GLU import gluOrtho2D + +from glfw import glfwGetWindowSize,glfwGetCurrentContext,glfwGetCursorPos,GLFW_RELEASE,GLFW_PRESS +from plugin import Plugin + +import logging +logger = logging.getLogger(__name__) + +class Trim_Marks(Plugin): + """docstring for Trim_Mark + """ + def __init__(self, g_pool,capture): + super(Trim_Marks, self).__init__() + self.order = .8 + self.g_pool = g_pool + self.capture = capture + self.frame_count = capture.get_frame_count() + self._in_mark = 0 + self._out_mark = self.frame_count + self.drag_in = False + self.drag_out = False + #display layout + self.padding = 20. #in sceen pixel + + @property + def in_mark(self): + return self._in_mark + + @in_mark.setter + def in_mark(self, value): + self._in_mark = int(min(self._out_mark,max(0,value))) + + @property + def out_mark(self): + return self._out_mark + + @out_mark.setter + def out_mark(self, value): + self._out_mark = int(min(self.frame_count,max(self.in_mark,value))) + + + def init_gui(self): + self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext())) + + def on_window_resize(self,window,w,h): + self.window_size = w,h + self.h_pad = self.padding * self.frame_count/float(w) + self.v_pad = self.padding * 1./h + + def update(self,frame,recent_pupil_positions,events): + + if frame.index == self.out_mark or frame.index == self.in_mark: + self.g_pool.play=False + + if self.drag_in: + x,y = glfwGetCursorPos(glfwGetCurrentContext()) + x,_ = self.screen_to_bar_space((x,y)) + self.in_mark = x + + elif self.drag_out: + x,y = glfwGetCursorPos(glfwGetCurrentContext()) + x,_ = self.screen_to_bar_space((x,y)) + self.out_mark = x + + + def on_click(self,img_pos,button,action): + """ + gets called when the user clicks in the window screen + """ + pos = glfwGetCursorPos(glfwGetCurrentContext()) + #drag the seek point + if action == GLFW_PRESS: + screen_in_mark_pos = self.bar_space_to_screen((self.in_mark,0)) + screen_out_mark_pos = self.bar_space_to_screen((self.out_mark,0)) + + #in mark + dist = abs(pos[0]-screen_in_mark_pos[0])+abs(pos[1]-screen_in_mark_pos[1]) + if dist < 10: + if self.distance_in_pix(self.in_mark,self.capture.get_frame_index()) > 20: + self.drag_in=True + return + #out mark + dist = abs(pos[0]-screen_out_mark_pos[0])+abs(pos[1]-screen_out_mark_pos[1]) + if dist < 10: + if self.distance_in_pix(self.out_mark,self.capture.get_frame_index()) > 20: + self.drag_out=True + + elif action == GLFW_RELEASE: + self.drag_out = False + self.drag_in = False + + def atb_get_in_mark(self): + return self.in_mark + def atb_get_out_mark(self): + return self.out_mark + def atb_set_in_mark(self,val): + self.in_mark = val + def atb_set_out_mark(self,val): + self.out_mark = val + + def distance_in_pix(self,frame_pos_0,frame_pos_1): + fr0_screen_x,_ = self.bar_space_to_screen((frame_pos_0,0)) + fr1_screen_x,_ = self.bar_space_to_screen((frame_pos_1,0)) + return abs(fr0_screen_x-fr1_screen_x) + + + def bar_space_to_screen(self,pos): + width,height = self.window_size + x,y=pos + y = 1-y + x = (x/float(self.frame_count))*(width-self.padding*2) +self.padding + y = y*(height-2*self.padding)+self.padding + return x,y + + + def screen_to_bar_space(self,pos): + width,height = glfwGetWindowSize(glfwGetCurrentContext()) + x,y=pos + x = (x-self.padding)/(width-2*self.padding)*self.frame_count + y = (y-self.padding)/(height-2*self.padding) + return x,1-y + + def gl_display(self): + + glMatrixMode(GL_PROJECTION) + glPushMatrix() + glLoadIdentity() + gluOrtho2D(-self.h_pad, (self.frame_count)+self.h_pad, -self.v_pad, 1+self.v_pad) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) + glMatrixMode(GL_MODELVIEW) + glPushMatrix() + glLoadIdentity() + + color1 = (.1,.9,.2,.5) + color2 = (.1,.9,.2,.5) + + if self.in_mark != 0 or self.out_mark != self.frame_count: + draw_gl_polyline( [(self.in_mark,0),(self.out_mark,0)],color=(.1,.9,.2,.5),thickness=2) + draw_gl_point((self.in_mark,0),color=color2,size=10) + draw_gl_point((self.out_mark,0),color=color2,size=10) + + glMatrixMode(GL_PROJECTION) + glPopMatrix() + glMatrixMode(GL_MODELVIEW) + glPopMatrix() \ No newline at end of file diff --git a/pupil_src/player/vis_polyline.py b/pupil_src/player/vis_polyline.py index 250f2f1d9f..3d6f42ef38 100644 --- a/pupil_src/player/vis_polyline.py +++ b/pupil_src/player/vis_polyline.py @@ -48,7 +48,7 @@ def init_gui(self,pos=None): self._bar = atb.Bar(name = self.__class__.__name__+str(id(self)), label=atb_label, help="polyline", color=(50, 50, 50), alpha=100, text='light', position=pos,refresh=.1, size=self.gui_settings['size']) - + self._bar.iconified = self.gui_settings['iconified'] self._bar.add_var('color',self.color) self._bar.add_var('thickness',self.thickness,min=1) self._bar.add_button('remove',self.unset_alive) diff --git a/pupil_src/shared_modules/atb/raw.py b/pupil_src/shared_modules/atb/raw.py index 073ba94621..38c912370e 100644 --- a/pupil_src/shared_modules/atb/raw.py +++ b/pupil_src/shared_modules/atb/raw.py @@ -65,6 +65,7 @@ TwMouseMotion = __dll__.TwMouseMotion TwWindowSize = __dll__.TwWindowSize TwMouseWheel = __dll__.TwMouseWheel +TwSetCurrentWindow = __dll__.TwSetCurrentWindow TwEventMouseButtonGLFW = __dll__.TwEventMouseButtonGLFW # TwEventMouseMotionGLUT = __dll__.TwEventMousePosGLFW diff --git a/pupil_src/shared_modules/cache_list.py b/pupil_src/shared_modules/cache_list.py new file mode 100644 index 0000000000..44dd32f9f3 --- /dev/null +++ b/pupil_src/shared_modules/cache_list.py @@ -0,0 +1,124 @@ + +import logging +logger = logging.getLogger(__name__) +import itertools + +class Cache_List(list): + """Cache list is a list of False + [False,False,False] + with update() 'False' can be overwritten with a result (anything not 'False') + self.visited_ranges show ranges where the cache contect is False + self.positive_ranges show ranges where the cache does not evaluate as 'False' using eval_fn + this allows to use ranges a a way of showing where no caching has happed (default) or whatever you do with eval_fn + self.complete indicated that the cache list has no unknowns aka False + """ + + def __init__(self, init_list,positive_eval_fn=None): + super(Cache_List, self).__init__(init_list) + + self.visited_eval_fn = lambda x: x!=False + self._visited_ranges = init_ranges(l = self,eval_fn = self.visited_eval_fn ) + self._togo = self.count(False) + self.length = len(self) + + if positive_eval_fn == None: + self.positive_eval_fn = lambda x: False + self._positive_ranges = [] + else: + self.positive_eval_fn = positive_eval_fn + self._positive_ranges = init_ranges(l = self,eval_fn = self.positive_eval_fn ) + + @property + def visited_ranges(self): + return self._visited_ranges + + @visited_ranges.setter + def visited_ranges(self, value): + raise Exception("Read only") + + @property + def positive_ranges(self): + return self._positive_ranges + + @positive_ranges.setter + def positive_ranges(self, value): + raise Exception("Read only") + + + @property + def complete(self): + return self._togo == 0 + + @complete.setter + def complete(self, value): + raise Exception("Read only") + + + def update(self,key,item): + if self[key] != False: + logger.warning("You are overwriting a precached result.") + self[key] = item + self._visited_ranges = init_ranges(l = self,eval_fn = self.visited_eval_fn ) + self._positive_ranges = init_ranges(l = self,eval_fn = self.positive_eval_fn ) + + elif item != False: + #unvisited + self._togo -= 1 + self[key] = item + + update_ranges(self._visited_ranges,key) + if self.positive_eval_fn(item): + update_ranges(self._positive_ranges,key) + else: + #writing False to list entry already false, do nothing + pass + + + def to_list(self): + return list(self) + + + +def init_ranges(l,eval_fn): + i = -1 + ranges = [] + for t,g in itertools.groupby(l,eval_fn): + l = i + 1 + i += len(list(g)) + if t: + ranges.append([l,i]) + return ranges + +def update_ranges(l,i): + for _range in l: + #most common case: extend a range + if i == _range[0]-1: + _range[0] = i + merge_ranges(l) + return + elif i == _range[1]+1: + _range[1] = i + merge_ranges(l) + return + #somewhere outside of range proximity + l.append([i,i]) + l.sort(key=lambda x: x[0]) + +def merge_ranges(l): + for i in range(len(l)-1): + if l[i][1] == l[i+1][0]-1: + #merge touching fields + l[i] = ([l[i][0],l[i+1][1]]) + #del second field + del l[i+1] + return + return + + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + cl = Cache_List(range(100),positive_eval_fn = lambda x: bool(x%2)) + cl.update(0,1) + cl.update(4,1) + print cl.positive_ranges + print cl \ No newline at end of file diff --git a/pupil_src/shared_modules/file_methods.py b/pupil_src/shared_modules/file_methods.py new file mode 100644 index 0000000000..6aab06bc9b --- /dev/null +++ b/pupil_src/shared_modules/file_methods.py @@ -0,0 +1,56 @@ + +import cPickle as pickle +import os +import logging +logger = logging.getLogger(__name__) + +class Persistent_Dict(dict): + """a dict class that uses pickle to save inself to file""" + def __init__(self, file_path): + super(Persistent_Dict, self).__init__() + self.file_path = os.path.expanduser(file_path) + try: + with open(self.file_path,'rb') as fh: + try: + self.update(pickle.load(fh)) + except: #KeyError,EOFError + logger.warning("Session settings file '%s'could not be read. Will overwrite on exit."%self.file_path) + except IOError: + logger.debug("Session settings file '%s' not found. Will make new one on exit."%self.file_path) + + + def save(self): + d = {} + d.update(self) + try: + with open(self.file_path,'wb') as fh: + pickle.dump(d,fh,-1) + except IOError: + logger.warning("Could not save session settings to '%s'"%self.file_path) + + + + def close(self): + self.save() + + + +def load_object(file_path): + file_path = os.path.expanduser(file_path) + with open(file_path,'rb') as fh: + return pickle.load(fh) + +def save_object(object,file_path): + file_path = os.path.expanduser(file_path) + with open(file_path,'wb') as fh: + pickle.dump(object,fh,-1) + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + settings = Persistent_Dict("~/Desktop/test") + settings['f'] = "this is a test" + settings['list'] = ["list 1","list2"] + settings.close() + + save_object("string",'test') + print load_object('test') \ No newline at end of file diff --git a/pupil_src/shared_modules/gl_utils/utils.py b/pupil_src/shared_modules/gl_utils/utils.py index 3765d4026e..b2af7db7d6 100644 --- a/pupil_src/shared_modules/gl_utils/utils.py +++ b/pupil_src/shared_modules/gl_utils/utils.py @@ -85,7 +85,8 @@ def adjust_gl_view(w,h,window): # glMatrixMode(GL_MODELVIEW) # glLoadIdentity() -def draw_gl_polyline((positions),color,type='Loop'): +def draw_gl_polyline((positions),color,type='Loop',thickness=1): + glLineWidth(thickness) glColor4f(*color) if type=='Loop': glBegin(GL_LINE_LOOP) @@ -99,7 +100,7 @@ def draw_gl_polyline((positions),color,type='Loop'): glVertex3f(x,y,0.0) glEnd() -def draw_gl_polyline_norm((positions),color,type='Loop'): +def draw_gl_polyline_norm((positions),color,type='Loop',thickness=1): glMatrixMode(GL_PROJECTION) glPushMatrix() @@ -108,7 +109,7 @@ def draw_gl_polyline_norm((positions),color,type='Loop'): glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() - draw_gl_polyline(positions,color,type) + draw_gl_polyline(positions,color,type,thickness) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index 4277a9a8d9..66706403c6 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -8,20 +8,16 @@ ----------------------------------------------------------------------------------~(*) ''' -import os +import sys, os,platform import cv2 import numpy as np -import shelve +from file_methods import Persistent_Dict from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture from methods import normalize,denormalize +from glfw import * import atb -import audio from ctypes import c_int,c_bool,create_string_buffer -from OpenGL.GL import * -from OpenGL.GLU import gluOrtho2D - -from glfw import * from plugin import Plugin #logging import logging @@ -31,16 +27,8 @@ from reference_surface import Reference_Surface from math import sqrt -# window calbacks -def on_resize(window,w, h): - active_window = glfwGetCurrentContext() - glfwMakeContextCurrent(window) - adjust_gl_view(w,h,window) - glfwMakeContextCurrent(active_window) - class Marker_Detector(Plugin): """docstring - """ def __init__(self,g_pool,atb_pos=(320,220)): super(Marker_Detector, self).__init__() @@ -51,21 +39,9 @@ def __init__(self,g_pool,atb_pos=(320,220)): self.markers = [] # all registered surfaces - if g_pool.app == 'capture': - self.surface_definitions = shelve.open(os.path.join(g_pool.user_dir,'surface_definitions'),protocol=2) - self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] - elif g_pool.app == 'player': - #in player we load from the rec_dir: but we have a couple options: - self.surface_definitions = shelve.open(os.path.join(g_pool.rec_dir,'surface_definitions'),protocol=2) - if self.load('offline_square_marker_surfaces',[]) != []: - logger.debug("Found ref surfaces defined or copied in previous session.") - self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)] - elif self.load('offline_square_marker_surfaces',[]) != []: - logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.") - self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] - else: - logger.debug("No surface defs found. Please define using GUI.") - self.surfaces = [] + self.surface_definitions = Persistent_Dict(os.path.join(g_pool.user_dir,'surface_definitions') ) + self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] + # edit surfaces self.surface_edit_mode = c_bool(0) self.edit_surfaces = [] @@ -82,34 +58,11 @@ def __init__(self,g_pool,atb_pos=(320,220)): self.img_shape = None - #multi monitor setup - self.window_should_open = False - self.window_should_close = False - self._window = None - self.fullscreen = c_bool(0) - self.monitor_idx = c_int(0) - monitor_handles = glfwGetMonitors() - self.monitor_names = [glfwGetMonitorName(m) for m in monitor_handles] - monitor_enum = atb.enum("Monitor",dict(((key,val) for val,key in enumerate(self.monitor_names)))) - #primary_monitor = glfwGetPrimaryMonitor() - atb_label = "marker detection" self._bar = atb.Bar(name =self.__class__.__name__, label=atb_label, - help="marker detection parameters", color=(50, 50, 50), alpha=100, - text='light', position=atb_pos,refresh=.3, size=(300, 100)) - self._bar.add_var("monitor",self.monitor_idx, vtype=monitor_enum,group="Window",) - self._bar.add_var("fullscreen", self.fullscreen,group="Window") - self._bar.add_button(" open Window ", self.do_open, key='m',group="Window") - self._bar.add_var("surface to show",self.show_surface_idx, step=1,min=0,group="Window") - self._bar.add_var('robust_detection',self.robust_detection,group="Detector") - self._bar.add_var("draw markers",self.draw_markers,group="Detector") - self._bar.add_button('close',self.unset_alive) - + help="marker detection parameters", color=(50, 150, 50), alpha=100, + text='light', position=atb_pos,refresh=.3, size=(300, 300)) - atb_pos = atb_pos[0],atb_pos[1]+110 - self._bar_markers = atb.Bar(name =self.__class__.__name__+'markers', label='registered surfaces', - help="list of registered ref surfaces", color=(50, 100, 50), alpha=100, - text='light', position=atb_pos,refresh=.3, size=(300, 120)) self.update_bar_markers() @@ -123,10 +76,6 @@ def save(self, var_name, var): self.surface_definitions[var_name] = var - def do_open(self): - if not self._window: - self.window_should_open = True - def on_click(self,pos,button,action): if self.surface_edit_mode.value: if self.edit_surfaces: @@ -147,98 +96,57 @@ def on_click(self,pos,button,action): def advance(self): pass - def open_window(self): - if not self._window: - if self.fullscreen.value: - monitor = glfwGetMonitors()[self.monitor_idx.value] - mode = glfwGetVideoMode(monitor) - height,width= mode[0],mode[1] - else: - monitor = None - height,width= 1280,720 - - self._window = glfwCreateWindow(height, width, "Reference Surface", monitor=monitor, share=glfwGetCurrentContext()) - if not self.fullscreen.value: - glfwSetWindowPos(self._window,200,0) - - on_resize(self._window,height,width) - - #Register callbacks - glfwSetWindowSizeCallback(self._window,on_resize) - glfwSetKeyCallback(self._window,self.on_key) - glfwSetWindowCloseCallback(self._window,self.on_close) - - # gl_state settings - active_window = glfwGetCurrentContext() - glfwMakeContextCurrent(self._window) - basic_gl_setup() - - # refresh speed settings - glfwSwapInterval(0) - - - glfwMakeContextCurrent(active_window) - - self.window_should_open = False - - - def on_key(self,window, key, scancode, action, mods): - if not atb.TwEventKeyboardGLFW(key,int(action == GLFW_PRESS)): - if action == GLFW_PRESS: - if key == GLFW_KEY_ESCAPE: - self.on_close() - - def on_close(self,window=None): - self.window_should_close = True - - def close_window(self): - if self._window: - glfwDestroyWindow(self._window) - self._window = None - self.window_should_close = False - def add_surface(self): self.surfaces.append(Reference_Surface()) self.update_bar_markers() def remove_surface(self,i): + self.surfaces[i].cleanup() del self.surfaces[i] self.update_bar_markers() - def update_bar_markers(self): - self._bar_markers.clear() - self._bar_markers.add_button(" add surface ", self.add_surface, key='a') - self._bar_markers.add_var(" edit mode ", self.surface_edit_mode ) - - for s,i in zip (self.surfaces,range(len(self.surfaces))): - self._bar_markers.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') - self._bar_markers.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) - self._bar_markers.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) + self._bar.clear() + self._bar.add_button('close',self.unset_alive) + self._bar.add_var('robust_detection',self.robust_detection) + self._bar.add_var("draw markers",self.draw_markers) + self._bar.add_button(" add surface ", self.add_surface, key='a') + self._bar.add_var(" edit mode ", self.surface_edit_mode ) + for s,i in zip(self.surfaces,range(len(self.surfaces)))[::-1]: + self._bar.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window') + self._bar.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') + self._bar.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) + self._bar.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) def update(self,frame,recent_pupil_positions,events): img = frame.img self.img_shape = frame.img.shape + if self.robust_detection.value: - self.markers = detect_markers_robust(img,grid_size = 5, - prev_markers=self.markers, - min_marker_perimeter=self.min_marker_perimeter, - aperture=self.aperture.value, - visualize=0, - true_detect_every_frame=3) + self.markers = detect_markers_robust(img, + grid_size = 5, + prev_markers=self.markers, + min_marker_perimeter=self.min_marker_perimeter, + aperture=self.aperture.value, + visualize=0, + true_detect_every_frame=3) else: - self.markers = detect_markers_simple(img,grid_size = 5,min_marker_perimeter=self.min_marker_perimeter,aperture=self.aperture.value,visualize=0) - - if self.draw_markers.value: - draw_markers(img,self.markers) - - # print self.markers + self.markers = detect_markers_simple(img, + grid_size = 5, + min_marker_perimeter=self.min_marker_perimeter, + aperture=self.aperture.value, + visualize=0) + # locate surfaces for s in self.surfaces: s.locate(self.markers) if s.detected: - events.append({'type':'marker_ref_surface','name':s.name,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) + events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) + + if self.draw_markers.value: + draw_markers(img,self.markers) + # edit surfaces by user if self.surface_edit_mode: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) @@ -252,106 +160,51 @@ def update(self,frame,recent_pupil_positions,events): s.move_vertex(v_idx,new_pos) #map recent gaze onto detected surfaces used for pupil server - for p in recent_pupil_positions: - if p['norm_pupil'] is not None: - for s in self.surfaces: - if s.detected: - p['realtime gaze on '+s.name] = tuple(s.img_to_ref_surface(np.array(p['norm_gaze']))) - - - if self._window: - # save a local copy for when we display gaze for debugging on ref surface - self.recent_pupil_positions = recent_pupil_positions + for s in self.surfaces: + if s.detected: + s.gaze_on_srf = [] + for p in recent_pupil_positions: + if p['norm_pupil'] is not None: + gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_gaze']))) + p['realtime gaze on '+s.name] = gp_on_s + s.gaze_on_srf.append(gp_on_s) - if self.window_should_close: - self.close_window() + #allow surfaces to open/close windows + for s in self.surfaces: + if s.window_should_close: + s.close_window() + if s.window_should_open: + s.open_window() - if self.window_should_open: - self.open_window() def gl_display(self): """ Display marker and surface info inside world screen """ - for m in self.markers: hat = np.array([[[0,0],[0,1],[.5,1.3],[1,1],[1,0],[0,0]]],dtype=np.float32) hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m)) draw_gl_polyline(hat.reshape((6,2)),(0.1,1.,1.,.5)) - for s in self.surfaces: + for s in self.surfaces: s.gl_draw_frame() + s.gl_display_in_window(self.g_pool.image_tex) if self.surface_edit_mode.value: for s in self.surfaces: s.gl_draw_corners() - if self._window and self.surfaces: - try: - s = self.surfaces[self.show_surface_idx.value] - except IndexError: - s = None - if s and s.detected: - self.gl_display_in_window(s) - - - - def gl_display_in_window(self,surface): - """ - here we map a selected surface onto a seperate window. - """ - active_window = glfwGetCurrentContext() - glfwMakeContextCurrent(self._window) - clear_gl_screen() - - # cv uses 3x3 gl uses 4x4 tranformation matricies - m = cvmat_to_glmat(surface.m_from_screen) - - glMatrixMode(GL_PROJECTION) - glPushMatrix() - glLoadIdentity() - gluOrtho2D(0, 1, 0, 1) # gl coord convention - - glMatrixMode(GL_MODELVIEW) - glPushMatrix() - #apply m to our quad - this will stretch the quad such that the ref suface will span the window extends - glLoadMatrixf(m) - - draw_named_texture(self.g_pool.image_tex) - - glMatrixMode(GL_PROJECTION) - glPopMatrix() - glMatrixMode(GL_MODELVIEW) - glPopMatrix() - - - #now lets get recent pupil positions on this surface: - try: - gaze_on_surface = [p['realtime gaze on '+surface.name] for p in self.recent_pupil_positions] - except KeyError: - gaze_on_surface = [] - draw_gl_points_norm(gaze_on_surface,color=(0.,8.,.5,.8), size=80) - - - glfwSwapBuffers(self._window) - glfwMakeContextCurrent(active_window) - - def cleanup(self): """ called when the plugin gets terminated. This happends either voluntary or forced. if you have an atb bar or glfw window destroy it here. """ - if self.g_pool.app == 'capture': - self.save("realtime_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) - elif self.g_pool.app == 'player': - self.save("offline_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) + self.save("realtime_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) + self.surface_definitions.close() - if self._window: - self.close_window() + for s in self.surfaces: + s.close_window() self._bar.destroy() - self._bar_markers.destroy() - diff --git a/pupil_src/shared_modules/marker_detector_cacher.py b/pupil_src/shared_modules/marker_detector_cacher.py new file mode 100644 index 0000000000..ea3a9bf60b --- /dev/null +++ b/pupil_src/shared_modules/marker_detector_cacher.py @@ -0,0 +1,115 @@ +''' +(*)~---------------------------------------------------------------------------------- + Pupil - eye tracking platform + Copyright (C) 2012-2014 Pupil Labs + + Distributed under the terms of the CC BY-NC-SA License. + License details are in the file license.txt, distributed as part of this software. +----------------------------------------------------------------------------------~(*) +''' + + + + +def fill_cache(visited_list,video_file_path,q,seek_idx,run): + ''' + this function is part of marker_detector it is run as a seperate process. + it must be kept in a seperate file for namespace sanatisation + ''' + import os + import logging + logger = logging.getLogger(__name__+' with pid: '+str(os.getpid()) ) + logger.debug('Started cacher process for Marker Detector') + import cv2 + from uvc_capture import autoCreateCapture, EndofVideoFileError,FileSeekError + from square_marker_detect import detect_markers_robust,detect_markers_simple + min_marker_perimeter = 80 + aperture = 9 + markers = [] + + cap = autoCreateCapture(video_file_path) + + def next_unvisited_idx(frame_idx): + try: + visited = visited_list[frame_idx] + except IndexError: + visited = True # trigger search + + if not visited: + next_unvisited = frame_idx + else: + # find next unvisited site in the future + try: + next_unvisited = visited_list.index(False,frame_idx) + except ValueError: + # any thing in the past? + try: + next_unvisited = visited_list.index(False,0,frame_idx) + except ValueError: + #no unvisited sites left. Done! + logger.debug("Caching completed.") + next_unvisited = None + return next_unvisited + + def handle_frame(next): + if next != cap.get_frame_index(): + #we need to seek: + logger.debug("Seeking to Frame %s" %next) + try: + cap.seek_to_frame(next) + except FileSeekError: + #could not seek to requested position + logger.warning("Could not evaluate frame: %s."%next) + visited_list[next] = True # this frame is now visited. + q.put((next,[])) # we cannot look at the frame, report no detection + return + #seeking invalidates prev markers for the detector + markers[:] = [] + + try: + frame = cap.get_frame() + except EndofVideoFileError: + logger.debug("Video File's last frame(s) not accesible") + #could not read frame + logger.warning("Could not evaluate frame: %s."%next) + visited_list[next] = True # this frame is now visited. + q.put((next,[])) # we cannot look at the frame, report no detection + return + + markers[:] = detect_markers_robust(frame.img, + grid_size = 5, + prev_markers=markers, + min_marker_perimeter=min_marker_perimeter, + aperture=aperture, + visualize=0, + true_detect_every_frame=1) + + # markers[:] = detect_markers_simple(frame.img, + # grid_size = 5, + # min_marker_perimeter=min_marker_perimeter, + # aperture=aperture, + # visualize=0) + visited_list[frame.index] = True + q.put((frame.index,markers[:])) #object passed will only be pickeld when collected from other process! need to make a copy ot avoid overwrite!!! + + while run.value: + next = cap.get_frame_index() + if seek_idx.value != -1: + next = seek_idx.value + seek_idx.value = -1 + logger.debug("User required seek. Marker caching at Frame: %s"%next) + + + #check the visited list + next = next_unvisited_idx(next) + if next == None: + #we are done here: + break + else: + handle_frame(next) + + + logger.debug("Closing Cacher Process") + cap.close() + q.close() + return \ No newline at end of file diff --git a/pupil_src/shared_modules/methods.py b/pupil_src/shared_modules/methods.py index 755b4986ab..81079285f9 100644 --- a/pupil_src/shared_modules/methods.py +++ b/pupil_src/shared_modules/methods.py @@ -16,7 +16,6 @@ import cv2 import logging logger = logging.getLogger(__name__) - class Temp(object): """Temp class to make objects""" def __init__(self): @@ -89,7 +88,6 @@ def get(self): return self.lX,self.lY,self.uX,self.uY,self.array_shape - def bin_thresholding(image, image_lower=0, image_upper=256): binary_img = cv2.inRange(image, np.asarray(image_lower), np.asarray(image_upper)) diff --git a/pupil_src/shared_modules/offline_marker_detector.py b/pupil_src/shared_modules/offline_marker_detector.py new file mode 100644 index 0000000000..4185e89555 --- /dev/null +++ b/pupil_src/shared_modules/offline_marker_detector.py @@ -0,0 +1,415 @@ +''' +(*)~---------------------------------------------------------------------------------- + Pupil - eye tracking platform + Copyright (C) 2012-2014 Pupil Labs + + Distributed under the terms of the CC BY-NC-SA License. + License details are in the file license.txt, distributed as part of this software. +----------------------------------------------------------------------------------~(*) +''' + +import sys, os,platform +import cv2 +import numpy as np +import csv + + +if platform.system() == 'Darwin': + from billiard import Process,Queue,forking_enable + from billiard.sharedctypes import Value +else: + from multiprocessing import Process, Pipe, Event, Queue + forking_enable = lambda x: x #dummy fn + from multiprocessing.sharedctypes import Value + + +from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture +from OpenGL.GL import * +from OpenGL.GLU import gluOrtho2D +from methods import normalize,denormalize +from file_methods import Persistent_Dict,save_object +from cache_list import Cache_List +from glfw import * +import atb +from ctypes import c_int,c_bool,c_float,create_string_buffer + +from plugin import Plugin +#logging +import logging +logger = logging.getLogger(__name__) + +from square_marker_detect import detect_markers_robust,detect_markers_simple, draw_markers,m_marker_to_screen +from offline_reference_surface import Offline_Reference_Surface +from math import sqrt + + +class Offline_Marker_Detector(Plugin): + """ + Special version of marker detector for use with videofile source. + It uses a seperate process to search all frames in the world.avi file for markers. + - self.cache is a list containing marker positions for each frame. + - self.surfaces[i].cache is a list containing surface positions for each frame + Both caches are build up over time. The marker cache is also session persistent. + See marker_tracker.py for more info on this marker tracker. + """ + + def __init__(self,g_pool,gui_settings={'pos':(220,200),'size':(300,300),'iconified':False}): + super(Offline_Marker_Detector, self).__init__() + self.g_pool = g_pool + self.gui_settings = gui_settings + self.order = .2 + + + # all markers that are detected in the most recent frame + self.markers = [] + # all registered surfaces + + if g_pool.app == 'capture': + raise Exception('For Player only.') + #in player we load from the rec_dir: but we have a couple options: + self.surface_definitions = Persistent_Dict(os.path.join(g_pool.rec_dir,'surface_definitions')) + if self.load('offline_square_marker_surfaces',[]) != []: + logger.debug("Found ref surfaces defined or copied in previous session.") + self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)] + elif self.load('realtime_square_marker_surfaces',[]) != []: + logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.") + self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] + else: + logger.debug("No surface defs found. Please define using GUI.") + self.surfaces = [] + + # edit surfaces + self.surface_edit_mode = c_bool(0) + self.edit_surfaces = [] + + #detector vars + self.robust_detection = c_bool(1) + self.aperture = c_int(11) + self.min_marker_perimeter = 80 + + #check if marker cache is available from last session + self.persistent_cache = Persistent_Dict(os.path.join(g_pool.rec_dir,'square_marker_cache')) + self.cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps])) + logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.cache)-self.cache.count(False),len(self.cache)) ) + self.init_marker_cacher() + + #debug vars + self.draw_markers = c_bool(0) + self.show_surface_idx = c_int(0) + self.recent_pupil_positions = [] + + self.img_shape = None + self.img = None + + + def init_gui(self): + import atb + pos = self.gui_settings['pos'] + atb_label = "Marker Detector" + self._bar = atb.Bar(name =self.__class__.__name__+str(id(self)), label=atb_label, + help="circle", color=(50, 150, 50), alpha=50, + text='light', position=pos,refresh=.1, size=self.gui_settings['size']) + self._bar.iconified = self.gui_settings['iconified'] + self.update_bar_markers() + + #set up bar display padding + self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext())) + + + def unset_alive(self): + self.alive = False + + def load(self, var_name, default): + return self.surface_definitions.get(var_name,default) + def save(self, var_name, var): + self.surface_definitions[var_name] = var + + def on_window_resize(self,window,w,h): + self.win_size = w,h + + + def on_click(self,pos,button,action): + if self.surface_edit_mode.value: + if self.edit_surfaces: + if action == GLFW_RELEASE: + self.edit_surfaces = [] + # no surfaces verts in edit mode, lets see if the curser is close to one: + else: + if action == GLFW_PRESS: + surf_verts = ((0.,0.),(1.,0.),(1.,1.),(0.,1.)) + x,y = pos + for s in self.surfaces: + if s.detected: + for (vx,vy),i in zip(s.ref_surface_to_img(np.array(surf_verts)),range(4)): + vx,vy = denormalize((vx,vy),(self.img_shape[1],self.img_shape[0]),flip_y=True) + if sqrt((x-vx)**2 + (y-vy)**2) <15: #img pixels + self.edit_surfaces.append((s,i)) + + def advance(self): + pass + + def add_surface(self): + self.surfaces.append(Offline_Reference_Surface(self.g_pool,gaze_positions_by_frame=self.g_pool.positions_by_frame)) + self.update_bar_markers() + + def remove_surface(self,i): + self.surfaces[i].cleanup() + del self.surfaces[i] + self.update_bar_markers() + + def update_bar_markers(self): + self._bar.clear() + self._bar.add_button('close',self.unset_alive) + self._bar.add_var("draw markers",self.draw_markers) + self._bar.add_button(" add surface ", self.add_surface, key='a') + self._bar.add_var(" edit mode ", self.surface_edit_mode ) + for s,i in zip(self.surfaces,range(len(self.surfaces)))[::-1]: + self._bar.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') + self._bar.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) + self._bar.add_var("%s_x_scale"%i,vtype=c_float, getter=s.atb_get_scale_x, min=1,setter=s.atb_set_scale_x,group=str(i),label='real width', help='this scale factor is used to adjust the coordinate space for your needs (think photo pixels or mm or whatever)' ) + self._bar.add_var("%s_y_scale"%i,vtype=c_float, getter=s.atb_get_scale_y,min=1,setter=s.atb_set_scale_y,group=str(i),label='real height',help='defining x and y scale factor you atumatically set the correct aspect ratio.' ) + self._bar.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window') + self._bar.add_button("%s_hm"%i, s.generate_heatmap, label='generate_heatmap',group=str(i)) + self._bar.add_button("%s_export"%i, self.save_surface_positions_to_file,data=i, label='export surface data',group=str(i)) + self._bar.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) + + def update(self,frame,recent_pupil_positions,events): + self.img = frame.img + self.img_shape = frame.img.shape + self.update_marker_cache() + self.markers = self.cache[frame.index] + if self.markers == False: + # locate markers because precacher has not anayzed this frame yet. Most likely a seek event + self.markers = [] + self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here analyzed + + # locate surfaces + for s in self.surfaces: + if not s.locate_from_cache(frame.index): + s.locate(self.markers) + if s.detected: + events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf}) + + if self.draw_markers.value: + draw_markers(frame.img,self.markers) + + # edit surfaces by user + if self.surface_edit_mode: + window = glfwGetCurrentContext() + pos = glfwGetCursorPos(window) + pos = normalize(pos,glfwGetWindowSize(window)) + pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels + + for s,v_idx in self.edit_surfaces: + if s.detected: + pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True) + new_pos = s.img_to_ref_surface(np.array(pos)) + s.move_vertex(v_idx,new_pos) + s.cache = None + else: + # update srf with no or invald cache: + for s in self.surfaces: + if s.cache == None: + s.init_cache(self.cache) + + + #allow surfaces to open/close windows + for s in self.surfaces: + if s.window_should_close: + s.close_window() + if s.window_should_open: + s.open_window() + + + + + def init_marker_cacher(self): + forking_enable(0) #for MacOs only + from marker_detector_cacher import fill_cache + visited_list = [False if x == False else True for x in self.cache] + video_file_path = os.path.join(self.g_pool.rec_dir,'world.avi') + self.cache_queue = Queue() + self.cacher_seek_idx = Value(c_int,0) + self.cacher_run = Value(c_bool,True) + self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.cache_queue,self.cacher_seek_idx,self.cacher_run)) + self.cacher.start() + + def update_marker_cache(self): + while not self.cache_queue.empty(): + idx,c_m = self.cache_queue.get() + self.cache.update(idx,c_m) + for s in self.surfaces: + s.update_cache(self.cache,idx=idx) + + def seek_marker_cacher(self,idx): + self.cacher_seek_idx.value = idx + + def close_marker_cacher(self): + self.update_marker_cache() + self.cacher_run.value = False + self.cacher.join() + + def gl_display(self): + """ + Display marker and surface info inside world screen + """ + self.gl_display_cache_bars() + + for m in self.markers: + hat = np.array([[[0,0],[0,1],[.5,1.3],[1,1],[1,0],[0,0]]],dtype=np.float32) + hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m)) + draw_gl_polyline(hat.reshape((6,2)),(0.1,1.,1.,.5)) + + for s in self.surfaces: + s.gl_draw_frame() + s.gl_display_in_window(self.g_pool.image_tex) + + if self.surface_edit_mode.value: + for s in self.surfaces: + s.gl_draw_corners() + + + def gl_display_cache_bars(self): + """ + """ + padding = 20. + + # Lines for areas that have be cached + cached_ranges = [] + for r in self.cache.visited_ranges: # [[0,1],[3,4]] + cached_ranges += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)] + + # Lines where surfaces have been found in video + cached_surfaces = [] + for s in self.surfaces: + found_at = [] + if s.cache is not None: + for r in s.cache.positive_ranges: # [[0,1],[3,4]] + found_at += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)] + cached_surfaces.append(found_at) + + glMatrixMode(GL_PROJECTION) + glPushMatrix() + glLoadIdentity() + width,height = self.win_size + h_pad = padding * (self.cache.length-2)/float(width) + v_pad = padding* 1./(height-2) + gluOrtho2D(-h_pad, (self.cache.length-1)+h_pad, -v_pad, 1+v_pad) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) + + + glMatrixMode(GL_MODELVIEW) + glPushMatrix() + glLoadIdentity() + + color = (8.,.6,.2,8.) + draw_gl_polyline(cached_ranges,color=color,type='Lines',thickness=4) + + color = (0.,.7,.3,8.) + + for s in cached_surfaces: + glTranslatef(0,.02,0) + draw_gl_polyline(s,color=color,type='Lines',thickness=2) + + glMatrixMode(GL_PROJECTION) + glPopMatrix() + glMatrixMode(GL_MODELVIEW) + glPopMatrix() + + + def save_surface_positions_to_file(self,i): + s = self.surfaces[i] + + in_mark = self.g_pool.trim_marks.in_mark + out_mark = self.g_pool.trim_marks.out_mark + + if s.cache == None: + logger.warning("The surface is not cached. Please wait for the cacher to collect data.") + return + + srf_dir = os.path.join(self.g_pool.rec_dir,'surface_data'+'_'+s.name.replace('/','')+'_'+s.uid) + logger.info("exporting surface gaze data to %s"%srf_dir) + if os.path.isdir(srf_dir): + logger.info("Will overwrite previous export for this referece surface") + else: + try: + os.mkdir(srf_dir) + except: + logger.warning("Could name make export dir %s"%srf_dir) + return + + #save surface_positions as pickle file + save_object(s.cache.to_list(),os.path.join(srf_dir,'srf_positons')) + + #save surface_positions as csv + with open(os.path.join(srf_dir,'srf_positons.csv'),'wb') as csvfile: + csw_writer =csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL) + csw_writer.writerow(('frame_idx','timestamp','m_to_screen','m_from_screen','detected_markers')) + for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache): + if in_mark <= idx <= out_mark: + if ref_srf_data is not None and ref_srf_data is not False: + csw_writer.writerow( (idx,ts,ref_srf_data['m_to_screen'],ref_srf_data['m_from_screen'],ref_srf_data['detected_markers']) ) + + + #save gaze on srf as csv. + with open(os.path.join(srf_dir,'gaze_positions_on_surface.csv'),'wb') as csvfile: + csw_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL) + csw_writer.writerow(('world_frame_idx','world_timestamp','eye_timestamp','x_norm','y_norm','x_scaled','y_scaled','on_srf')) + for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache): + if in_mark <= idx <= out_mark: + if ref_srf_data is not None and ref_srf_data is not False: + for gp in ref_srf_data['gaze_on_srf']: + gp_x,gp_y = gp['norm_gaze_on_srf'] + on_srf = (0 <= gp_x <= 1) and (0 <= gp_y <= 1) + csw_writer.writerow( (idx,ts,gp['timestamp'],gp_x,gp_y,gp_x*s.scale_factor[0],gp_x*s.scale_factor[1],on_srf) ) + + logger.info("Saved surface positon data and gaze on surface data for '%s' with uid:'%s'"%(s.name,s.uid)) + + if s.heatmap is not None: + logger.info("Saved Heatmap as .png file.") + cv2.imwrite(os.path.join(srf_dir,'heatmap.png'),s.heatmap) + + if s.detected and self.img is not None: + #let save out the current surface image found in video + + #here we get the verts of the surface quad in norm_coords + mapped_space_one = np.array(((0,0),(1,0),(1,1),(0,1)),dtype=np.float32).reshape(-1,1,2) + screen_space = cv2.perspectiveTransform(mapped_space_one,s.m_to_screen).reshape(-1,2) + #now we convert to image pixel coods + screen_space[:,1] = 1-screen_space[:,1] + screen_space[:,1] *= self.img.shape[0] + screen_space[:,0] *= self.img.shape[1] + s_0,s_1 = s.scale_factor + #no we need to flip vertically again by setting the mapped_space verts accordingly. + mapped_space_scaled = np.array(((0,s_1),(s_0,s_1),(s_0,0),(0,0)),dtype=np.float32) + M = cv2.getPerspectiveTransform(screen_space,mapped_space_scaled) + #here we do the actual perspactive transform of the image. + srf_in_video = cv2.warpPerspective(self.img,M, (int(s.scale_factor[0]),int(s.scale_factor[1])) ) + cv2.imwrite(os.path.join(srf_dir,'surface.png'),srf_in_video) + logger.info("Saved current image as .png file.") + + + + def get_init_dict(self): + d = {} + if hasattr(self,'_bar'): + gui_settings = {'pos':self._bar.position,'size':self._bar.size,'iconified':self._bar.iconified} + d['gui_settings'] = gui_settings + + return d + + def cleanup(self): + """ called when the plugin gets terminated. + This happends either voluntary or forced. + if you have an atb bar or glfw window destroy it here. + """ + + self.save("offline_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) + self.close_marker_cacher() + self.persistent_cache["marker_cache"] = self.cache.to_list() + self.persistent_cache.close() + + self.surface_definitions.close() + + for s in self.surfaces: + s.close_window() + self._bar.destroy() diff --git a/pupil_src/shared_modules/offline_reference_surface.py b/pupil_src/shared_modules/offline_reference_surface.py new file mode 100644 index 0000000000..c951dbd902 --- /dev/null +++ b/pupil_src/shared_modules/offline_reference_surface.py @@ -0,0 +1,233 @@ +''' +(*)~---------------------------------------------------------------------------------- + Pupil - eye tracking platform + Copyright (C) 2012-2014 Pupil Labs + + Distributed under the terms of the CC BY-NC-SA License. + License details are in the file license.txt, distributed as part of this software. +----------------------------------------------------------------------------------~(*) +''' + +import numpy as np +import cv2 +from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture,create_named_texture +from glfw import * +from OpenGL.GL import * +from OpenGL.GLU import gluOrtho2D + +from methods import GetAnglesPolyline,normalize +from cache_list import Cache_List + +#ctypes import for atb_vars: +from ctypes import c_int,c_bool,create_string_buffer +from time import time + +import logging +logger = logging.getLogger(__name__) + +from reference_surface import Reference_Surface + +class Offline_Reference_Surface(Reference_Surface): + """docstring for Offline_Reference_Surface""" + def __init__(self,g_pool,name="unnamed",saved_definition=None, gaze_positions_by_frame = None): + super(Offline_Reference_Surface, self).__init__(name,saved_definition) + self.g_pool = g_pool + self.gaze_positions_by_frame = gaze_positions_by_frame + self.cache = None + self.gaze_on_srf = [] # points on surface for realtime feedback display + + self.heatmap_detail = .2 + self.heatmap = None + self.heatmap_texture = None + #cache fn for offline marker + def locate_from_cache(self,frame_idx): + if self.cache == None: + #no cache available cannot update from cache + return False + cache_result = self.cache[frame_idx] + if cache_result == False: + #cached data not avaible for this frame + return False + elif cache_result == None: + #cached data shows surface not found: + self.detected = False + self.m_from_screen = None + self.m_to_screen = None + self.gaze_on_srf = [] + self.detected_markers = 0 + return True + else: + self.detected = True + self.m_from_screen = cache_result['m_from_screen'] + self.m_to_screen = cache_result['m_to_screen'] + self.detected_markers = cache_result['detected_markers'] + self.gaze_on_srf = cache_result['gaze_on_srf'] + return True + raise Exception("Invalid cache entry. Please report Bug.") + + + def update_cache(self,marker_cache,idx=None): + ''' + compute surface m's and gaze points from cached marker data + entries are: + - False: when marker cache entry was False (not yet searched) + - None: when surface was not found + - {'m_to_screen':,'m_from_screen':,'detected_markers':,gaze_on_srf} + ''' + + # iterations = 0 + + if self.cache == None: + pass + # self.init_cache(marker_cache) + elif idx != None: + #update single data pt + self.cache.update(idx,self.answer_caching_request(marker_cache,idx)) + else: + # update where markercache is not False but surface cache is still false + # this happens when the markercache was incomplete when this fn was run before + for i in range(len(marker_cache)): + if self.cache[i] == False and marker_cache[i] != False: + self.cache.update(i,self.answer_caching_request(marker_cache,i)) + # iterations +=1 + # return iterations + + + + def init_cache(self,marker_cache): + if self.defined: + logger.debug("Full update of surface '%s' positons cache"%self.name) + self.cache = Cache_List([self.answer_caching_request(marker_cache,i) for i in xrange(len(marker_cache))],positive_eval_fn=lambda x: (x!=False) and (x!=None)) + + + def answer_caching_request(self,marker_cache,frame_index): + visible_markers = marker_cache[frame_index] + # cache point had not been visited + if visible_markers == False: + return False + # cache point had been visited + marker_by_id = dict( [ (m['id'],m) for m in visible_markers] ) + visible_ids = set(marker_by_id.keys()) + requested_ids = set(self.markers.keys()) + overlap = visible_ids & requested_ids + detected_markers = len(overlap) + if len(overlap)>=min(2,len(requested_ids)): + yx = np.array( [marker_by_id[i]['verts_norm'] for i in overlap] ) + uv = np.array( [self.markers[i].uv_coords for i in overlap] ) + yx.shape=(-1,1,2) + uv.shape=(-1,1,2) + m_to_screen,mask = cv2.findHomography(uv,yx) + m_from_screen,mask = cv2.findHomography(yx,uv) + + return {'m_to_screen':m_to_screen, + 'm_from_screen':m_from_screen, + 'detected_markers':len(overlap), + 'gaze_on_srf':self.gaze_on_srf_by_frame_idx(frame_index,m_from_screen)} + else: + #surface not found + return None + + + def gaze_on_srf_by_frame_idx(self,frame_index,m_from_screen): + gaze_positions = self.gaze_positions_by_frame[frame_index] + gaze_on_src = [] + for g_p in gaze_positions: + gaze_points = np.array([g_p['norm_gaze']]).reshape(1,1,2) + gaze_points_on_srf = cv2.perspectiveTransform(gaze_points , m_from_screen ) + gaze_points_on_srf.shape = (2) + gaze_on_src.append( {'norm_gaze_on_srf':(gaze_points_on_srf[0],gaze_points_on_srf[1]),'timestamp':g_p['timestamp'] } ) + return gaze_on_src + + + + #### fns to draw surface in seperate window + def gl_display_in_window(self,world_tex_id): + """ + here we map a selected surface onto a seperate window. + """ + if self._window and self.detected: + active_window = glfwGetCurrentContext() + glfwMakeContextCurrent(self._window) + clear_gl_screen() + + # cv uses 3x3 gl uses 4x4 tranformation matricies + m = cvmat_to_glmat(self.m_from_screen) + + glMatrixMode(GL_PROJECTION) + glPushMatrix() + glLoadIdentity() + gluOrtho2D(0, 1, 0, 1) # gl coord convention + + glMatrixMode(GL_MODELVIEW) + glPushMatrix() + #apply m to our quad - this will stretch the quad such that the ref suface will span the window extends + glLoadMatrixf(m) + + draw_named_texture(world_tex_id) + + glMatrixMode(GL_PROJECTION) + glPopMatrix() + glMatrixMode(GL_MODELVIEW) + glPopMatrix() + + + if self.heatmap_texture: + draw_named_texture(self.heatmap_texture) + + # now lets get recent pupil positions on this surface: + for gp in self.gaze_on_srf: + draw_gl_points_norm([gp['norm_gaze_on_srf']],color=(0.,8.,.5,.8), size=80) + + glfwSwapBuffers(self._window) + glfwMakeContextCurrent(active_window) + + + def generate_heatmap(self): + + in_mark = self.g_pool.trim_marks.in_mark + out_mark = self.g_pool.trim_marks.out_mark + + x,y = self.scale_factor + x = max(1,int(x)) + y = max(1,int(y)) + + filter_size = (int(self.heatmap_detail * x)/2)*2 +1 + std_dev = filter_size /6. + self.heatmap = np.ones((y,x,4),dtype=np.uint8) + all_gaze = [] + for idx,c_e in enumerate(self.cache): + if in_mark <= idx <= out_mark: + if c_e: + for gp in c_e['gaze_on_srf']: + all_gaze.append(gp['norm_gaze_on_srf']) + + if not all_gaze: + logger.warning("No gaze data on surface for heatmap found.") + all_gaze.append((-1.,-1.)) + all_gaze = np.array(all_gaze) + all_gaze *= self.scale_factor + hist,xedge,yedge = np.histogram2d(all_gaze[:,0], all_gaze[:,1], + bins=[x,y], + range=[[0, self.scale_factor[0]], [0,self.scale_factor[1]]], + normed=False, + weights=None) + + + hist = np.rot90(hist) + + #smoothing.. + hist = cv2.GaussianBlur(hist, (filter_size,filter_size),std_dev) + maxval = np.amax(hist) + if maxval: + scale = 255./maxval + else: + scale = 0 + + hist = np.uint8( hist*(scale) ) + + #colormapping + c_map = cv2.applyColorMap(hist, cv2.COLORMAP_JET) + + self.heatmap[:,:,:3] = c_map + self.heatmap[:,:,3] = 125 + self.heatmap_texture = create_named_texture(self.heatmap) diff --git a/pupil_src/shared_modules/plugin.py b/pupil_src/shared_modules/plugin.py index 9ea003f4f5..4bd067e9f2 100644 --- a/pupil_src/shared_modules/plugin.py +++ b/pupil_src/shared_modules/plugin.py @@ -53,6 +53,13 @@ def on_click(self,pos,button,action): """ pass + def on_window_resize(self,window,w,h): + ''' + gets called when user resizes window. + window is the glfw window handle of the resized window. + ''' + pass + def update(self,frame,recent_pupil_positions,events): """ gets called once every frame @@ -61,6 +68,8 @@ def update(self,frame,recent_pupil_positions,events): """ pass + + def gl_display(self): """ gets called once every frame when its time to draw onto the gl canvas. diff --git a/pupil_src/shared_modules/reference_surface.py b/pupil_src/shared_modules/reference_surface.py index ae7fe42a26..c3d95ddbd1 100644 --- a/pupil_src/shared_modules/reference_surface.py +++ b/pupil_src/shared_modules/reference_surface.py @@ -10,12 +10,19 @@ import numpy as np import cv2 -from gl_utils import draw_gl_polyline_norm,draw_gl_points_norm,draw_gl_point_norm +from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture,make_coord_system_norm_based +from glfw import * +from OpenGL.GL import * +from OpenGL.GLU import gluOrtho2D + from methods import GetAnglesPolyline,normalize #ctypes import for atb_vars: -from ctypes import create_string_buffer +from ctypes import c_int,c_bool,create_string_buffer +from time import time +import logging +logger = logging.getLogger(__name__) def m_verts_to_screen(verts): #verts need to be sorted counterclockwise stating at bottom left @@ -65,10 +72,31 @@ def __init__(self,name="unnamed",saved_definition=None): self.detected = False self.m_to_screen = None self.m_from_screen = None + self.uid = str(time()) + self.scale_factor = [1.,1.] + if saved_definition is not None: self.load_from_dict(saved_definition) + ###window and gui vars + self._window = None + self.fullscreen = False + self.window_should_open = False + self.window_should_close = False + + #multi monitor setup + self.window_should_open = False + self.window_should_close = False + self._window = None + self.fullscreen = c_bool(0) + self.monitor_idx = c_int(0) + monitor_handles = glfwGetMonitors() + self.monitor_names = [glfwGetMonitorName(m) for m in monitor_handles] + # monitor_enum = atb.enum("Monitor",dict(((key,val) for val,key in enumerate(self.monitor_names)))) + #primary_monitor = glfwGetPrimaryMonitor() + + self.gaze_on_srf = [] # points on surface for realtime feedback display def save_to_dict(self): @@ -76,7 +104,7 @@ def save_to_dict(self): save all markers and name of this surface to a dict. """ markers = dict([(m_id,m.uv_coords) for m_id,m in self.markers.iteritems()]) - return {'name':self.name,'markers':markers} + return {'name':self.name,'uid':self.uid,'markers':markers,'scale_factor':self.scale_factor} def load_from_dict(self,d): @@ -84,6 +112,9 @@ def load_from_dict(self,d): load all markers of this surface to a dict. """ self.name = d['name'] + self.uid = d['uid'] + self.scale_factor = d['scale_factor'] + marker_dict = d['markers'] for m_id,uv_coords in marker_dict.iteritems(): self.markers[m_id] = Support_Marker(m_id) @@ -197,7 +228,6 @@ def locate(self, visible_markers): self.m_to_screen = None - def img_to_ref_surface(self,pos): if self.m_from_screen is not None: #convenience lines to allow 'simple' vectors (x,y) to be used @@ -221,6 +251,7 @@ def ref_surface_to_img(self,pos): return None + def move_vertex(self,vert_idx,new_pos): """ this fn is used to manipulate the surface boundary (coordinate system) @@ -236,6 +267,7 @@ def move_vertex(self,vert_idx,new_pos): for m in self.markers.values(): m.uv_coords = cv2.perspectiveTransform(m.uv_coords,transform) + def atb_marker_status(self): return create_string_buffer("%s / %s" %(self.detected_markers,len(self.markers)),512) @@ -245,6 +277,16 @@ def atb_get_name(self): def atb_set_name(self,name): self.name = name.value + def atb_set_scale_x(self,val): + self.scale_factor[0]=val + def atb_set_scale_y(self,val): + self.scale_factor[1]=val + def atb_get_scale_x(self): + return self.scale_factor[0] + def atb_get_scale_y(self): + return self.scale_factor[1] + + def gl_draw_frame(self): """ draw surface and markers @@ -269,6 +311,116 @@ def gl_draw_corners(self): draw_gl_points_norm(frame.reshape((4,2)),15,(1.0,0.2,0.6,.5)) + + #### fns to draw surface in separate window + def gl_display_in_window(self,world_tex_id): + """ + here we map a selected surface onto a seperate window. + """ + if self._window and self.detected: + active_window = glfwGetCurrentContext() + glfwMakeContextCurrent(self._window) + clear_gl_screen() + + # cv uses 3x3 gl uses 4x4 tranformation matricies + m = cvmat_to_glmat(self.m_from_screen) + + glMatrixMode(GL_PROJECTION) + glPushMatrix() + glLoadIdentity() + gluOrtho2D(0, 1, 0, 1) # gl coord convention + + glMatrixMode(GL_MODELVIEW) + glPushMatrix() + #apply m to our quad - this will stretch the quad such that the ref suface will span the window extends + glLoadMatrixf(m) + + draw_named_texture(world_tex_id) + + glMatrixMode(GL_PROJECTION) + glPopMatrix() + glMatrixMode(GL_MODELVIEW) + glPopMatrix() + + # now lets get recent pupil positions on this surface: + draw_gl_points_norm(self.gaze_on_srf,color=(0.,8.,.5,.8), size=80) + + glfwSwapBuffers(self._window) + glfwMakeContextCurrent(active_window) + + def toggle_window(self,_): + if self._window: + self.window_should_close = True + else: + self.window_should_open = True + + def window_open(self): + return bool(self._window) + + + def open_window(self): + if not self._window: + if self.fullscreen: + monitor = glfwGetMonitors()[self.monitor_idx.value] + mode = glfwGetVideoMode(monitor) + height,width= mode[0],mode[1] + else: + monitor = None + height,width= 640,int(640./(self.scale_factor[0]/self.scale_factor[1])) #open with same aspect ratio as surface + + self._window = glfwCreateWindow(height, width, "Reference Surface: " + self.name, monitor=monitor, share=glfwGetCurrentContext()) + if not self.fullscreen.value: + glfwSetWindowPos(self._window,200,0) + + self.on_resize(self._window,height,width) + + #Register callbacks + glfwSetWindowSizeCallback(self._window,self.on_resize) + glfwSetKeyCallback(self._window,self.on_key) + glfwSetWindowCloseCallback(self._window,self.on_close) + + # gl_state settings + active_window = glfwGetCurrentContext() + glfwMakeContextCurrent(self._window) + basic_gl_setup() + make_coord_system_norm_based() + + + # refresh speed settings + glfwSwapInterval(0) + + glfwMakeContextCurrent(active_window) + + self.window_should_open = False + + # window calbacks + def on_resize(self,window,w, h): + active_window = glfwGetCurrentContext() + glfwMakeContextCurrent(window) + adjust_gl_view(w,h,window) + glfwMakeContextCurrent(active_window) + + def on_key(self,window, key, scancode, action, mods): + if action == GLFW_PRESS: + if key == GLFW_KEY_ESCAPE: + self.on_close() + + def on_close(self,window=None): + self.window_should_close = True + + def close_window(self): + if self._window: + glfwDestroyWindow(self._window) + self._window = None + self.window_should_close = False + + + def cleanup(self): + if self._window: + self.close_window() + + + class Support_Marker(object): ''' This is a class only to be used by Reference_Surface diff --git a/pupil_src/shared_modules/square_marker_detect.py b/pupil_src/shared_modules/square_marker_detect.py index a65398423f..4a0d4e24bb 100644 --- a/pupil_src/shared_modules/square_marker_detect.py +++ b/pupil_src/shared_modules/square_marker_detect.py @@ -200,7 +200,7 @@ def draw_markers(img,markers): for m in markers: centroid = [m['verts'].sum(axis=0)/4.] origin = m['verts'][0] - hat = np.array([[[0,0],[0,1],[.5,1.5],[1,1],[1,0]]],dtype=np.float32) + hat = np.array([[[0,0],[0,1],[.5,1.25],[1,1],[1,0]]],dtype=np.float32) hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m)) cv2.polylines(img,np.int0(hat),color = (0,0,255),isClosed=True) cv2.polylines(img,np.int0(centroid),color = (255,255,0),isClosed=True,thickness=2) @@ -245,7 +245,7 @@ def detect_markers_simple(img,grid_size,min_marker_perimeter=40,aperture=11,visu maxLevel = 1, criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03)) prev_img = None -tick = 4 +tick = 0 def detect_markers_robust(img,grid_size,prev_markers,min_marker_perimeter=40,aperture=11,visualize=False,true_detect_every_frame = 1): global prev_img diff --git a/pupil_src/shared_modules/uvc_capture/__init__.py b/pupil_src/shared_modules/uvc_capture/__init__.py index 94a2de455e..79a6ae8fc2 100644 --- a/pupil_src/shared_modules/uvc_capture/__init__.py +++ b/pupil_src/shared_modules/uvc_capture/__init__.py @@ -43,7 +43,7 @@ from other_video import Camera_Capture,Camera_List,CameraCaptureError from fake_capture import FakeCapture -from file_capture import File_Capture, FileCaptureError, EndofVideoFileError +from file_capture import File_Capture, FileCaptureError, EndofVideoFileError,FileSeekError def autoCreateCapture(src,size=(640,480),fps=30,timestamps=None,timebase = None): diff --git a/pupil_src/shared_modules/uvc_capture/file_capture.py b/pupil_src/shared_modules/uvc_capture/file_capture.py index af189d8af1..c3d053c9e8 100644 --- a/pupil_src/shared_modules/uvc_capture/file_capture.py +++ b/pupil_src/shared_modules/uvc_capture/file_capture.py @@ -44,6 +44,12 @@ def __init__(self, arg): self.arg = arg +class FileSeekError(Exception): + """docstring for EndofVideoFileError""" + def __init__(self): + super(FileSeekError, self).__init__() + + class Frame(object): """docstring of Frame""" def __init__(self, timestamp,img,index=None,compressed_img=None, compressed_pix_fmt=None): @@ -65,7 +71,7 @@ def __init__(self,src,timestamps=None): self.controls = None #No UVC controls available with file capture # we initialize the actual capture based on cv2.VideoCapture self.cap = cv2.VideoCapture(src) - if timestamps is None: + if timestamps is None and src.endswith("eye.avi"): timestamps_loc = os.path.join(src.rsplit(os.path.sep,1)[0],'eye_timestamps.npy') logger.debug("trying to auto load eye_video timestamps with video at: %s"%timestamps_loc) else: @@ -120,11 +126,29 @@ def get_frame(self): return Frame(timestamp,img,index=idx) def seek_to_frame(self, seek_pos): - logger.debug("seeking to frame: %s"%seek_pos) if self.cap.set(cv2.cv.CV_CAP_PROP_POS_FRAMES,seek_pos): - return True + offset = seek_pos - self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) + if offset == 0: + logger.debug("Seeked to frame: %s"%self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)) + return + # elif 0 < offset < 100: + # offset +=10 + # if not self.seek_to_frame(seek_pos-offset): + # logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) + # return False + # logger.warning("Seek was not precice need to do manual seek for %s frames"%offset) + # while seek_pos != self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES): + # try: + # self.read() + # except EndofVideoFileError: + # logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) + # return True + else: + logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) + raise FileSeekError() logger.error("Could not perform seek on cv2.VideoCapture. Command gave negative return.") - return False + raise FileSeekError() + def get_now(self): idx = int(self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))