From 63b303ff294f90eba2ca2cb835304b49c59857bc Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Wed, 30 Apr 2014 20:07:26 +0200 Subject: [PATCH 01/41] first commit for offline marker tracker --- pupil_src/player/main.py | 6 +- pupil_src/shared_modules/marker_detector.py | 253 +++++++----------- .../shared_modules/marker_detector_cacher.py | 83 ++++++ pupil_src/shared_modules/reference_surface.py | 165 +++++++++++- 4 files changed, 345 insertions(+), 162 deletions(-) create mode 100644 pupil_src/shared_modules/marker_detector_cacher.py diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index 1063d49f1a..6bd833b7ab 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -167,7 +167,7 @@ def on_close(window): rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: - rec_dir = "/Users/mkassner/Downloads/1-4/000/" + rec_dir = "/Users/mkassner/Downloads/ET-lottery-ticket/4055/000" if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: @@ -250,6 +250,7 @@ def save(var_name,var): g.user_dir = user_dir g.rec_dir = rec_dir g.app = 'player' + g.timestamps = timestamps @@ -388,6 +389,8 @@ def get_from_data(data): if g.play or g.new_seek: try: new_frame = cap.get_frame() + print new_frame.index + except EndofVideoFileError: #end of video logic: pause at last frame. g.play=False @@ -397,7 +400,6 @@ def get_from_data(data): g.new_seek = False frame = new_frame.copy() - #new positons and events we make a deepcopy just like the image should be a copy. current_pupil_positions = deepcopy(positions_by_frame[frame.index]) events = [] diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index 4277a9a8d9..5d890c14c4 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -8,7 +8,7 @@ ----------------------------------------------------------------------------------~(*) ''' -import os +import sys, os,platform import cv2 import numpy as np import shelve @@ -18,10 +18,6 @@ import audio from ctypes import c_int,c_bool,create_string_buffer -from OpenGL.GL import * -from OpenGL.GLU import gluOrtho2D - -from glfw import * from plugin import Plugin #logging import logging @@ -31,12 +27,14 @@ from reference_surface import Reference_Surface from math import sqrt -# window calbacks -def on_resize(window,w, h): - active_window = glfwGetCurrentContext() - glfwMakeContextCurrent(window) - adjust_gl_view(w,h,window) - glfwMakeContextCurrent(active_window) +if platform.system() == 'Darwin': + from billiard import Process,Queue,forking_enable + from billiard.sharedctypes import Value +else: + from multiprocessing import Process, Pipe, Event, Queue + forking_enable = lambda x: x #dummy fn + from multiprocessing.sharedctypes import Value + class Marker_Detector(Plugin): """docstring @@ -75,6 +73,17 @@ def __init__(self,g_pool,atb_pos=(320,220)): self.aperture = c_int(11) self.min_marker_perimeter = 80 + # caching vars when working with video file src + if g_pool.app == "player": + #check if marker cache is available from last session + self.persistent_cache = shelve.open(os.path.join(g_pool.rec_dir,'square_marker_cache'),protocol=2) + self.marker_cache = self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps]) + logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.marker_cache)-self.marker_cache.count(False),len(self.marker_cache)) ) + self.init_marker_cacher() + else: + self.persistent_cache = None + self.marker_cache = None + #debug vars self.draw_markers = c_bool(0) self.show_surface_idx = c_int(0) @@ -82,25 +91,10 @@ def __init__(self,g_pool,atb_pos=(320,220)): self.img_shape = None - #multi monitor setup - self.window_should_open = False - self.window_should_close = False - self._window = None - self.fullscreen = c_bool(0) - self.monitor_idx = c_int(0) - monitor_handles = glfwGetMonitors() - self.monitor_names = [glfwGetMonitorName(m) for m in monitor_handles] - monitor_enum = atb.enum("Monitor",dict(((key,val) for val,key in enumerate(self.monitor_names)))) - #primary_monitor = glfwGetPrimaryMonitor() - atb_label = "marker detection" self._bar = atb.Bar(name =self.__class__.__name__, label=atb_label, help="marker detection parameters", color=(50, 50, 50), alpha=100, text='light', position=atb_pos,refresh=.3, size=(300, 100)) - self._bar.add_var("monitor",self.monitor_idx, vtype=monitor_enum,group="Window",) - self._bar.add_var("fullscreen", self.fullscreen,group="Window") - self._bar.add_button(" open Window ", self.do_open, key='m',group="Window") - self._bar.add_var("surface to show",self.show_surface_idx, step=1,min=0,group="Window") self._bar.add_var('robust_detection',self.robust_detection,group="Detector") self._bar.add_var("draw markers",self.draw_markers,group="Detector") self._bar.add_button('close',self.unset_alive) @@ -147,61 +141,12 @@ def on_click(self,pos,button,action): def advance(self): pass - def open_window(self): - if not self._window: - if self.fullscreen.value: - monitor = glfwGetMonitors()[self.monitor_idx.value] - mode = glfwGetVideoMode(monitor) - height,width= mode[0],mode[1] - else: - monitor = None - height,width= 1280,720 - - self._window = glfwCreateWindow(height, width, "Reference Surface", monitor=monitor, share=glfwGetCurrentContext()) - if not self.fullscreen.value: - glfwSetWindowPos(self._window,200,0) - - on_resize(self._window,height,width) - - #Register callbacks - glfwSetWindowSizeCallback(self._window,on_resize) - glfwSetKeyCallback(self._window,self.on_key) - glfwSetWindowCloseCallback(self._window,self.on_close) - - # gl_state settings - active_window = glfwGetCurrentContext() - glfwMakeContextCurrent(self._window) - basic_gl_setup() - - # refresh speed settings - glfwSwapInterval(0) - - - glfwMakeContextCurrent(active_window) - - self.window_should_open = False - - - def on_key(self,window, key, scancode, action, mods): - if not atb.TwEventKeyboardGLFW(key,int(action == GLFW_PRESS)): - if action == GLFW_PRESS: - if key == GLFW_KEY_ESCAPE: - self.on_close() - - def on_close(self,window=None): - self.window_should_close = True - - def close_window(self): - if self._window: - glfwDestroyWindow(self._window) - self._window = None - self.window_should_close = False - def add_surface(self): self.surfaces.append(Reference_Surface()) self.update_bar_markers() def remove_surface(self,i): + self.surfaces[i].cleanup() del self.surfaces[i] self.update_bar_markers() @@ -210,8 +155,8 @@ def update_bar_markers(self): self._bar_markers.clear() self._bar_markers.add_button(" add surface ", self.add_surface, key='a') self._bar_markers.add_var(" edit mode ", self.surface_edit_mode ) - - for s,i in zip (self.surfaces,range(len(self.surfaces))): + for s,i in zip(self.surfaces,range(len(self.surfaces)))[::-1]: + self._bar_markers.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window') self._bar_markers.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') self._bar_markers.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) self._bar_markers.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) @@ -219,26 +164,48 @@ def update_bar_markers(self): def update(self,frame,recent_pupil_positions,events): img = frame.img self.img_shape = frame.img.shape - if self.robust_detection.value: - self.markers = detect_markers_robust(img,grid_size = 5, - prev_markers=self.markers, - min_marker_perimeter=self.min_marker_perimeter, - aperture=self.aperture.value, - visualize=0, - true_detect_every_frame=3) + + if self.marker_cache is not None: + self.update_marker_cache() + self.markers = self.marker_cache[frame.index] + if self.markers == False: + # locate markers because precacher has not anayzed this frame yet. Most likely a seek event + self.markers = detect_markers_simple(img, + grid_size = 5, + min_marker_perimeter=self.min_marker_perimeter, + aperture=self.aperture.value, + visualize=0) + + self.seek_marker_cacher(frame.index+1) # tell precacher that it better have the next frame analyzed + else: - self.markers = detect_markers_simple(img,grid_size = 5,min_marker_perimeter=self.min_marker_perimeter,aperture=self.aperture.value,visualize=0) + # locate markers during realtime, dont use robust detection for video file when seeks can throw thigs off. + if self.robust_detection.value: + self.markers = detect_markers_robust(img, + grid_size = 5, + prev_markers=self.markers, + min_marker_perimeter=self.min_marker_perimeter, + aperture=self.aperture.value, + visualize=0, + true_detect_every_frame=3) + else: + self.markers = detect_markers_simple(img, + grid_size = 5, + min_marker_perimeter=self.min_marker_perimeter, + aperture=self.aperture.value, + visualize=0) if self.draw_markers.value: draw_markers(img,self.markers) - # print self.markers + # locate surfaces for s in self.surfaces: s.locate(self.markers) if s.detected: - events.append({'type':'marker_ref_surface','name':s.name,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) + events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) + # edit surfaces by user if self.surface_edit_mode: window = glfwGetCurrentContext() pos = glfwGetCursorPos(window) @@ -252,23 +219,49 @@ def update(self,frame,recent_pupil_positions,events): s.move_vertex(v_idx,new_pos) #map recent gaze onto detected surfaces used for pupil server - for p in recent_pupil_positions: - if p['norm_pupil'] is not None: - for s in self.surfaces: - if s.detected: - p['realtime gaze on '+s.name] = tuple(s.img_to_ref_surface(np.array(p['norm_gaze']))) + for s in self.surfaces: + if s.detected: + s.recent_gaze = [] + for p in recent_pupil_positions: + if p['norm_pupil'] is not None: + gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_gaze']))) + p['realtime gaze on '+s.name] = gp_on_s + s.recent_gaze.append(gp_on_s) + + + #allow surfaces to open/close windows + for s in self.surfaces: + if s.window_should_close: + s.close_window() + if s.window_should_open: + s.open_window() + + + def init_marker_cacher(self): + forking_enable(0) #for MacOs only + from marker_detector_cacher import fill_cache + visited_list = [False if x == False else True for x in self.marker_cache] + video_file_path = os.path.join(self.g_pool.rec_dir,'world.avi') + self.marker_cache_queue = Queue() + self.marker_cacher_seek_idx = Value(c_int,0) + self.marker_cacher_run = Value(c_bool,True) + self.marker_cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.marker_cache_queue,self.marker_cacher_seek_idx,self.marker_cacher_run)) + self.marker_cacher.start() - if self._window: - # save a local copy for when we display gaze for debugging on ref surface - self.recent_pupil_positions = recent_pupil_positions + def update_marker_cache(self): + while not self.marker_cache_queue.empty(): + idx,c_m = self.marker_cache_queue.get() + self.marker_cache[idx] = c_m - if self.window_should_close: - self.close_window() + def seek_marker_cacher(self,idx): + self.marker_cacher_seek_idx.value = idx - if self.window_should_open: - self.open_window() + def close_marker_cacher(self): + self.update_marker_cache() + self.marker_cacher_run.value = False + self.marker_cacher.join() def gl_display(self): """ @@ -282,62 +275,13 @@ def gl_display(self): for s in self.surfaces: s.gl_draw_frame() + s.gl_display_in_window(self.g_pool.image_tex) if self.surface_edit_mode.value: for s in self.surfaces: s.gl_draw_corners() - if self._window and self.surfaces: - try: - s = self.surfaces[self.show_surface_idx.value] - except IndexError: - s = None - if s and s.detected: - self.gl_display_in_window(s) - - - - def gl_display_in_window(self,surface): - """ - here we map a selected surface onto a seperate window. - """ - active_window = glfwGetCurrentContext() - glfwMakeContextCurrent(self._window) - clear_gl_screen() - - # cv uses 3x3 gl uses 4x4 tranformation matricies - m = cvmat_to_glmat(surface.m_from_screen) - - glMatrixMode(GL_PROJECTION) - glPushMatrix() - glLoadIdentity() - gluOrtho2D(0, 1, 0, 1) # gl coord convention - - glMatrixMode(GL_MODELVIEW) - glPushMatrix() - #apply m to our quad - this will stretch the quad such that the ref suface will span the window extends - glLoadMatrixf(m) - - draw_named_texture(self.g_pool.image_tex) - - glMatrixMode(GL_PROJECTION) - glPopMatrix() - glMatrixMode(GL_MODELVIEW) - glPopMatrix() - - - #now lets get recent pupil positions on this surface: - try: - gaze_on_surface = [p['realtime gaze on '+surface.name] for p in self.recent_pupil_positions] - except KeyError: - gaze_on_surface = [] - draw_gl_points_norm(gaze_on_surface,color=(0.,8.,.5,.8), size=80) - - - glfwSwapBuffers(self._window) - glfwMakeContextCurrent(active_window) - def cleanup(self): """ called when the plugin gets terminated. @@ -348,10 +292,13 @@ def cleanup(self): self.save("realtime_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) elif self.g_pool.app == 'player': self.save("offline_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) + self.close_marker_cacher() + self.persistent_cache["marker_cache"] = self.marker_cache + self.persistent_cache.close() + self.surface_definitions.close() - if self._window: - self.close_window() + for s in self.surfaces: + s.close_window() self._bar.destroy() self._bar_markers.destroy() - diff --git a/pupil_src/shared_modules/marker_detector_cacher.py b/pupil_src/shared_modules/marker_detector_cacher.py new file mode 100644 index 0000000000..e7030d1025 --- /dev/null +++ b/pupil_src/shared_modules/marker_detector_cacher.py @@ -0,0 +1,83 @@ + + +def fill_cache(visited_list,video_file_path,q,seek_idx,run): + import logging,os + logger = logging.getLogger(__name__+' with pid: '+str(os.getpid()) ) + logger.debug('Started Cacher Process for Marker Detector') + import cv2 + from uvc_capture import autoCreateCapture, EndofVideoFileError + from square_marker_detect import detect_markers_robust + min_marker_perimeter = 80 + aperture = 11 + markers = [] + + cap = autoCreateCapture(video_file_path) + frame_idx = 0 + + def next_unvisited_idx(frame_idx): + try: + visited = visited_list[frame_idx] + except IndexError: + visited = True # trigger search + + if not visited: + next_unvisited = frame_idx + else: + # find next unvisited site in the future + try: + next_unvisited = visited_list.index(False,frame_idx) + except IndexError: + # any thing in the past? + try: + next_unvisited = visited_list.index(False,0,frame_idx) + except IndexError: + #no unvisited sites left. Done! + logger.debug("Caching Completed") + next_unvisited = None + return next_unvisited + + + while run.value: + + if seek_idx.value != -1: + frame_idx = seek_idx.value + seek_idx.value = -1 + + #check the visited list + next = next_unvisited_idx(frame_idx) + if next == None: + #we are done here: + break + elif next != frame_idx: + #we need to seek: + logger.debug("seeking to Frame %s" %next) + cap.seek_to_frame(next) + frame_idx = next + else: + #next frame is unvisited + pass + + + try: + frame = cap.get_frame() + except EndofVideoFileError: + logger.debug('Reached end of video. Rewinding.') + frame = None + cap.seek_to_frame(0) + frame_idx = 0 + else: + markers = detect_markers_robust(frame.img, + grid_size = 5, + prev_markers=markers, + min_marker_perimeter=min_marker_perimeter, + aperture=aperture, + visualize=0, + true_detect_every_frame=1) + visited_list[frame.index] = True + logger.debug("adding Frame %s"%frame.index) + q.put((frame.index,markers)) + frame_idx +=1 + logger.debug("Closing Cacher Process") + cap.close() + q.close() + return \ No newline at end of file diff --git a/pupil_src/shared_modules/reference_surface.py b/pupil_src/shared_modules/reference_surface.py index ae7fe42a26..aebbe5d1a7 100644 --- a/pupil_src/shared_modules/reference_surface.py +++ b/pupil_src/shared_modules/reference_surface.py @@ -10,12 +10,16 @@ import numpy as np import cv2 -from gl_utils import draw_gl_polyline_norm,draw_gl_points_norm,draw_gl_point_norm +from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture +from glfw import * +from OpenGL.GL import * +from OpenGL.GLU import gluOrtho2D + from methods import GetAnglesPolyline,normalize #ctypes import for atb_vars: -from ctypes import create_string_buffer - +from ctypes import c_int,c_bool,create_string_buffer +from time import time def m_verts_to_screen(verts): #verts need to be sorted counterclockwise stating at bottom left @@ -65,18 +69,38 @@ def __init__(self,name="unnamed",saved_definition=None): self.detected = False self.m_to_screen = None self.m_from_screen = None + self.cache = None if saved_definition is not None: self.load_from_dict(saved_definition) - - + else: + self.uid = str(time()) + + ###window and gui vars + self._window = None + self.fullscreen = False + self.window_should_open = False + self.window_should_close = False + + #multi monitor setup + self.window_should_open = False + self.window_should_close = False + self._window = None + self.fullscreen = c_bool(0) + self.monitor_idx = c_int(0) + monitor_handles = glfwGetMonitors() + self.monitor_names = [glfwGetMonitorName(m) for m in monitor_handles] + # monitor_enum = atb.enum("Monitor",dict(((key,val) for val,key in enumerate(self.monitor_names)))) + #primary_monitor = glfwGetPrimaryMonitor() + + self.recent_gaze = [] # points on surface for realtime feedback display def save_to_dict(self): """ save all markers and name of this surface to a dict. """ markers = dict([(m_id,m.uv_coords) for m_id,m in self.markers.iteritems()]) - return {'name':self.name,'markers':markers} + return {'name':self.name,'uid':self.uid,'markers':markers} def load_from_dict(self,d): @@ -84,6 +108,7 @@ def load_from_dict(self,d): load all markers of this surface to a dict. """ self.name = d['name'] + self.uid = d['uid'] marker_dict = d['markers'] for m_id,uv_coords in marker_dict.iteritems(): self.markers[m_id] = Support_Marker(m_id) @@ -196,7 +221,25 @@ def locate(self, visible_markers): self.m_from_screen = None self.m_to_screen = None - + def answer_caching_request(self,visible_markers): + marker_by_id = dict([(m['id'],m) for m in visible_markers]) + visible_ids = set(marker_by_id.keys()) + requested_ids = set(self.markers.keys()) + overlap = visible_ids & requested_ids + detected_markers = len(overlap) + if len(overlap)>=min(2,len(requested_ids)): + yx = np.array( [marker_by_id[i]['verts_norm'] for i in overlap] ) + uv = np.array( [self.markers[i].uv_coords for i in overlap] ) + yx.shape=(-1,1,2) + uv.shape=(-1,1,2) + # print 'uv',uv + # print 'yx',yx + m_to_screen,mask = cv2.findHomography(uv,yx) + m_from_screen,mask = cv2.findHomography(yx,uv) + + return {'m_to_screen':m_to_screen,'m_from_screen':m_from_screen,'detected_markers':len(overlap)} + else: + return None def img_to_ref_surface(self,pos): if self.m_from_screen is not None: @@ -269,6 +312,114 @@ def gl_draw_corners(self): draw_gl_points_norm(frame.reshape((4,2)),15,(1.0,0.2,0.6,.5)) + + #### fns to draw surface in seperate window + def gl_display_in_window(self,world_tex_id): + """ + here we map a selected surface onto a seperate window. + """ + if self._window and self.detected: + active_window = glfwGetCurrentContext() + glfwMakeContextCurrent(self._window) + clear_gl_screen() + + # cv uses 3x3 gl uses 4x4 tranformation matricies + m = cvmat_to_glmat(self.m_from_screen) + + glMatrixMode(GL_PROJECTION) + glPushMatrix() + glLoadIdentity() + gluOrtho2D(0, 1, 0, 1) # gl coord convention + + glMatrixMode(GL_MODELVIEW) + glPushMatrix() + #apply m to our quad - this will stretch the quad such that the ref suface will span the window extends + glLoadMatrixf(m) + + draw_named_texture(world_tex_id) + + glMatrixMode(GL_PROJECTION) + glPopMatrix() + glMatrixMode(GL_MODELVIEW) + glPopMatrix() + + + #now lets get recent pupil positions on this surface: + draw_gl_points_norm(self.recent_gaze,color=(0.,8.,.5,.8), size=80) + + glfwSwapBuffers(self._window) + glfwMakeContextCurrent(active_window) + + def toggle_window(self,_): + if self._window: + self.window_should_close = True + else: + self.window_should_open = True + + def window_open(self): + return bool(self._window) + + + def open_window(self): + if not self._window: + if self.fullscreen: + monitor = glfwGetMonitors()[self.monitor_idx.value] + mode = glfwGetVideoMode(monitor) + height,width= mode[0],mode[1] + else: + monitor = None + height,width= 640,640 + + self._window = glfwCreateWindow(height, width, "Reference Surface: " + self.name, monitor=monitor, share=glfwGetCurrentContext()) + if not self.fullscreen.value: + glfwSetWindowPos(self._window,200,0) + + self.on_resize(self._window,height,width) + + #Register callbacks + glfwSetWindowSizeCallback(self._window,self.on_resize) + glfwSetKeyCallback(self._window,self.on_key) + glfwSetWindowCloseCallback(self._window,self.on_close) + + # gl_state settings + active_window = glfwGetCurrentContext() + glfwMakeContextCurrent(self._window) + basic_gl_setup() + + # refresh speed settings + glfwSwapInterval(0) + + glfwMakeContextCurrent(active_window) + + self.window_should_open = False + + # window calbacks + def on_resize(self,window,w, h): + active_window = glfwGetCurrentContext() + glfwMakeContextCurrent(window) + adjust_gl_view(w,h,window) + glfwMakeContextCurrent(active_window) + + def on_key(self,window, key, scancode, action, mods): + if action == GLFW_PRESS: + if key == GLFW_KEY_ESCAPE: + self.on_close() + + def on_close(self,window=None): + self.window_should_close = True + + def close_window(self): + if self._window: + glfwDestroyWindow(self._window) + self._window = None + self.window_should_close = False + + + def cleanup(self): + if self._window: + self.close_window() + + class Support_Marker(object): ''' This is a class only to be used by Reference_Surface From 04ea998e4a2571b597986c86d137adc9ff362e1e Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Wed, 30 Apr 2014 20:42:08 +0200 Subject: [PATCH 02/41] first working version of markerdetection with prefetching --- pupil_src/player/main.py | 5 ++--- pupil_src/shared_modules/marker_detector.py | 2 +- .../shared_modules/marker_detector_cacher.py | 22 +++++++++---------- 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index 6bd833b7ab..9893c9226a 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -58,7 +58,7 @@ fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() -ch.setLevel(logging.WARNING) +ch.setLevel(logging.DEBUG) # create formatter and add it to the handlers formatter = logging.Formatter('Player: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) @@ -167,7 +167,7 @@ def on_close(window): rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: - rec_dir = "/Users/mkassner/Downloads/ET-lottery-ticket/4055/000" + rec_dir = '/home/mkassner/Desktop/000' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: @@ -389,7 +389,6 @@ def get_from_data(data): if g.play or g.new_seek: try: new_frame = cap.get_frame() - print new_frame.index except EndofVideoFileError: #end of video logic: pause at last frame. diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index 5d890c14c4..d741af8b65 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -176,7 +176,7 @@ def update(self,frame,recent_pupil_positions,events): aperture=self.aperture.value, visualize=0) - self.seek_marker_cacher(frame.index+1) # tell precacher that it better have the next frame analyzed + self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here analyzed else: # locate markers during realtime, dont use robust detection for video file when seeks can throw thigs off. diff --git a/pupil_src/shared_modules/marker_detector_cacher.py b/pupil_src/shared_modules/marker_detector_cacher.py index e7030d1025..b9b7f2fe5e 100644 --- a/pupil_src/shared_modules/marker_detector_cacher.py +++ b/pupil_src/shared_modules/marker_detector_cacher.py @@ -12,7 +12,6 @@ def fill_cache(visited_list,video_file_path,q,seek_idx,run): markers = [] cap = autoCreateCapture(video_file_path) - frame_idx = 0 def next_unvisited_idx(frame_idx): try: @@ -26,11 +25,11 @@ def next_unvisited_idx(frame_idx): # find next unvisited site in the future try: next_unvisited = visited_list.index(False,frame_idx) - except IndexError: + except ValueError: # any thing in the past? try: next_unvisited = visited_list.index(False,0,frame_idx) - except IndexError: + except ValueError: #no unvisited sites left. Done! logger.debug("Caching Completed") next_unvisited = None @@ -38,33 +37,32 @@ def next_unvisited_idx(frame_idx): while run.value: - + next = cap.get_frame_index() if seek_idx.value != -1: - frame_idx = seek_idx.value + next = seek_idx.value seek_idx.value = -1 + logger.debug("User seek require marker caching at %s"%next) + #check the visited list - next = next_unvisited_idx(frame_idx) + next = next_unvisited_idx(next) if next == None: #we are done here: break - elif next != frame_idx: + elif next != cap.get_frame_index(): #we need to seek: logger.debug("seeking to Frame %s" %next) cap.seek_to_frame(next) - frame_idx = next else: #next frame is unvisited pass - try: frame = cap.get_frame() except EndofVideoFileError: logger.debug('Reached end of video. Rewinding.') frame = None cap.seek_to_frame(0) - frame_idx = 0 else: markers = detect_markers_robust(frame.img, grid_size = 5, @@ -74,9 +72,9 @@ def next_unvisited_idx(frame_idx): visualize=0, true_detect_every_frame=1) visited_list[frame.index] = True - logger.debug("adding Frame %s"%frame.index) + # logger.debug("adding Frame %s"%frame.index) q.put((frame.index,markers)) - frame_idx +=1 + logger.debug("Closing Cacher Process") cap.close() q.close() From b2d1bf068fd32a756516485e38366982fbed0a7c Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Thu, 1 May 2014 08:31:21 +0200 Subject: [PATCH 03/41] varous bugfixes detected during developement --- pupil_src/capture/main.py | 4 +-- pupil_src/capture/recorder.py | 4 ++- pupil_src/player/main.py | 3 +- pupil_src/player/seek_bar.py | 2 +- pupil_src/shared_modules/marker_detector.py | 3 +- .../shared_modules/marker_detector_cacher.py | 28 +++++++++++++++---- .../shared_modules/square_marker_detect.py | 2 +- .../uvc_capture/file_capture.py | 17 +++++++++-- 8 files changed, 47 insertions(+), 16 deletions(-) diff --git a/pupil_src/capture/main.py b/pupil_src/capture/main.py index ca33ade22d..b14c36d26e 100644 --- a/pupil_src/capture/main.py +++ b/pupil_src/capture/main.py @@ -87,7 +87,7 @@ with open(version_file) as f: version = f.read() else: - from git_version import get_tag_commit + from gitversion import get_tag_commit version = get_tag_commit() @@ -103,7 +103,7 @@ def main(): # to use a pre-recorded video. # Use a string to specify the path to your video file as demonstrated below # eye_src = '/Users/mkassner/Pupil/datasets/p1-left/frames/test.avi' - # world_src = "/Users/mkassner/Desktop/2014_01_21/000/world.avi" + # world_src = "~/Desktop/000" # Camera video size in pixels (width,height) eye_size = (640,360) diff --git a/pupil_src/capture/recorder.py b/pupil_src/capture/recorder.py index 064df0613f..6fcc059785 100644 --- a/pupil_src/capture/recorder.py +++ b/pupil_src/capture/recorder.py @@ -91,13 +91,15 @@ def get_rec_time_str(self): return strftime("%H:%M:%S", rec_time) def update(self,frame,recent_pupil_positons,events): - self.frame_count += 1 + # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)) for p in recent_pupil_positons: if p['norm_pupil'] is not None: gaze_pt = p['norm_gaze'][0],p['norm_gaze'][1],p['norm_pupil'][0],p['norm_pupil'][1],p['timestamp'],p['confidence'] self.gaze_list.append(gaze_pt) self.timestamps.append(frame.timestamp) self.writer.write(frame.img) + self.frame_count += 1 + def stop_and_destruct(self): #explicit release of VideoWriter diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index 9893c9226a..3ec2264d0b 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -58,7 +58,7 @@ fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) +ch.setLevel(logging.WARNING) # create formatter and add it to the handlers formatter = logging.Formatter('Player: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) @@ -389,7 +389,6 @@ def get_from_data(data): if g.play or g.new_seek: try: new_frame = cap.get_frame() - except EndofVideoFileError: #end of video logic: pause at last frame. g.play=False diff --git a/pupil_src/player/seek_bar.py b/pupil_src/player/seek_bar.py index 9089596c29..145290f1eb 100644 --- a/pupil_src/player/seek_bar.py +++ b/pupil_src/player/seek_bar.py @@ -48,7 +48,7 @@ def update(self,frame,recent_pupil_positions,events): norm_seek_pos, _ = self.screen_to_seek_bar(pos) norm_seek_pos = min(1,max(0,norm_seek_pos)) if abs(norm_seek_pos-self.norm_seek_pos) >=.01: - seek_pos = int(norm_seek_pos*self.frame_count) + seek_pos = min(int(norm_seek_pos*self.frame_count),self.frame_count-1) self.cap.seek_to_frame(seek_pos) self.g_pool.new_seek = True diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index d741af8b65..3a1b592061 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -239,7 +239,6 @@ def update(self,frame,recent_pupil_positions,events): def init_marker_cacher(self): forking_enable(0) #for MacOs only - from marker_detector_cacher import fill_cache visited_list = [False if x == False else True for x in self.marker_cache] video_file_path = os.path.join(self.g_pool.rec_dir,'world.avi') @@ -254,6 +253,8 @@ def update_marker_cache(self): while not self.marker_cache_queue.empty(): idx,c_m = self.marker_cache_queue.get() self.marker_cache[idx] = c_m + # status = ['o' if x!=False else 'x' for x in self.marker_cache[::1000]] + # print "".join(status) def seek_marker_cacher(self,idx): self.marker_cacher_seek_idx.value = idx diff --git a/pupil_src/shared_modules/marker_detector_cacher.py b/pupil_src/shared_modules/marker_detector_cacher.py index b9b7f2fe5e..3d2bbc6dd7 100644 --- a/pupil_src/shared_modules/marker_detector_cacher.py +++ b/pupil_src/shared_modules/marker_detector_cacher.py @@ -1,9 +1,25 @@ +''' +(*)~---------------------------------------------------------------------------------- + Pupil - eye tracking platform + Copyright (C) 2012-2014 Pupil Labs + Distributed under the terms of the CC BY-NC-SA License. + License details are in the file license.txt, distributed as part of this software. +----------------------------------------------------------------------------------~(*) +''' -def fill_cache(visited_list,video_file_path,q,seek_idx,run): - import logging,os + + + +def fill_cache(visited_list,video_file_path,q,seek_idx,run): + ''' + this function is part of marker_detector it is run as a seperate process. + it must be kept in a seperate file for namespace sanatisation + ''' + import os + import logging logger = logging.getLogger(__name__+' with pid: '+str(os.getpid()) ) - logger.debug('Started Cacher Process for Marker Detector') + logger.debug('Started cacher process for Marker Detector') import cv2 from uvc_capture import autoCreateCapture, EndofVideoFileError from square_marker_detect import detect_markers_robust @@ -31,7 +47,7 @@ def next_unvisited_idx(frame_idx): next_unvisited = visited_list.index(False,0,frame_idx) except ValueError: #no unvisited sites left. Done! - logger.debug("Caching Completed") + logger.debug("Caching completed.") next_unvisited = None return next_unvisited @@ -51,8 +67,9 @@ def next_unvisited_idx(frame_idx): break elif next != cap.get_frame_index(): #we need to seek: - logger.debug("seeking to Frame %s" %next) + logger.debug("Seeking to Frame %s" %next) cap.seek_to_frame(next) + markers = [] else: #next frame is unvisited pass @@ -63,6 +80,7 @@ def next_unvisited_idx(frame_idx): logger.debug('Reached end of video. Rewinding.') frame = None cap.seek_to_frame(0) + markers = [] else: markers = detect_markers_robust(frame.img, grid_size = 5, diff --git a/pupil_src/shared_modules/square_marker_detect.py b/pupil_src/shared_modules/square_marker_detect.py index a65398423f..78629d68d0 100644 --- a/pupil_src/shared_modules/square_marker_detect.py +++ b/pupil_src/shared_modules/square_marker_detect.py @@ -245,7 +245,7 @@ def detect_markers_simple(img,grid_size,min_marker_perimeter=40,aperture=11,visu maxLevel = 1, criteria = (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 10, 0.03)) prev_img = None -tick = 4 +tick = 0 def detect_markers_robust(img,grid_size,prev_markers,min_marker_perimeter=40,aperture=11,visualize=False,true_detect_every_frame = 1): global prev_img diff --git a/pupil_src/shared_modules/uvc_capture/file_capture.py b/pupil_src/shared_modules/uvc_capture/file_capture.py index af189d8af1..23fe146d01 100644 --- a/pupil_src/shared_modules/uvc_capture/file_capture.py +++ b/pupil_src/shared_modules/uvc_capture/file_capture.py @@ -65,7 +65,7 @@ def __init__(self,src,timestamps=None): self.controls = None #No UVC controls available with file capture # we initialize the actual capture based on cv2.VideoCapture self.cap = cv2.VideoCapture(src) - if timestamps is None: + if timestamps is None and src.endswith("eye.avi"): timestamps_loc = os.path.join(src.rsplit(os.path.sep,1)[0],'eye_timestamps.npy') logger.debug("trying to auto load eye_video timestamps with video at: %s"%timestamps_loc) else: @@ -120,9 +120,20 @@ def get_frame(self): return Frame(timestamp,img,index=idx) def seek_to_frame(self, seek_pos): - logger.debug("seeking to frame: %s"%seek_pos) if self.cap.set(cv2.cv.CV_CAP_PROP_POS_FRAMES,seek_pos): - return True + offset = seek_pos - self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) + if offset == 0: + logger.debug("seeking to frame: %s"%self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)) + return True + # elif 0 < offset < 100: + # logger("Seek was not precice need to do manual seek for %s frames"%offset) + # while seek_pos != self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES): + # self.read() + # logger.debug("seeking to frame: %s"%self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)) + # return True + else: + logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) + return False logger.error("Could not perform seek on cv2.VideoCapture. Command gave negative return.") return False From 32e8c8ff76fab6e5f0444777f9e517fdc96ee222 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Thu, 1 May 2014 10:52:24 +0200 Subject: [PATCH 04/41] fixed typo --- pupil_src/capture/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pupil_src/capture/main.py b/pupil_src/capture/main.py index b14c36d26e..12bc01380a 100644 --- a/pupil_src/capture/main.py +++ b/pupil_src/capture/main.py @@ -37,6 +37,7 @@ # Make all pupil shared_modules available to this Python session. pupil_base_dir = os.path.abspath(__file__).rsplit('pupil_src', 1)[0] sys.path.append(os.path.join(pupil_base_dir, 'pupil_src', 'shared_modules')) + print sys.path # Specifiy user dirs. rec_dir = os.path.join(pupil_base_dir,'recordings') user_dir = os.path.join(pupil_base_dir,'settings') @@ -87,7 +88,7 @@ with open(version_file) as f: version = f.read() else: - from gitversion import get_tag_commit + from git_version import get_tag_commit version = get_tag_commit() From 925b4f0c36b917ea269cb276bc8f7325a4246cbe Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Thu, 1 May 2014 10:52:49 +0200 Subject: [PATCH 05/41] fixed typo --- pupil_src/capture/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pupil_src/capture/main.py b/pupil_src/capture/main.py index 12bc01380a..21049b922c 100644 --- a/pupil_src/capture/main.py +++ b/pupil_src/capture/main.py @@ -37,7 +37,6 @@ # Make all pupil shared_modules available to this Python session. pupil_base_dir = os.path.abspath(__file__).rsplit('pupil_src', 1)[0] sys.path.append(os.path.join(pupil_base_dir, 'pupil_src', 'shared_modules')) - print sys.path # Specifiy user dirs. rec_dir = os.path.join(pupil_base_dir,'recordings') user_dir = os.path.join(pupil_base_dir,'settings') From 8f26715ee8c4494b9a05b8a7e5a155d2cf84b5b7 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Thu, 1 May 2014 14:45:33 +0200 Subject: [PATCH 06/41] fixed atb frame index value type --- pupil_src/player/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index 3ec2264d0b..f7e54d61c4 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -167,7 +167,7 @@ def on_close(window): rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: - rec_dir = '/home/mkassner/Desktop/000' + rec_dir = '/home/mkassner/Desktop/001' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: @@ -332,7 +332,7 @@ def get_from_data(data): bar.add_var("play",vtype=c_bool,getter=get_play,setter=set_play,key="space") bar.add_button('step next',next_frame,key='right') bar.add_button('step prev',prev_frame,key='left') - bar.add_var("frame index",getter=lambda:cap.get_frame_index()-1 ) + bar.add_var("frame index",vtype=c_int,getter=lambda:cap.get_frame_index()-1 ) bar.plugin_to_load = c_int(0) plugin_type_enum = atb.enum("Plug In",index_by_name) From 74beb5cdcc43ad6d12fd8f5de5010d0274c18894 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Fri, 2 May 2014 00:01:06 +0200 Subject: [PATCH 07/41] added cache list a class for handling caching data --- pupil_src/player/main.py | 4 +- pupil_src/shared_modules/marker_detector.py | 21 ++--- pupil_src/shared_modules/methods.py | 83 +++++++++++++++++++ pupil_src/shared_modules/reference_surface.py | 41 ++++++++- .../uvc_capture/file_capture.py | 2 +- 5 files changed, 136 insertions(+), 15 deletions(-) diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index f7e54d61c4..bcbca41539 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -58,7 +58,7 @@ fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() -ch.setLevel(logging.WARNING) +ch.setLevel(logging.DEBUG) # create formatter and add it to the handlers formatter = logging.Formatter('Player: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) @@ -167,7 +167,7 @@ def on_close(window): rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: - rec_dir = '/home/mkassner/Desktop/001' + rec_dir = '/home/mkassner/Desktop/002' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index 3a1b592061..22e6d5bab6 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -12,10 +12,11 @@ import cv2 import numpy as np import shelve + from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture -from methods import normalize,denormalize +from methods import normalize,denormalize,Cache_List +from glfw import * import atb -import audio from ctypes import c_int,c_bool,create_string_buffer from plugin import Plugin @@ -77,7 +78,7 @@ def __init__(self,g_pool,atb_pos=(320,220)): if g_pool.app == "player": #check if marker cache is available from last session self.persistent_cache = shelve.open(os.path.join(g_pool.rec_dir,'square_marker_cache'),protocol=2) - self.marker_cache = self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps]) + self.marker_cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps])) logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.marker_cache)-self.marker_cache.count(False),len(self.marker_cache)) ) self.init_marker_cacher() else: @@ -117,10 +118,6 @@ def save(self, var_name, var): self.surface_definitions[var_name] = var - def do_open(self): - if not self._window: - self.window_should_open = True - def on_click(self,pos,button,action): if self.surface_edit_mode.value: if self.edit_surfaces: @@ -160,6 +157,7 @@ def update_bar_markers(self): self._bar_markers.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') self._bar_markers.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) self._bar_markers.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) + self._bar_markers.add_button("%s_update_cache"%i, s.update_cache,label='update cache',group=str(i),data=self.marker_cache) def update(self,frame,recent_pupil_positions,events): img = frame.img @@ -252,7 +250,7 @@ def init_marker_cacher(self): def update_marker_cache(self): while not self.marker_cache_queue.empty(): idx,c_m = self.marker_cache_queue.get() - self.marker_cache[idx] = c_m + self.marker_cache.update(idx,c_m) # status = ['o' if x!=False else 'x' for x in self.marker_cache[::1000]] # print "".join(status) @@ -274,7 +272,7 @@ def gl_display(self): hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m)) draw_gl_polyline(hat.reshape((6,2)),(0.1,1.,1.,.5)) - for s in self.surfaces: + for s in self.surfaces: s.gl_draw_frame() s.gl_display_in_window(self.g_pool.image_tex) @@ -294,7 +292,7 @@ def cleanup(self): elif self.g_pool.app == 'player': self.save("offline_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) self.close_marker_cacher() - self.persistent_cache["marker_cache"] = self.marker_cache + self.persistent_cache["marker_cache"] = self.marker_cache.to_list() self.persistent_cache.close() self.surface_definitions.close() @@ -303,3 +301,6 @@ def cleanup(self): s.close_window() self._bar.destroy() self._bar_markers.destroy() + + + diff --git a/pupil_src/shared_modules/methods.py b/pupil_src/shared_modules/methods.py index 755b4986ab..109095480a 100644 --- a/pupil_src/shared_modules/methods.py +++ b/pupil_src/shared_modules/methods.py @@ -88,7 +88,90 @@ def set(self,vals): def get(self): return self.lX,self.lY,self.uX,self.uY,self.array_shape +import itertools +class Cache_List(list): + """Cache list is a list of False + [False,False,False] + with update() 'False' can be overwritten with a result (anything not 'False') + self.complete_ranges show ranges where the cache does not evaluate as 'False' using eval_fn + this allows to use ranges a a way of showing where no caching has happed (default) or whatever you do with eval_fn + Warning: a positve result cannot be overwritten by a negative one in this implementation + self.complete indicated that the cache list has no unknowns (False) + + + """ + + def __init__(self, init_list,eval_fn=lambda x: not x==False): + super(Cache_List, self).__init__(init_list) + self.eval_fn = eval_fn + self.complete_ranges = self.init_complete_ranges() + self._togo = self.count(False) + + + @property + def complete(self): + return self._togo == 0 + + @complete.setter + def complete(self, value): + raise ValueError("Read only") + + + def update(self,key,item): + if self[key] != False: + logger.waring("You are overwriting a precached result. Please report this BUG") + self[key] = item + self.complete_ranges = self.init_complete_ranges() + else: + #unvisited need to update ranges + self[key] = item + self.update_complete_ranges(key) + self._togo -= 1 + + + def init_complete_ranges(self): + l = self + i = -1 + ranges = [] + for t,g in itertools.groupby(l,self.eval_fn): + l = i + 1 + i += len(list(g)) + if t: + ranges.append([l,i]) + return ranges + + def merge_ranges(self): + l = self.complete_ranges + for i in range(len(l)-1): + if l[i][1] == l[i+1][0]-1: + #merge touching fields + l.append([l[i][0],l[i+1][1]]) + del l[i],l[i] + l.sort(key=lambda x: x[0]) + self.merge_ranges() + return + return + + def update_complete_ranges(self,i): + l = self.complete_ranges + for _range in l: + #most usual case extend a range + if i == _range[0]-1: + _range[0] = i + self.merge_ranges() + return + elif i == _range[1]+1: + _range[1] = i + self.merge_ranges() + return + #somewhere outside of range proximity + l.append([i,i]) + l.sort(key=lambda x: x[0]) + self.merge_ranges() + + def to_list(self): + return list(self) def bin_thresholding(image, image_lower=0, image_upper=256): binary_img = cv2.inRange(image, np.asarray(image_lower), diff --git a/pupil_src/shared_modules/reference_surface.py b/pupil_src/shared_modules/reference_surface.py index aebbe5d1a7..569942c540 100644 --- a/pupil_src/shared_modules/reference_surface.py +++ b/pupil_src/shared_modules/reference_surface.py @@ -15,12 +15,15 @@ from OpenGL.GL import * from OpenGL.GLU import gluOrtho2D -from methods import GetAnglesPolyline,normalize +from methods import GetAnglesPolyline,normalize,Cache_List #ctypes import for atb_vars: from ctypes import c_int,c_bool,create_string_buffer from time import time +import logging +logger = logging.getLogger(__name__) + def m_verts_to_screen(verts): #verts need to be sorted counterclockwise stating at bottom left mapped_space_one = np.array(((0,0),(1,0),(1,1),(0,1)),dtype=np.float32) @@ -69,7 +72,7 @@ def __init__(self,name="unnamed",saved_definition=None): self.detected = False self.m_to_screen = None self.m_from_screen = None - self.cache = None + self.cache = None if saved_definition is not None: self.load_from_dict(saved_definition) @@ -222,6 +225,10 @@ def locate(self, visible_markers): self.m_to_screen = None def answer_caching_request(self,visible_markers): + # cache point had not been visited + if visible_markers == False: + return False + # cache point had been visited marker_by_id = dict([(m['id'],m) for m in visible_markers]) visible_ids = set(marker_by_id.keys()) requested_ids = set(self.markers.keys()) @@ -239,8 +246,36 @@ def answer_caching_request(self,visible_markers): return {'m_to_screen':m_to_screen,'m_from_screen':m_from_screen,'detected_markers':len(overlap)} else: + #surface not found return None + + def update_cache(self,marker_cache): + ''' + compute surface m's and gaze points from cached marker data + entries are: + - False: when marker cache entry was False (not yet searched) + - None: when surface was not found + - {'m_to_screen':,'m_from_screen':,'detected_markers':} + ''' + + # iterations = 0 + if not self.defined: + return + if self.cache == None: + logger.debug("Full update of surface '%s' positons cache"%self.name) + self.cache = Cache_List([self.answer_caching_request(c_m) for c_m in marker_cache],eval_fn=lambda x: not (x==False or x==None)) + # iterations = len(self.cache) + else: + # update if markercache is not False but surface cache is still false + # this happens when the markercache was incomplete when this fn was run before + for i in range(len(marker_cache)): + if self.cache[i] == False and marker_cache[i] != False: + self.cache.update(i,self.answer_caching_request(marker_cache[i])) + # iterations +=1 + # return iterations + print self.cache.complete_ranges + def img_to_ref_surface(self,pos): if self.m_from_screen is not None: #convenience lines to allow 'simple' vectors (x,y) to be used @@ -279,6 +314,8 @@ def move_vertex(self,vert_idx,new_pos): for m in self.markers.values(): m.uv_coords = cv2.perspectiveTransform(m.uv_coords,transform) + self.cache = None #purge cache as it is not valid anymore + def atb_marker_status(self): return create_string_buffer("%s / %s" %(self.detected_markers,len(self.markers)),512) diff --git a/pupil_src/shared_modules/uvc_capture/file_capture.py b/pupil_src/shared_modules/uvc_capture/file_capture.py index 23fe146d01..f17a8fa18a 100644 --- a/pupil_src/shared_modules/uvc_capture/file_capture.py +++ b/pupil_src/shared_modules/uvc_capture/file_capture.py @@ -65,7 +65,7 @@ def __init__(self,src,timestamps=None): self.controls = None #No UVC controls available with file capture # we initialize the actual capture based on cv2.VideoCapture self.cap = cv2.VideoCapture(src) - if timestamps is None and src.endswith("eye.avi"): + if timestamps is None and src.endswith("eye.avi"): timestamps_loc = os.path.join(src.rsplit(os.path.sep,1)[0],'eye_timestamps.npy') logger.debug("trying to auto load eye_video timestamps with video at: %s"%timestamps_loc) else: From 325dbcd40304d0ccf7b2652b32992e513ea96a1c Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Fri, 2 May 2014 12:31:46 +0200 Subject: [PATCH 08/41] more work on offline marker display --- pupil_src/player/main.py | 2 +- pupil_src/shared_modules/marker_detector.py | 44 ++++++++++--------- pupil_src/shared_modules/methods.py | 16 +++---- pupil_src/shared_modules/reference_surface.py | 1 - 4 files changed, 33 insertions(+), 30 deletions(-) diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index bcbca41539..e72dc173db 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -462,7 +462,7 @@ def get_from_data(data): if __name__ == '__main__': freeze_support() - if 1: + if 0: main() else: import cProfile,subprocess,os diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index 22e6d5bab6..297164a53f 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -78,12 +78,16 @@ def __init__(self,g_pool,atb_pos=(320,220)): if g_pool.app == "player": #check if marker cache is available from last session self.persistent_cache = shelve.open(os.path.join(g_pool.rec_dir,'square_marker_cache'),protocol=2) - self.marker_cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps])) - logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.marker_cache)-self.marker_cache.count(False),len(self.marker_cache)) ) + self.cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps])) + logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.cache)-self.cache.count(False),len(self.cache)) ) self.init_marker_cacher() else: self.persistent_cache = None - self.marker_cache = None + self.cache = None + + if g_pool.app == 'player': + from surface_display import Surface_Display + self.g_pool.plugins.append(Surface_Display(self.g_pool,running_detector_plugin=self)) #debug vars self.draw_markers = c_bool(0) @@ -157,15 +161,15 @@ def update_bar_markers(self): self._bar_markers.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') self._bar_markers.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) self._bar_markers.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) - self._bar_markers.add_button("%s_update_cache"%i, s.update_cache,label='update cache',group=str(i),data=self.marker_cache) + self._bar_markers.add_button("%s_update_cache"%i, s.update_cache,label='update cache',group=str(i),data=self.cache) def update(self,frame,recent_pupil_positions,events): img = frame.img self.img_shape = frame.img.shape - if self.marker_cache is not None: + if self.cache is not None: self.update_marker_cache() - self.markers = self.marker_cache[frame.index] + self.markers = self.cache[frame.index] if self.markers == False: # locate markers because precacher has not anayzed this frame yet. Most likely a seek event self.markers = detect_markers_simple(img, @@ -238,29 +242,29 @@ def update(self,frame,recent_pupil_positions,events): def init_marker_cacher(self): forking_enable(0) #for MacOs only from marker_detector_cacher import fill_cache - visited_list = [False if x == False else True for x in self.marker_cache] + visited_list = [False if x == False else True for x in self.cache] video_file_path = os.path.join(self.g_pool.rec_dir,'world.avi') - self.marker_cache_queue = Queue() - self.marker_cacher_seek_idx = Value(c_int,0) - self.marker_cacher_run = Value(c_bool,True) - self.marker_cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.marker_cache_queue,self.marker_cacher_seek_idx,self.marker_cacher_run)) - self.marker_cacher.start() + self.cache_queue = Queue() + self.cacher_seek_idx = Value(c_int,0) + self.cacher_run = Value(c_bool,True) + self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.cache_queue,self.cacher_seek_idx,self.cacher_run)) + self.cacher.start() def update_marker_cache(self): - while not self.marker_cache_queue.empty(): - idx,c_m = self.marker_cache_queue.get() - self.marker_cache.update(idx,c_m) - # status = ['o' if x!=False else 'x' for x in self.marker_cache[::1000]] + while not self.cache_queue.empty(): + idx,c_m = self.cache_queue.get() + self.cache.update(idx,c_m) + # status = ['o' if x!=False else 'x' for x in self.cache[::1000]] # print "".join(status) def seek_marker_cacher(self,idx): - self.marker_cacher_seek_idx.value = idx + self.cacher_seek_idx.value = idx def close_marker_cacher(self): self.update_marker_cache() - self.marker_cacher_run.value = False - self.marker_cacher.join() + self.cacher_run.value = False + self.cacher.join() def gl_display(self): """ @@ -292,7 +296,7 @@ def cleanup(self): elif self.g_pool.app == 'player': self.save("offline_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) self.close_marker_cacher() - self.persistent_cache["marker_cache"] = self.marker_cache.to_list() + self.persistent_cache["marker_cache"] = self.cache.to_list() self.persistent_cache.close() self.surface_definitions.close() diff --git a/pupil_src/shared_modules/methods.py b/pupil_src/shared_modules/methods.py index 109095480a..300f46671a 100644 --- a/pupil_src/shared_modules/methods.py +++ b/pupil_src/shared_modules/methods.py @@ -94,7 +94,7 @@ class Cache_List(list): """Cache list is a list of False [False,False,False] with update() 'False' can be overwritten with a result (anything not 'False') - self.complete_ranges show ranges where the cache does not evaluate as 'False' using eval_fn + self.ranges show ranges where the cache does not evaluate as 'False' using eval_fn this allows to use ranges a a way of showing where no caching has happed (default) or whatever you do with eval_fn Warning: a positve result cannot be overwritten by a negative one in this implementation self.complete indicated that the cache list has no unknowns (False) @@ -105,7 +105,7 @@ class Cache_List(list): def __init__(self, init_list,eval_fn=lambda x: not x==False): super(Cache_List, self).__init__(init_list) self.eval_fn = eval_fn - self.complete_ranges = self.init_complete_ranges() + self.ranges = self.init_ranges() self._togo = self.count(False) @@ -122,15 +122,15 @@ def update(self,key,item): if self[key] != False: logger.waring("You are overwriting a precached result. Please report this BUG") self[key] = item - self.complete_ranges = self.init_complete_ranges() + self.ranges = self.init_ranges() else: #unvisited need to update ranges self[key] = item - self.update_complete_ranges(key) + self.update_ranges(key) self._togo -= 1 - def init_complete_ranges(self): + def init_ranges(self): l = self i = -1 ranges = [] @@ -142,7 +142,7 @@ def init_complete_ranges(self): return ranges def merge_ranges(self): - l = self.complete_ranges + l = self.ranges for i in range(len(l)-1): if l[i][1] == l[i+1][0]-1: #merge touching fields @@ -153,8 +153,8 @@ def merge_ranges(self): return return - def update_complete_ranges(self,i): - l = self.complete_ranges + def update_ranges(self,i): + l = self.ranges for _range in l: #most usual case extend a range if i == _range[0]-1: diff --git a/pupil_src/shared_modules/reference_surface.py b/pupil_src/shared_modules/reference_surface.py index 569942c540..cae6892bec 100644 --- a/pupil_src/shared_modules/reference_surface.py +++ b/pupil_src/shared_modules/reference_surface.py @@ -274,7 +274,6 @@ def update_cache(self,marker_cache): self.cache.update(i,self.answer_caching_request(marker_cache[i])) # iterations +=1 # return iterations - print self.cache.complete_ranges def img_to_ref_surface(self,pos): if self.m_from_screen is not None: From 8ea11f606ac8a8cc04b2c522fb5d70dca9708c02 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Sat, 3 May 2014 13:23:29 +0200 Subject: [PATCH 09/41] refactored code. Made version of marker tracker called offline marker tracker. --- pupil_src/player/main.py | 6 +- pupil_src/shared_modules/marker_detector.py | 24 ++-- pupil_src/shared_modules/methods.py | 41 +++--- pupil_src/shared_modules/reference_surface.py | 130 +++++++++++------- .../shared_modules/square_marker_detect.py | 2 +- 5 files changed, 122 insertions(+), 81 deletions(-) diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index e72dc173db..a82c776745 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -107,12 +107,12 @@ from seek_bar import Seek_Bar from export_launcher import Export_Launcher from scan_path import Scan_Path -from marker_detector import Marker_Detector +from offline_marker_detector import Offline_Marker_Detector from pupil_server import Pupil_Server from filter_fixations import Filter_Fixations from manual_gaze_correction import Manual_Gaze_Correction -plugin_by_index = (Vis_Circle,Vis_Cross, Vis_Polyline, Vis_Light_Points,Scan_Path,Filter_Fixations,Manual_Gaze_Correction,Marker_Detector,Pupil_Server) +plugin_by_index = (Vis_Circle,Vis_Cross, Vis_Polyline, Vis_Light_Points,Scan_Path,Filter_Fixations,Manual_Gaze_Correction,Offline_Marker_Detector,Pupil_Server) name_by_index = [p.__name__ for p in plugin_by_index] index_by_name = dict(zip(name_by_index,range(len(name_by_index)))) plugin_by_name = dict(zip(name_by_index,plugin_by_index)) @@ -167,7 +167,7 @@ def on_close(window): rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: - rec_dir = '/home/mkassner/Desktop/002' + rec_dir = '/home/mkassner/Desktop/000' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index 297164a53f..5a23a7cc47 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -172,12 +172,7 @@ def update(self,frame,recent_pupil_positions,events): self.markers = self.cache[frame.index] if self.markers == False: # locate markers because precacher has not anayzed this frame yet. Most likely a seek event - self.markers = detect_markers_simple(img, - grid_size = 5, - min_marker_perimeter=self.min_marker_perimeter, - aperture=self.aperture.value, - visualize=0) - + self.markers = [] self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here analyzed else: @@ -197,9 +192,7 @@ def update(self,frame,recent_pupil_positions,events): aperture=self.aperture.value, visualize=0) - if self.draw_markers.value: - draw_markers(img,self.markers) - + # locate surfaces for s in self.surfaces: @@ -207,6 +200,9 @@ def update(self,frame,recent_pupil_positions,events): if s.detected: events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) + if self.draw_markers.value: + draw_markers(img,self.markers) + # edit surfaces by user if self.surface_edit_mode: window = glfwGetCurrentContext() @@ -255,8 +251,14 @@ def update_marker_cache(self): while not self.cache_queue.empty(): idx,c_m = self.cache_queue.get() self.cache.update(idx,c_m) - # status = ['o' if x!=False else 'x' for x in self.cache[::1000]] - # print "".join(status) + for s in self.surfaces: + s.update_cache(self.cache,idx=idx) + + # update srf with no or invald cache: + for s in self.surfaces: + if s.cache == None: + s.init_cache(self.cache) + def seek_marker_cacher(self,idx): self.cacher_seek_idx.value = idx diff --git a/pupil_src/shared_modules/methods.py b/pupil_src/shared_modules/methods.py index 300f46671a..bf51cc85e3 100644 --- a/pupil_src/shared_modules/methods.py +++ b/pupil_src/shared_modules/methods.py @@ -97,36 +97,45 @@ class Cache_List(list): self.ranges show ranges where the cache does not evaluate as 'False' using eval_fn this allows to use ranges a a way of showing where no caching has happed (default) or whatever you do with eval_fn Warning: a positve result cannot be overwritten by a negative one in this implementation - self.complete indicated that the cache list has no unknowns (False) - - + self.complete indicated that the cache list has no unknowns eval_fn(n) == False """ - def __init__(self, init_list,eval_fn=lambda x: not x==False): + def __init__(self, init_list,eval_fn=lambda x: x!=False): super(Cache_List, self).__init__(init_list) self.eval_fn = eval_fn - self.ranges = self.init_ranges() + self._ranges = self.init_ranges() self._togo = self.count(False) + self.length = len(self) + @property + def ranges(self): + return self._ranges + + @ranges.setter + def ranges(self, value): + raise Exception("Read only") + @property def complete(self): return self._togo == 0 @complete.setter def complete(self, value): - raise ValueError("Read only") + raise Exception("Read only") + def update(self,key,item): if self[key] != False: - logger.waring("You are overwriting a precached result. Please report this BUG") + logger.waring("You are overwriting a precached result.") self[key] = item self.ranges = self.init_ranges() else: - #unvisited need to update ranges + #unvisited self[key] = item - self.update_ranges(key) + if self.eval_fn(item): #need to update ranges + self.update_ranges(key) self._togo -= 1 @@ -142,21 +151,20 @@ def init_ranges(self): return ranges def merge_ranges(self): - l = self.ranges + l = self._ranges for i in range(len(l)-1): if l[i][1] == l[i+1][0]-1: #merge touching fields - l.append([l[i][0],l[i+1][1]]) - del l[i],l[i] - l.sort(key=lambda x: x[0]) - self.merge_ranges() + l[i] = ([l[i][0],l[i+1][1]]) + #del second field + del l[i+1] return return def update_ranges(self,i): - l = self.ranges + l = self._ranges for _range in l: - #most usual case extend a range + #most common case: extend a range if i == _range[0]-1: _range[0] = i self.merge_ranges() @@ -168,7 +176,6 @@ def update_ranges(self,i): #somewhere outside of range proximity l.append([i,i]) l.sort(key=lambda x: x[0]) - self.merge_ranges() def to_list(self): return list(self) diff --git a/pupil_src/shared_modules/reference_surface.py b/pupil_src/shared_modules/reference_surface.py index cae6892bec..e5dfb4e5fc 100644 --- a/pupil_src/shared_modules/reference_surface.py +++ b/pupil_src/shared_modules/reference_surface.py @@ -224,56 +224,7 @@ def locate(self, visible_markers): self.m_from_screen = None self.m_to_screen = None - def answer_caching_request(self,visible_markers): - # cache point had not been visited - if visible_markers == False: - return False - # cache point had been visited - marker_by_id = dict([(m['id'],m) for m in visible_markers]) - visible_ids = set(marker_by_id.keys()) - requested_ids = set(self.markers.keys()) - overlap = visible_ids & requested_ids - detected_markers = len(overlap) - if len(overlap)>=min(2,len(requested_ids)): - yx = np.array( [marker_by_id[i]['verts_norm'] for i in overlap] ) - uv = np.array( [self.markers[i].uv_coords for i in overlap] ) - yx.shape=(-1,1,2) - uv.shape=(-1,1,2) - # print 'uv',uv - # print 'yx',yx - m_to_screen,mask = cv2.findHomography(uv,yx) - m_from_screen,mask = cv2.findHomography(yx,uv) - - return {'m_to_screen':m_to_screen,'m_from_screen':m_from_screen,'detected_markers':len(overlap)} - else: - #surface not found - return None - - def update_cache(self,marker_cache): - ''' - compute surface m's and gaze points from cached marker data - entries are: - - False: when marker cache entry was False (not yet searched) - - None: when surface was not found - - {'m_to_screen':,'m_from_screen':,'detected_markers':} - ''' - - # iterations = 0 - if not self.defined: - return - if self.cache == None: - logger.debug("Full update of surface '%s' positons cache"%self.name) - self.cache = Cache_List([self.answer_caching_request(c_m) for c_m in marker_cache],eval_fn=lambda x: not (x==False or x==None)) - # iterations = len(self.cache) - else: - # update if markercache is not False but surface cache is still false - # this happens when the markercache was incomplete when this fn was run before - for i in range(len(marker_cache)): - if self.cache[i] == False and marker_cache[i] != False: - self.cache.update(i,self.answer_caching_request(marker_cache[i])) - # iterations +=1 - # return iterations def img_to_ref_surface(self,pos): if self.m_from_screen is not None: @@ -456,6 +407,87 @@ def cleanup(self): self.close_window() + #cache fn for offline marker + def locate_from_cache(self,frame_idx): + if self.cache == None: + #no cache available cannot update from cache + return False + cache_result = self.cache[frame_idx] + if cache_result == False: + #cached data not avaible for this frame + return False + elif cache_result == None: + #cached data shows surface not found: + self.detected = False + self.m_from_screen = None + self.m_to_screen = None + return True + else: + self.detected = True + self.m_from_screen = cache_result['m_from_screen'] + self.m_to_screen = cache_result['m_to_screen'] + self.detected_markers = cache_result['detected_markers'] + return True + raise Exception("Invalid cache entry. Please report Bug.") + + + def update_cache(self,marker_cache,idx=None): + ''' + compute surface m's and gaze points from cached marker data + entries are: + - False: when marker cache entry was False (not yet searched) + - None: when surface was not found + - {'m_to_screen':,'m_from_screen':,'detected_markers':} + ''' + + # iterations = 0 + + if self.cache == None: + self.init_cache(marker_cache) + elif idx != None: + #update single data pt + self.cache.update(idx,self.answer_caching_request(marker_cache[idx])) + else: + # update where markercache is not False but surface cache is still false + # this happens when the markercache was incomplete when this fn was run before + for i in range(len(marker_cache)): + if self.cache[i] == False and marker_cache[i] != False: + self.cache.update(i,self.answer_caching_request(marker_cache[i])) + # iterations +=1 + # return iterations + + def init_cache(self,marker_cache): + if self.defined: + logger.debug("Full update of surface '%s' positons cache"%self.name) + self.cache = Cache_List([self.answer_caching_request(c_m) for c_m in marker_cache],eval_fn=lambda x: (x!=False) and (x!=None)) + + + def answer_caching_request(self,visible_markers): + # cache point had not been visited + if visible_markers == False: + return False + # cache point had been visited + marker_by_id = dict([(m['id'],m) for m in visible_markers]) + visible_ids = set(marker_by_id.keys()) + requested_ids = set(self.markers.keys()) + overlap = visible_ids & requested_ids + detected_markers = len(overlap) + if len(overlap)>=min(2,len(requested_ids)): + yx = np.array( [marker_by_id[i]['verts_norm'] for i in overlap] ) + uv = np.array( [self.markers[i].uv_coords for i in overlap] ) + yx.shape=(-1,1,2) + uv.shape=(-1,1,2) + # print 'uv',uv + # print 'yx',yx + m_to_screen,mask = cv2.findHomography(uv,yx) + m_from_screen,mask = cv2.findHomography(yx,uv) + + return {'m_to_screen':m_to_screen,'m_from_screen':m_from_screen,'detected_markers':len(overlap)} + else: + #surface not found + return None + + class Support_Marker(object): ''' This is a class only to be used by Reference_Surface diff --git a/pupil_src/shared_modules/square_marker_detect.py b/pupil_src/shared_modules/square_marker_detect.py index 78629d68d0..4a0d4e24bb 100644 --- a/pupil_src/shared_modules/square_marker_detect.py +++ b/pupil_src/shared_modules/square_marker_detect.py @@ -200,7 +200,7 @@ def draw_markers(img,markers): for m in markers: centroid = [m['verts'].sum(axis=0)/4.] origin = m['verts'][0] - hat = np.array([[[0,0],[0,1],[.5,1.5],[1,1],[1,0]]],dtype=np.float32) + hat = np.array([[[0,0],[0,1],[.5,1.25],[1,1],[1,0]]],dtype=np.float32) hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m)) cv2.polylines(img,np.int0(hat),color = (0,0,255),isClosed=True) cv2.polylines(img,np.int0(centroid),color = (255,255,0),isClosed=True,thickness=2) From ed972b7f6c30a0d123fcdc01d04bd68d288c3ec9 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Sat, 3 May 2014 13:31:12 +0200 Subject: [PATCH 10/41] adding offline marker detector --- pupil_src/shared_modules/marker_detector.py | 119 +------ .../shared_modules/offline_marker_detector.py | 329 ++++++++++++++++++ 2 files changed, 347 insertions(+), 101 deletions(-) create mode 100644 pupil_src/shared_modules/offline_marker_detector.py diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index 5a23a7cc47..9626318cb1 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -28,15 +28,6 @@ from reference_surface import Reference_Surface from math import sqrt -if platform.system() == 'Darwin': - from billiard import Process,Queue,forking_enable - from billiard.sharedctypes import Value -else: - from multiprocessing import Process, Pipe, Event, Queue - forking_enable = lambda x: x #dummy fn - from multiprocessing.sharedctypes import Value - - class Marker_Detector(Plugin): """docstring @@ -50,21 +41,9 @@ def __init__(self,g_pool,atb_pos=(320,220)): self.markers = [] # all registered surfaces - if g_pool.app == 'capture': - self.surface_definitions = shelve.open(os.path.join(g_pool.user_dir,'surface_definitions'),protocol=2) - self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] - elif g_pool.app == 'player': - #in player we load from the rec_dir: but we have a couple options: - self.surface_definitions = shelve.open(os.path.join(g_pool.rec_dir,'surface_definitions'),protocol=2) - if self.load('offline_square_marker_surfaces',[]) != []: - logger.debug("Found ref surfaces defined or copied in previous session.") - self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)] - elif self.load('offline_square_marker_surfaces',[]) != []: - logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.") - self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] - else: - logger.debug("No surface defs found. Please define using GUI.") - self.surfaces = [] + self.surface_definitions = shelve.open(os.path.join(g_pool.user_dir,'surface_definitions'),protocol=2) + self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] + # edit surfaces self.surface_edit_mode = c_bool(0) self.edit_surfaces = [] @@ -74,21 +53,6 @@ def __init__(self,g_pool,atb_pos=(320,220)): self.aperture = c_int(11) self.min_marker_perimeter = 80 - # caching vars when working with video file src - if g_pool.app == "player": - #check if marker cache is available from last session - self.persistent_cache = shelve.open(os.path.join(g_pool.rec_dir,'square_marker_cache'),protocol=2) - self.cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps])) - logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.cache)-self.cache.count(False),len(self.cache)) ) - self.init_marker_cacher() - else: - self.persistent_cache = None - self.cache = None - - if g_pool.app == 'player': - from surface_display import Surface_Display - self.g_pool.plugins.append(Surface_Display(self.g_pool,running_detector_plugin=self)) - #debug vars self.draw_markers = c_bool(0) self.show_surface_idx = c_int(0) @@ -167,33 +131,23 @@ def update(self,frame,recent_pupil_positions,events): img = frame.img self.img_shape = frame.img.shape - if self.cache is not None: - self.update_marker_cache() - self.markers = self.cache[frame.index] - if self.markers == False: - # locate markers because precacher has not anayzed this frame yet. Most likely a seek event - self.markers = [] - self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here analyzed - + + if self.robust_detection.value: + self.markers = detect_markers_robust(img, + grid_size = 5, + prev_markers=self.markers, + min_marker_perimeter=self.min_marker_perimeter, + aperture=self.aperture.value, + visualize=0, + true_detect_every_frame=3) else: - # locate markers during realtime, dont use robust detection for video file when seeks can throw thigs off. - if self.robust_detection.value: - self.markers = detect_markers_robust(img, - grid_size = 5, - prev_markers=self.markers, - min_marker_perimeter=self.min_marker_perimeter, - aperture=self.aperture.value, - visualize=0, - true_detect_every_frame=3) - else: - self.markers = detect_markers_simple(img, - grid_size = 5, - min_marker_perimeter=self.min_marker_perimeter, - aperture=self.aperture.value, - visualize=0) + self.markers = detect_markers_simple(img, + grid_size = 5, + min_marker_perimeter=self.min_marker_perimeter, + aperture=self.aperture.value, + visualize=0) - # locate surfaces for s in self.surfaces: s.locate(self.markers) @@ -235,38 +189,6 @@ def update(self,frame,recent_pupil_positions,events): s.open_window() - def init_marker_cacher(self): - forking_enable(0) #for MacOs only - from marker_detector_cacher import fill_cache - visited_list = [False if x == False else True for x in self.cache] - video_file_path = os.path.join(self.g_pool.rec_dir,'world.avi') - self.cache_queue = Queue() - self.cacher_seek_idx = Value(c_int,0) - self.cacher_run = Value(c_bool,True) - self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.cache_queue,self.cacher_seek_idx,self.cacher_run)) - self.cacher.start() - - - def update_marker_cache(self): - while not self.cache_queue.empty(): - idx,c_m = self.cache_queue.get() - self.cache.update(idx,c_m) - for s in self.surfaces: - s.update_cache(self.cache,idx=idx) - - # update srf with no or invald cache: - for s in self.surfaces: - if s.cache == None: - s.init_cache(self.cache) - - - def seek_marker_cacher(self,idx): - self.cacher_seek_idx.value = idx - - def close_marker_cacher(self): - self.update_marker_cache() - self.cacher_run.value = False - self.cacher.join() def gl_display(self): """ @@ -295,12 +217,7 @@ def cleanup(self): """ if self.g_pool.app == 'capture': self.save("realtime_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) - elif self.g_pool.app == 'player': - self.save("offline_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) - self.close_marker_cacher() - self.persistent_cache["marker_cache"] = self.cache.to_list() - self.persistent_cache.close() - + self.surface_definitions.close() for s in self.surfaces: diff --git a/pupil_src/shared_modules/offline_marker_detector.py b/pupil_src/shared_modules/offline_marker_detector.py new file mode 100644 index 0000000000..7571831310 --- /dev/null +++ b/pupil_src/shared_modules/offline_marker_detector.py @@ -0,0 +1,329 @@ +''' +(*)~---------------------------------------------------------------------------------- + Pupil - eye tracking platform + Copyright (C) 2012-2014 Pupil Labs + + Distributed under the terms of the CC BY-NC-SA License. + License details are in the file license.txt, distributed as part of this software. +----------------------------------------------------------------------------------~(*) +''' + +import sys, os,platform +import cv2 +import numpy as np +import shelve + + +if platform.system() == 'Darwin': + from billiard import Process,Queue,forking_enable + from billiard.sharedctypes import Value +else: + from multiprocessing import Process, Pipe, Event, Queue + forking_enable = lambda x: x #dummy fn + from multiprocessing.sharedctypes import Value + + + +from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture +from OpenGL.GL import * +from OpenGL.GLU import gluOrtho2D +from methods import normalize,denormalize,Cache_List +from glfw import * +import atb +from ctypes import c_int,c_bool,create_string_buffer + +from plugin import Plugin +#logging +import logging +logger = logging.getLogger(__name__) + +from square_marker_detect import detect_markers_robust,detect_markers_simple, draw_markers,m_marker_to_screen +from reference_surface import Reference_Surface +from math import sqrt + + +class Offline_Marker_Detector(Plugin): + """docstring + + """ + def __init__(self,g_pool,atb_pos=(320,220)): + super(Offline_Marker_Detector, self).__init__() + self.g_pool = g_pool + self.order = .2 + + # all markers that are detected in the most recent frame + self.markers = [] + # all registered surfaces + + if g_pool.app == 'capture': + raise Exception('For Player only.') + #in player we load from the rec_dir: but we have a couple options: + self.surface_definitions = shelve.open(os.path.join(g_pool.rec_dir,'surface_definitions'),protocol=2) + if self.load('offline_square_marker_surfaces',[]) != []: + logger.debug("Found ref surfaces defined or copied in previous session.") + self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)] + elif self.load('offline_square_marker_surfaces',[]) != []: + logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.") + self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] + else: + logger.debug("No surface defs found. Please define using GUI.") + self.surfaces = [] + + # edit surfaces + self.surface_edit_mode = c_bool(0) + self.edit_surfaces = [] + + #detector vars + self.robust_detection = c_bool(1) + self.aperture = c_int(11) + self.min_marker_perimeter = 80 + + + #check if marker cache is available from last session + self.persistent_cache = shelve.open(os.path.join(g_pool.rec_dir,'square_marker_cache'),protocol=2) + self.cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps])) + logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.cache)-self.cache.count(False),len(self.cache)) ) + self.init_marker_cacher() + + #debug vars + self.draw_markers = c_bool(0) + self.show_surface_idx = c_int(0) + self.recent_pupil_positions = [] + + self.img_shape = None + + atb_label = "marker detection" + self._bar = atb.Bar(name =self.__class__.__name__, label=atb_label, + help="marker detection parameters", color=(50, 50, 50), alpha=100, + text='light', position=atb_pos,refresh=.3, size=(300, 80)) + self._bar.add_var("draw markers",self.draw_markers) + self._bar.add_button('close',self.unset_alive) + + + atb_pos = atb_pos[0],atb_pos[1]+90 + self._bar_markers = atb.Bar(name =self.__class__.__name__+'markers', label='registered surfaces', + help="list of registered ref surfaces", color=(50, 100, 50), alpha=100, + text='light', position=atb_pos,refresh=.3, size=(300, 120)) + self.update_bar_markers() + + + + def unset_alive(self): + self.alive = False + + def load(self, var_name, default): + return self.surface_definitions.get(var_name,default) + def save(self, var_name, var): + self.surface_definitions[var_name] = var + + + def on_click(self,pos,button,action): + if self.surface_edit_mode.value: + if self.edit_surfaces: + if action == GLFW_RELEASE: + self.edit_surfaces = [] + # no surfaces verts in edit mode, lets see if the curser is close to one: + else: + if action == GLFW_PRESS: + surf_verts = ((0.,0.),(1.,0.),(1.,1.),(0.,1.)) + x,y = pos + for s in self.surfaces: + if s.detected: + for (vx,vy),i in zip(s.ref_surface_to_img(np.array(surf_verts)),range(4)): + vx,vy = denormalize((vx,vy),(self.img_shape[1],self.img_shape[0]),flip_y=True) + if sqrt((x-vx)**2 + (y-vy)**2) <15: #img pixels + self.edit_surfaces.append((s,i)) + + def advance(self): + pass + + def add_surface(self): + self.surfaces.append(Reference_Surface()) + self.update_bar_markers() + + def remove_surface(self,i): + self.surfaces[i].cleanup() + del self.surfaces[i] + self.update_bar_markers() + + + def update_bar_markers(self): + self._bar_markers.clear() + self._bar_markers.add_button(" add surface ", self.add_surface, key='a') + self._bar_markers.add_var(" edit mode ", self.surface_edit_mode ) + for s,i in zip(self.surfaces,range(len(self.surfaces)))[::-1]: + self._bar_markers.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window') + self._bar_markers.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') + self._bar_markers.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) + self._bar_markers.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) + + def update(self,frame,recent_pupil_positions,events): + self.img_shape = frame.img.shape + self.update_marker_cache() + self.markers = self.cache[frame.index] + if self.markers == False: + # locate markers because precacher has not anayzed this frame yet. Most likely a seek event + self.markers = [] + self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here analyzed + return + + + # locate surfaces + for s in self.surfaces: + if not s.locate_from_cache(frame.index): + # logger.debug("location not from cache") + s.locate(self.markers) + if s.detected: + events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) + + if self.draw_markers.value: + draw_markers(frame.img,self.markers) + + # edit surfaces by user + if self.surface_edit_mode: + window = glfwGetCurrentContext() + pos = glfwGetCursorPos(window) + pos = normalize(pos,glfwGetWindowSize(window)) + pos = denormalize(pos,(frame.img.shape[1],frame.img.shape[0]) ) # Position in img pixels + + for s,v_idx in self.edit_surfaces: + if s.detected: + pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True) + new_pos = s.img_to_ref_surface(np.array(pos)) + s.move_vertex(v_idx,new_pos) + else: + # update srf with no or invald cache: + for s in self.surfaces: + if s.cache == None: + s.init_cache(self.cache) + + #map recent gaze onto detected surfaces used for pupil server + for s in self.surfaces: + if s.detected: + s.recent_gaze = [] + for p in recent_pupil_positions: + if p['norm_pupil'] is not None: + gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_gaze']))) + p['realtime gaze on '+s.name] = gp_on_s + s.recent_gaze.append(gp_on_s) + + + #allow surfaces to open/close windows + for s in self.surfaces: + if s.window_should_close: + s.close_window() + if s.window_should_open: + s.open_window() + + def init_marker_cacher(self): + forking_enable(0) #for MacOs only + from marker_detector_cacher import fill_cache + visited_list = [False if x == False else True for x in self.cache] + video_file_path = os.path.join(self.g_pool.rec_dir,'world.avi') + self.cache_queue = Queue() + self.cacher_seek_idx = Value(c_int,0) + self.cacher_run = Value(c_bool,True) + self.cacher = Process(target=fill_cache, args=(visited_list,video_file_path,self.cache_queue,self.cacher_seek_idx,self.cacher_run)) + self.cacher.start() + + def update_marker_cache(self): + while not self.cache_queue.empty(): + idx,c_m = self.cache_queue.get() + self.cache.update(idx,c_m) + for s in self.surfaces: + s.update_cache(self.cache,idx=idx) + + def seek_marker_cacher(self,idx): + self.cacher_seek_idx.value = idx + + def close_marker_cacher(self): + self.update_marker_cache() + self.cacher_run.value = False + self.cacher.join() + + def gl_display(self): + """ + Display marker and surface info inside world screen + """ + self.gl_display_cache_bars() + + for m in self.markers: + hat = np.array([[[0,0],[0,1],[.5,1.3],[1,1],[1,0],[0,0]]],dtype=np.float32) + hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m)) + draw_gl_polyline(hat.reshape((6,2)),(0.1,1.,1.,.5)) + + for s in self.surfaces: + s.gl_draw_frame() + s.gl_display_in_window(self.g_pool.image_tex) + + if self.surface_edit_mode.value: + for s in self.surfaces: + s.gl_draw_corners() + + + def gl_display_cache_bars(self): + """ + """ + padding = 20. + + # Lines for areas that have be cached + cached_ranges = [] + for r in self.cache.ranges: # [[0,1],[3,4]] + cached_ranges += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)] + + # Lines where surfaces have been found in video + cached_surfaces = [] + for s in self.surfaces: + found_at = [] + if s.cache is not None: + for r in s.cache.ranges: # [[0,1],[3,4]] + found_at += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)] + cached_surfaces.append(found_at) + + glMatrixMode(GL_PROJECTION) + glPushMatrix() + glLoadIdentity() + width,height = glfwGetWindowSize(glfwGetCurrentContext()) + h_pad = padding/width * self.cache.length + v_pad = padding/height + gluOrtho2D(-h_pad, self.cache.length+h_pad, -v_pad, 1+v_pad) # ranging from 0 to cache_len (horizontal) and 0 to 1 (vertical) + glMatrixMode(GL_MODELVIEW) + glPushMatrix() + glLoadIdentity() + + color = (1.,.4,.5,8.) + draw_gl_polyline(cached_ranges,color=color,type='Lines') + + for s in cached_surfaces: + glTranslatef(0,.02,0) + draw_gl_polyline(s,color=color,type='Lines') + + glMatrixMode(GL_PROJECTION) + glPopMatrix() + glMatrixMode(GL_MODELVIEW) + glPopMatrix() + + + + + + def cleanup(self): + """ called when the plugin gets terminated. + This happends either voluntary or forced. + if you have an atb bar or glfw window destroy it here. + """ + + self.save("offline_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) + self.close_marker_cacher() + self.persistent_cache["marker_cache"] = self.cache.to_list() + self.persistent_cache.close() + + self.surface_definitions.close() + + for s in self.surfaces: + s.close_window() + self._bar.destroy() + self._bar_markers.destroy() + + + From 4250551a47e04009fea238615c23055fc9c2a5de Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Sat, 3 May 2014 14:00:53 +0200 Subject: [PATCH 11/41] typo fix. --- pupil_src/shared_modules/methods.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pupil_src/shared_modules/methods.py b/pupil_src/shared_modules/methods.py index bf51cc85e3..4107adc87c 100644 --- a/pupil_src/shared_modules/methods.py +++ b/pupil_src/shared_modules/methods.py @@ -125,14 +125,13 @@ def complete(self, value): raise Exception("Read only") - def update(self,key,item): if self[key] != False: - logger.waring("You are overwriting a precached result.") + logger.warning("You are overwriting a precached result.") self[key] = item - self.ranges = self.init_ranges() + self._ranges = self.init_ranges() else: - #unvisited + #unvisited self[key] = item if self.eval_fn(item): #need to update ranges self.update_ranges(key) From b1be7f987ffdf7e650f478fab871118e9f4214eb Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Mon, 5 May 2014 10:59:51 +0200 Subject: [PATCH 12/41] small fix --- pupil_src/player/main.py | 2 +- pupil_src/shared_modules/marker_detector.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index a82c776745..9b6ffbd229 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -167,7 +167,7 @@ def on_close(window): rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: - rec_dir = '/home/mkassner/Desktop/000' + rec_dir = '/Users/mkassner/Downloads/ET-lottery-ticket/4044/001' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index 9626318cb1..191c980da1 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -14,7 +14,7 @@ import shelve from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture -from methods import normalize,denormalize,Cache_List +from methods import normalize,denormalize from glfw import * import atb from ctypes import c_int,c_bool,create_string_buffer @@ -125,13 +125,12 @@ def update_bar_markers(self): self._bar_markers.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') self._bar_markers.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) self._bar_markers.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) - self._bar_markers.add_button("%s_update_cache"%i, s.update_cache,label='update cache',group=str(i),data=self.cache) def update(self,frame,recent_pupil_positions,events): img = frame.img self.img_shape = frame.img.shape - + if self.robust_detection.value: self.markers = detect_markers_robust(img, grid_size = 5, @@ -147,7 +146,7 @@ def update(self,frame,recent_pupil_positions,events): aperture=self.aperture.value, visualize=0) - + # locate surfaces for s in self.surfaces: s.locate(self.markers) @@ -217,7 +216,7 @@ def cleanup(self): """ if self.g_pool.app == 'capture': self.save("realtime_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) - + self.surface_definitions.close() for s in self.surfaces: From 91076a63386a37fc9cfbbd0e6799b2f31451e9d0 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Mon, 5 May 2014 11:08:40 +0200 Subject: [PATCH 13/41] trying to fix frame seeker --- .../uvc_capture/file_capture.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/pupil_src/shared_modules/uvc_capture/file_capture.py b/pupil_src/shared_modules/uvc_capture/file_capture.py index f17a8fa18a..325793f93b 100644 --- a/pupil_src/shared_modules/uvc_capture/file_capture.py +++ b/pupil_src/shared_modules/uvc_capture/file_capture.py @@ -122,15 +122,21 @@ def get_frame(self): def seek_to_frame(self, seek_pos): if self.cap.set(cv2.cv.CV_CAP_PROP_POS_FRAMES,seek_pos): offset = seek_pos - self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) - if offset == 0: - logger.debug("seeking to frame: %s"%self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)) + if offset == 0: + logger.debug("Seeked to frame: %s"%self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)) + return True + elif 0 < offset < 100: + offset +=10 + if not self.seek_to_frame(offset): + logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) + return False + logger.warning("Seek was not precice need to do manual seek for %s frames"%offset) + while seek_pos != self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES): + try: + self.read() + except EndofVideoFileError: + logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) return True - # elif 0 < offset < 100: - # logger("Seek was not precice need to do manual seek for %s frames"%offset) - # while seek_pos != self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES): - # self.read() - # logger.debug("seeking to frame: %s"%self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)) - # return True else: logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) return False From 89d6f57a06cdec2824921412f1125bcd464bd05b Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Mon, 5 May 2014 11:10:08 +0200 Subject: [PATCH 14/41] small fix --- pupil_src/shared_modules/uvc_capture/file_capture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/uvc_capture/file_capture.py b/pupil_src/shared_modules/uvc_capture/file_capture.py index 325793f93b..eca9bd13e9 100644 --- a/pupil_src/shared_modules/uvc_capture/file_capture.py +++ b/pupil_src/shared_modules/uvc_capture/file_capture.py @@ -127,7 +127,7 @@ def seek_to_frame(self, seek_pos): return True elif 0 < offset < 100: offset +=10 - if not self.seek_to_frame(offset): + if not self.seek_to_frame(seek_pos-offset): logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) return False logger.warning("Seek was not precice need to do manual seek for %s frames"%offset) From e795089651139188c9a4000af78ae02e05263f1a Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Mon, 5 May 2014 12:48:27 +0200 Subject: [PATCH 15/41] made file seeking more robust. --- pupil_src/player/seek_bar.py | 14 ++-- .../shared_modules/marker_detector_cacher.py | 65 +++++++++++-------- .../uvc_capture/file_capture.py | 37 ++++++----- 3 files changed, 70 insertions(+), 46 deletions(-) diff --git a/pupil_src/player/seek_bar.py b/pupil_src/player/seek_bar.py index 145290f1eb..bbb09047f7 100644 --- a/pupil_src/player/seek_bar.py +++ b/pupil_src/player/seek_bar.py @@ -48,8 +48,11 @@ def update(self,frame,recent_pupil_positions,events): norm_seek_pos, _ = self.screen_to_seek_bar(pos) norm_seek_pos = min(1,max(0,norm_seek_pos)) if abs(norm_seek_pos-self.norm_seek_pos) >=.01: - seek_pos = min(int(norm_seek_pos*self.frame_count),self.frame_count-1) - self.cap.seek_to_frame(seek_pos) + seek_pos = min(int(norm_seek_pos*self.frame_count),self.frame_count-5) #the last frames can be problematic to seek to + try: + self.cap.seek_to_frame(seek_pos) + except: + pass self.g_pool.new_seek = True @@ -71,8 +74,11 @@ def on_click(self,img_pos,button,action): if self.drag_mode: norm_seek_pos, _ = self.screen_to_seek_bar(pos) norm_seek_pos = min(1,max(0,norm_seek_pos)) - seek_pos = int(norm_seek_pos*self.frame_count) - self.cap.seek_to_frame(seek_pos) + seek_pos = min(int(norm_seek_pos*self.frame_count),self.frame_count-5) + try: + self.cap.seek_to_frame(seek_pos) + except: + pass self.g_pool.new_seek = True self.drag_mode=False self.g_pool.play = self.was_playing diff --git a/pupil_src/shared_modules/marker_detector_cacher.py b/pupil_src/shared_modules/marker_detector_cacher.py index 3d2bbc6dd7..0cc6889c43 100644 --- a/pupil_src/shared_modules/marker_detector_cacher.py +++ b/pupil_src/shared_modules/marker_detector_cacher.py @@ -11,7 +11,7 @@ -def fill_cache(visited_list,video_file_path,q,seek_idx,run): +def fill_cache(visited_list,video_file_path,q,seek_idx,run): ''' this function is part of marker_detector it is run as a seperate process. it must be kept in a seperate file for namespace sanatisation @@ -51,13 +51,48 @@ def next_unvisited_idx(frame_idx): next_unvisited = None return next_unvisited + def handle_frame(next): + if next != cap.get_frame_index(): + #we need to seek: + logger.debug("Seeking to Frame %s" %next) + try: + cap.seek_to_frame(next) + except: + #could not seek to requested position + logger.warning("Could not evaluate frame: %s."%next) + visited_list[next] = True # this frame is now visited. + q.put((next,[])) # we cannot look at the frame, report no detection + return + #seeking invalidates prev markers for the detector + markers = [] + + try: + frame = cap.get_frame() + except EndofVideoFileError: + logger.debug('Reached end of video. Rewinding.') + try: + cap.seek_to_frame(0) + except: + pass + return + + markers = detect_markers_robust(frame.img, + grid_size = 5, + prev_markers=markers, + min_marker_perimeter=min_marker_perimeter, + aperture=aperture, + visualize=0, + true_detect_every_frame=1) + visited_list[frame.index] = True + # logger.debug("adding Frame %s"%frame.index) + q.put((frame.index,markers)) while run.value: next = cap.get_frame_index() if seek_idx.value != -1: next = seek_idx.value seek_idx.value = -1 - logger.debug("User seek require marker caching at %s"%next) + logger.debug("User required seek. Marker caching at Frame: %s"%next) #check the visited list @@ -65,33 +100,9 @@ def next_unvisited_idx(frame_idx): if next == None: #we are done here: break - elif next != cap.get_frame_index(): - #we need to seek: - logger.debug("Seeking to Frame %s" %next) - cap.seek_to_frame(next) - markers = [] else: - #next frame is unvisited - pass + handle_frame(next) - try: - frame = cap.get_frame() - except EndofVideoFileError: - logger.debug('Reached end of video. Rewinding.') - frame = None - cap.seek_to_frame(0) - markers = [] - else: - markers = detect_markers_robust(frame.img, - grid_size = 5, - prev_markers=markers, - min_marker_perimeter=min_marker_perimeter, - aperture=aperture, - visualize=0, - true_detect_every_frame=1) - visited_list[frame.index] = True - # logger.debug("adding Frame %s"%frame.index) - q.put((frame.index,markers)) logger.debug("Closing Cacher Process") cap.close() diff --git a/pupil_src/shared_modules/uvc_capture/file_capture.py b/pupil_src/shared_modules/uvc_capture/file_capture.py index eca9bd13e9..c3d053c9e8 100644 --- a/pupil_src/shared_modules/uvc_capture/file_capture.py +++ b/pupil_src/shared_modules/uvc_capture/file_capture.py @@ -44,6 +44,12 @@ def __init__(self, arg): self.arg = arg +class FileSeekError(Exception): + """docstring for EndofVideoFileError""" + def __init__(self): + super(FileSeekError, self).__init__() + + class Frame(object): """docstring of Frame""" def __init__(self, timestamp,img,index=None,compressed_img=None, compressed_pix_fmt=None): @@ -124,24 +130,25 @@ def seek_to_frame(self, seek_pos): offset = seek_pos - self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES) if offset == 0: logger.debug("Seeked to frame: %s"%self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)) - return True - elif 0 < offset < 100: - offset +=10 - if not self.seek_to_frame(seek_pos-offset): - logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) - return False - logger.warning("Seek was not precice need to do manual seek for %s frames"%offset) - while seek_pos != self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES): - try: - self.read() - except EndofVideoFileError: - logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) - return True + return + # elif 0 < offset < 100: + # offset +=10 + # if not self.seek_to_frame(seek_pos-offset): + # logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) + # return False + # logger.warning("Seek was not precice need to do manual seek for %s frames"%offset) + # while seek_pos != self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES): + # try: + # self.read() + # except EndofVideoFileError: + # logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) + # return True else: logger.warning('Could not seek to %s. Seeked to %s'%(seek_pos,self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES))) - return False + raise FileSeekError() logger.error("Could not perform seek on cv2.VideoCapture. Command gave negative return.") - return False + raise FileSeekError() + def get_now(self): idx = int(self.cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)) From a77832d798a74459347085a2117d2007d0ad3b9c Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Mon, 5 May 2014 12:55:53 +0200 Subject: [PATCH 16/41] small fix --- pupil_src/shared_modules/marker_detector_cacher.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pupil_src/shared_modules/marker_detector_cacher.py b/pupil_src/shared_modules/marker_detector_cacher.py index 0cc6889c43..3d9561163f 100644 --- a/pupil_src/shared_modules/marker_detector_cacher.py +++ b/pupil_src/shared_modules/marker_detector_cacher.py @@ -52,6 +52,7 @@ def next_unvisited_idx(frame_idx): return next_unvisited def handle_frame(next): + global markers if next != cap.get_frame_index(): #we need to seek: logger.debug("Seeking to Frame %s" %next) From 8d9bdf8faf27eb2d610d560de586e8ccdc83403b Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Mon, 5 May 2014 13:20:13 +0200 Subject: [PATCH 17/41] more robustness improvement for marker tracker cacher --- .../shared_modules/marker_detector_cacher.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pupil_src/shared_modules/marker_detector_cacher.py b/pupil_src/shared_modules/marker_detector_cacher.py index 3d9561163f..77cf27f1c4 100644 --- a/pupil_src/shared_modules/marker_detector_cacher.py +++ b/pupil_src/shared_modules/marker_detector_cacher.py @@ -21,7 +21,7 @@ def fill_cache(visited_list,video_file_path,q,seek_idx,run): logger = logging.getLogger(__name__+' with pid: '+str(os.getpid()) ) logger.debug('Started cacher process for Marker Detector') import cv2 - from uvc_capture import autoCreateCapture, EndofVideoFileError + from uvc_capture import autoCreateCapture, EndofVideoFileError,FileSeekError from square_marker_detect import detect_markers_robust min_marker_perimeter = 80 aperture = 11 @@ -58,7 +58,7 @@ def handle_frame(next): logger.debug("Seeking to Frame %s" %next) try: cap.seek_to_frame(next) - except: + except FileSeekError: #could not seek to requested position logger.warning("Could not evaluate frame: %s."%next) visited_list[next] = True # this frame is now visited. @@ -70,11 +70,11 @@ def handle_frame(next): try: frame = cap.get_frame() except EndofVideoFileError: - logger.debug('Reached end of video. Rewinding.') - try: - cap.seek_to_frame(0) - except: - pass + logger.debug("Video File's last frame(s) not accesible") + #could not read frame + logger.warning("Could not evaluate frame: %s."%next) + visited_list[next] = True # this frame is now visited. + q.put((next,[])) # we cannot look at the frame, report no detection return markers = detect_markers_robust(frame.img, @@ -85,7 +85,6 @@ def handle_frame(next): visualize=0, true_detect_every_frame=1) visited_list[frame.index] = True - # logger.debug("adding Frame %s"%frame.index) q.put((frame.index,markers)) while run.value: From d4afd743e7d97afac9ee4f83d7ec650fbfb03d6e Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Mon, 5 May 2014 13:22:55 +0200 Subject: [PATCH 18/41] added missing import. --- pupil_src/shared_modules/uvc_capture/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/uvc_capture/__init__.py b/pupil_src/shared_modules/uvc_capture/__init__.py index 94a2de455e..79a6ae8fc2 100644 --- a/pupil_src/shared_modules/uvc_capture/__init__.py +++ b/pupil_src/shared_modules/uvc_capture/__init__.py @@ -43,7 +43,7 @@ from other_video import Camera_Capture,Camera_List,CameraCaptureError from fake_capture import FakeCapture -from file_capture import File_Capture, FileCaptureError, EndofVideoFileError +from file_capture import File_Capture, FileCaptureError, EndofVideoFileError,FileSeekError def autoCreateCapture(src,size=(640,480),fps=30,timestamps=None,timebase = None): From 8829b6cd2976702570cebf660438964b8ff52658 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Mon, 5 May 2014 15:08:13 +0200 Subject: [PATCH 19/41] various fixes --- pupil_src/player/main.py | 14 ++++++++++---- pupil_src/shared_modules/marker_detector_cacher.py | 5 ++--- .../shared_modules/offline_marker_detector.py | 14 ++++++++------ 3 files changed, 20 insertions(+), 13 deletions(-) diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index 9b6ffbd229..0389919269 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -81,7 +81,7 @@ from glfw import * import atb -from uvc_capture import autoCreateCapture,EndofVideoFileError,FakeCapture +from uvc_capture import autoCreateCapture,EndofVideoFileError,FileSeekError,FakeCapture # helpers/utils from methods import normalize, denormalize,Temp @@ -167,7 +167,7 @@ def on_close(window): rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: - rec_dir = '/Users/mkassner/Downloads/ET-lottery-ticket/4044/001' + rec_dir = '/home/mkassner/Desktop/002' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: @@ -281,11 +281,17 @@ def set_play(value): g.play = value def next_frame(): - cap.seek_to_frame(cap.get_frame_index()) + try: + cap.seek_to_frame(cap.get_frame_index()) + except FileSeekError: + pass g.new_seek = True def prev_frame(): - cap.seek_to_frame(cap.get_frame_index()-2) + try: + cap.seek_to_frame(cap.get_frame_index()-2) + except FileSeekError: + pass g.new_seek = True diff --git a/pupil_src/shared_modules/marker_detector_cacher.py b/pupil_src/shared_modules/marker_detector_cacher.py index 77cf27f1c4..007ffe277e 100644 --- a/pupil_src/shared_modules/marker_detector_cacher.py +++ b/pupil_src/shared_modules/marker_detector_cacher.py @@ -52,7 +52,6 @@ def next_unvisited_idx(frame_idx): return next_unvisited def handle_frame(next): - global markers if next != cap.get_frame_index(): #we need to seek: logger.debug("Seeking to Frame %s" %next) @@ -65,7 +64,7 @@ def handle_frame(next): q.put((next,[])) # we cannot look at the frame, report no detection return #seeking invalidates prev markers for the detector - markers = [] + markers[:] = [] try: frame = cap.get_frame() @@ -77,7 +76,7 @@ def handle_frame(next): q.put((next,[])) # we cannot look at the frame, report no detection return - markers = detect_markers_robust(frame.img, + markers[:] = detect_markers_robust(frame.img, grid_size = 5, prev_markers=markers, min_marker_perimeter=min_marker_perimeter, diff --git a/pupil_src/shared_modules/offline_marker_detector.py b/pupil_src/shared_modules/offline_marker_detector.py index 7571831310..f0a240858b 100644 --- a/pupil_src/shared_modules/offline_marker_detector.py +++ b/pupil_src/shared_modules/offline_marker_detector.py @@ -284,19 +284,21 @@ def gl_display_cache_bars(self): glPushMatrix() glLoadIdentity() width,height = glfwGetWindowSize(glfwGetCurrentContext()) - h_pad = padding/width * self.cache.length - v_pad = padding/height - gluOrtho2D(-h_pad, self.cache.length+h_pad, -v_pad, 1+v_pad) # ranging from 0 to cache_len (horizontal) and 0 to 1 (vertical) + h_pad = padding * (self.cache.length-2)/float(width) + v_pad = padding* 1./(height-2) + gluOrtho2D(-h_pad, (self.cache.length-1)+h_pad, -v_pad, 1+v_pad) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() - color = (1.,.4,.5,8.) - draw_gl_polyline(cached_ranges,color=color,type='Lines') + color = (8.,.6,.2,8.) + draw_gl_polyline(cached_ranges,color=color,type='Lines',thickness=4) + + color = (0.,.7,.3,8.) for s in cached_surfaces: glTranslatef(0,.02,0) - draw_gl_polyline(s,color=color,type='Lines') + draw_gl_polyline(s,color=color,type='Lines',thickness=2) glMatrixMode(GL_PROJECTION) glPopMatrix() From 5132c37b30354183e14033047e856b6bee550b80 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Mon, 5 May 2014 15:10:38 +0200 Subject: [PATCH 20/41] adding theckness option to lines --- pupil_src/shared_modules/gl_utils/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pupil_src/shared_modules/gl_utils/utils.py b/pupil_src/shared_modules/gl_utils/utils.py index 3765d4026e..b2af7db7d6 100644 --- a/pupil_src/shared_modules/gl_utils/utils.py +++ b/pupil_src/shared_modules/gl_utils/utils.py @@ -85,7 +85,8 @@ def adjust_gl_view(w,h,window): # glMatrixMode(GL_MODELVIEW) # glLoadIdentity() -def draw_gl_polyline((positions),color,type='Loop'): +def draw_gl_polyline((positions),color,type='Loop',thickness=1): + glLineWidth(thickness) glColor4f(*color) if type=='Loop': glBegin(GL_LINE_LOOP) @@ -99,7 +100,7 @@ def draw_gl_polyline((positions),color,type='Loop'): glVertex3f(x,y,0.0) glEnd() -def draw_gl_polyline_norm((positions),color,type='Loop'): +def draw_gl_polyline_norm((positions),color,type='Loop',thickness=1): glMatrixMode(GL_PROJECTION) glPushMatrix() @@ -108,7 +109,7 @@ def draw_gl_polyline_norm((positions),color,type='Loop'): glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() - draw_gl_polyline(positions,color,type) + draw_gl_polyline(positions,color,type,thickness) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) From 09abac105c219b77486066d8afdf10e7a938c4ea Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Tue, 6 May 2014 15:04:04 +0200 Subject: [PATCH 21/41] plugin not support persisten gui settings. --- .../shared_modules/offline_marker_detector.py | 64 ++++++++++--------- 1 file changed, 35 insertions(+), 29 deletions(-) diff --git a/pupil_src/shared_modules/offline_marker_detector.py b/pupil_src/shared_modules/offline_marker_detector.py index f0a240858b..48788e27f6 100644 --- a/pupil_src/shared_modules/offline_marker_detector.py +++ b/pupil_src/shared_modules/offline_marker_detector.py @@ -23,7 +23,6 @@ from multiprocessing.sharedctypes import Value - from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture from OpenGL.GL import * from OpenGL.GLU import gluOrtho2D @@ -43,12 +42,19 @@ class Offline_Marker_Detector(Plugin): - """docstring - """ - def __init__(self,g_pool,atb_pos=(320,220)): + Special version of marker detector for use with videofile source. + It uses a seperate process to search all frames in the world.avi file for markers. + - self.cache is a list containing marker positions for each frame. + - self.surfaces[i].cache is a list containing surface positions for each frame + Both caches are build up over time. The marker cache is also session persistent. + See marker_tracker.py for more info on this marker tracker. + """ + + def __init__(self,g_pool,gui_settings={'pos':(220,200),'size':(300,300),'iconified':False}): super(Offline_Marker_Detector, self).__init__() self.g_pool = g_pool + self.gui_settings = gui_settings self.order = .2 # all markers that are detected in the most recent frame @@ -78,7 +84,6 @@ def __init__(self,g_pool,atb_pos=(320,220)): self.aperture = c_int(11) self.min_marker_perimeter = 80 - #check if marker cache is available from last session self.persistent_cache = shelve.open(os.path.join(g_pool.rec_dir,'square_marker_cache'),protocol=2) self.cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps])) @@ -92,22 +97,18 @@ def __init__(self,g_pool,atb_pos=(320,220)): self.img_shape = None - atb_label = "marker detection" - self._bar = atb.Bar(name =self.__class__.__name__, label=atb_label, - help="marker detection parameters", color=(50, 50, 50), alpha=100, - text='light', position=atb_pos,refresh=.3, size=(300, 80)) - self._bar.add_var("draw markers",self.draw_markers) - self._bar.add_button('close',self.unset_alive) - - atb_pos = atb_pos[0],atb_pos[1]+90 - self._bar_markers = atb.Bar(name =self.__class__.__name__+'markers', label='registered surfaces', - help="list of registered ref surfaces", color=(50, 100, 50), alpha=100, - text='light', position=atb_pos,refresh=.3, size=(300, 120)) + def init_gui(self): + import atb + pos = self.gui_settings['pos'] + atb_label = "Marker Detector" + self._bar = atb.Bar(name =self.__class__.__name__+str(id(self)), label=atb_label, + help="circle", color=(150, 150, 50), alpha=50, + text='light', position=pos,refresh=.1, size=self.gui_settings['size']) + self._bar.iconified = self.gui_settings['iconified'] self.update_bar_markers() - def unset_alive(self): self.alive = False @@ -146,16 +147,17 @@ def remove_surface(self,i): del self.surfaces[i] self.update_bar_markers() - def update_bar_markers(self): - self._bar_markers.clear() - self._bar_markers.add_button(" add surface ", self.add_surface, key='a') - self._bar_markers.add_var(" edit mode ", self.surface_edit_mode ) + self._bar.clear() + self._bar.add_button('close',self.unset_alive) + self._bar.add_var("draw markers",self.draw_markers) + self._bar.add_button(" add surface ", self.add_surface, key='a') + self._bar.add_var(" edit mode ", self.surface_edit_mode ) for s,i in zip(self.surfaces,range(len(self.surfaces)))[::-1]: - self._bar_markers.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window') - self._bar_markers.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') - self._bar_markers.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) - self._bar_markers.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) + self._bar.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window') + self._bar.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') + self._bar.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) + self._bar.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) def update(self,frame,recent_pupil_positions,events): self.img_shape = frame.img.shape @@ -307,6 +309,13 @@ def gl_display_cache_bars(self): + def get_init_dict(self): + d = {} + if hasattr(self,'_bar'): + gui_settings = {'pos':self._bar.position,'size':self._bar.size,'iconified':self._bar.iconified} + d['gui_settings'] = gui_settings + + return d def cleanup(self): @@ -325,7 +334,4 @@ def cleanup(self): for s in self.surfaces: s.close_window() self._bar.destroy() - self._bar_markers.destroy() - - - + self._bar.destroy() From 7dd6bddaaff94d4faa31b48af9e4e47567ef7dff Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Tue, 6 May 2014 23:46:57 +0200 Subject: [PATCH 22/41] deal with no gaze pos in recordings --- pupil_src/player/player_methods.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pupil_src/player/player_methods.py b/pupil_src/player/player_methods.py index d94b7f1eeb..94a5a939db 100644 --- a/pupil_src/player/player_methods.py +++ b/pupil_src/player/player_methods.py @@ -32,6 +32,9 @@ def correlate_gaze(gaze_list,timestamps): positions_by_frame = [[] for i in timestamps] + if len(gaze_list)==0: + logger.warning("No gaze positons in this recording.") + return positions_by_frame frame_idx = 0 data_point = gaze_list.pop(0) gaze_timestamp = data_point[4] From 288f17546ce988fc301df43e5afc808b24f836e7 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Tue, 6 May 2014 23:47:31 +0200 Subject: [PATCH 23/41] updated gui --- pupil_src/shared_modules/marker_detector.py | 56 ++++++++------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index 191c980da1..0f832ff9b6 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -62,17 +62,9 @@ def __init__(self,g_pool,atb_pos=(320,220)): atb_label = "marker detection" self._bar = atb.Bar(name =self.__class__.__name__, label=atb_label, - help="marker detection parameters", color=(50, 50, 50), alpha=100, - text='light', position=atb_pos,refresh=.3, size=(300, 100)) - self._bar.add_var('robust_detection',self.robust_detection,group="Detector") - self._bar.add_var("draw markers",self.draw_markers,group="Detector") - self._bar.add_button('close',self.unset_alive) - + help="marker detection parameters", color=(50, 150, 50), alpha=100, + text='light', position=atb_pos,refresh=.3, size=(300, 300)) - atb_pos = atb_pos[0],atb_pos[1]+110 - self._bar_markers = atb.Bar(name =self.__class__.__name__+'markers', label='registered surfaces', - help="list of registered ref surfaces", color=(50, 100, 50), alpha=100, - text='light', position=atb_pos,refresh=.3, size=(300, 120)) self.update_bar_markers() @@ -115,16 +107,18 @@ def remove_surface(self,i): del self.surfaces[i] self.update_bar_markers() - def update_bar_markers(self): - self._bar_markers.clear() - self._bar_markers.add_button(" add surface ", self.add_surface, key='a') - self._bar_markers.add_var(" edit mode ", self.surface_edit_mode ) + self._bar.clear() + self._bar.add_button('close',self.unset_alive) + self._bar.add_var('robust_detection',self.robust_detection) + self._bar.add_var("draw markers",self.draw_markers) + self._bar.add_button(" add surface ", self.add_surface, key='a') + self._bar.add_var(" edit mode ", self.surface_edit_mode ) for s,i in zip(self.surfaces,range(len(self.surfaces)))[::-1]: - self._bar_markers.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window') - self._bar_markers.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') - self._bar_markers.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) - self._bar_markers.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) + self._bar.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window') + self._bar.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') + self._bar.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) + self._bar.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) def update(self,frame,recent_pupil_positions,events): img = frame.img @@ -133,18 +127,18 @@ def update(self,frame,recent_pupil_positions,events): if self.robust_detection.value: self.markers = detect_markers_robust(img, - grid_size = 5, - prev_markers=self.markers, - min_marker_perimeter=self.min_marker_perimeter, - aperture=self.aperture.value, - visualize=0, - true_detect_every_frame=3) + grid_size = 5, + prev_markers=self.markers, + min_marker_perimeter=self.min_marker_perimeter, + aperture=self.aperture.value, + visualize=0, + true_detect_every_frame=3) else: self.markers = detect_markers_simple(img, - grid_size = 5, - min_marker_perimeter=self.min_marker_perimeter, - aperture=self.aperture.value, - visualize=0) + grid_size = 5, + min_marker_perimeter=self.min_marker_perimeter, + aperture=self.aperture.value, + visualize=0) # locate surfaces @@ -188,7 +182,6 @@ def update(self,frame,recent_pupil_positions,events): s.open_window() - def gl_display(self): """ Display marker and surface info inside world screen @@ -208,7 +201,6 @@ def gl_display(self): s.gl_draw_corners() - def cleanup(self): """ called when the plugin gets terminated. This happends either voluntary or forced. @@ -222,7 +214,3 @@ def cleanup(self): for s in self.surfaces: s.close_window() self._bar.destroy() - self._bar_markers.destroy() - - - From 880d155d441f61b8da8e647552a1bd1014d49d72 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Tue, 6 May 2014 23:48:09 +0200 Subject: [PATCH 24/41] various fixes for REALLY nasty bugs. --- pupil_src/shared_modules/marker_detector_cacher.py | 12 +++++++++--- pupil_src/shared_modules/offline_marker_detector.py | 3 --- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pupil_src/shared_modules/marker_detector_cacher.py b/pupil_src/shared_modules/marker_detector_cacher.py index 007ffe277e..ea3a9bf60b 100644 --- a/pupil_src/shared_modules/marker_detector_cacher.py +++ b/pupil_src/shared_modules/marker_detector_cacher.py @@ -22,9 +22,9 @@ def fill_cache(visited_list,video_file_path,q,seek_idx,run): logger.debug('Started cacher process for Marker Detector') import cv2 from uvc_capture import autoCreateCapture, EndofVideoFileError,FileSeekError - from square_marker_detect import detect_markers_robust + from square_marker_detect import detect_markers_robust,detect_markers_simple min_marker_perimeter = 80 - aperture = 11 + aperture = 9 markers = [] cap = autoCreateCapture(video_file_path) @@ -83,8 +83,14 @@ def handle_frame(next): aperture=aperture, visualize=0, true_detect_every_frame=1) + + # markers[:] = detect_markers_simple(frame.img, + # grid_size = 5, + # min_marker_perimeter=min_marker_perimeter, + # aperture=aperture, + # visualize=0) visited_list[frame.index] = True - q.put((frame.index,markers)) + q.put((frame.index,markers[:])) #object passed will only be pickeld when collected from other process! need to make a copy ot avoid overwrite!!! while run.value: next = cap.get_frame_index() diff --git a/pupil_src/shared_modules/offline_marker_detector.py b/pupil_src/shared_modules/offline_marker_detector.py index 48788e27f6..8a7e34998d 100644 --- a/pupil_src/shared_modules/offline_marker_detector.py +++ b/pupil_src/shared_modules/offline_marker_detector.py @@ -167,13 +167,10 @@ def update(self,frame,recent_pupil_positions,events): # locate markers because precacher has not anayzed this frame yet. Most likely a seek event self.markers = [] self.seek_marker_cacher(frame.index) # tell precacher that it better have every thing from here analyzed - return - # locate surfaces for s in self.surfaces: if not s.locate_from_cache(frame.index): - # logger.debug("location not from cache") s.locate(self.markers) if s.detected: events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) From 46612e37a0c27b1f295ec1287b594bff617d8ae0 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Fri, 9 May 2014 18:24:23 +0200 Subject: [PATCH 25/41] small fixes --- pupil_src/player/main.py | 5 +++-- pupil_src/player/player_methods.py | 10 ++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index 0389919269..89fab51041 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -58,7 +58,7 @@ fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() -ch.setLevel(logging.DEBUG) +ch.setLevel(logging.INFO) # create formatter and add it to the handlers formatter = logging.Formatter('Player: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) @@ -167,7 +167,7 @@ def on_close(window): rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: - rec_dir = '/home/mkassner/Desktop/002' + rec_dir = '/home/mkassner/Desktop/003' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: @@ -251,6 +251,7 @@ def save(var_name,var): g.rec_dir = rec_dir g.app = 'player' g.timestamps = timestamps + g.positions_by_frame = positions_by_frame diff --git a/pupil_src/player/player_methods.py b/pupil_src/player/player_methods.py index 94a5a939db..2fd957f2fb 100644 --- a/pupil_src/player/player_methods.py +++ b/pupil_src/player/player_methods.py @@ -30,13 +30,15 @@ def correlate_gaze(gaze_list,timestamps): gaze_list = list(gaze_list) timestamps = list(timestamps) - positions_by_frame = [[] for i in timestamps] - if len(gaze_list)==0: + + frame_idx = 0 + try: + data_point = gaze_list.pop(0) + except: logger.warning("No gaze positons in this recording.") return positions_by_frame - frame_idx = 0 - data_point = gaze_list.pop(0) + gaze_timestamp = data_point[4] while gaze_list: From b1c92be02835e93f8b2ea3c9def81615a7367dce Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Fri, 9 May 2014 19:44:24 +0200 Subject: [PATCH 26/41] removed shelve with custom PersistenDict Class that uses cross plattform compatible cPickle --- pupil_src/capture/eye.py | 4 ++-- pupil_src/capture/pupil_detectors/canny_detector.py | 4 ++-- pupil_src/capture/world.py | 4 ++-- pupil_src/player/main.py | 4 ++-- pupil_src/shared_modules/marker_detector.py | 5 ++--- pupil_src/shared_modules/offline_marker_detector.py | 7 ++++--- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pupil_src/capture/eye.py b/pupil_src/capture/eye.py index 82e129f9b9..aed356ef05 100644 --- a/pupil_src/capture/eye.py +++ b/pupil_src/capture/eye.py @@ -10,7 +10,7 @@ import os from time import time, sleep -import shelve +from file_methods import Persistent_Dict import logging from ctypes import c_int,c_bool,c_float import numpy as np @@ -119,7 +119,7 @@ def get_from_data(data): # load session persistent settings - session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_eye'),protocol=2) + session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_eye') ) def load(var_name,default): return session_settings.get(var_name,default) def save(var_name,var): diff --git a/pupil_src/capture/pupil_detectors/canny_detector.py b/pupil_src/capture/pupil_detectors/canny_detector.py index 45ca693018..164b94dd66 100644 --- a/pupil_src/capture/pupil_detectors/canny_detector.py +++ b/pupil_src/capture/pupil_detectors/canny_detector.py @@ -19,7 +19,7 @@ import cv2 from time import sleep -import shelve +from file_methods import Persistent_Dict import numpy as np from methods import * import atb @@ -47,7 +47,7 @@ def __init__(self,g_pool): super(Canny_Detector, self).__init__() # load session persistent settings - self.session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_detector'),protocol=2) + self.session_settings =Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_detector') ) # coase pupil filter params self.coarse_detection = c_bool(self.load('coarse_detection',True)) diff --git a/pupil_src/capture/world.py b/pupil_src/capture/world.py index 43e41945ed..3a861c7cde 100644 --- a/pupil_src/capture/world.py +++ b/pupil_src/capture/world.py @@ -19,7 +19,7 @@ import os, sys from time import time -import shelve +from file_methods import Persistent_Dict import logging from ctypes import c_int,c_bool,c_float,create_string_buffer import numpy as np @@ -104,7 +104,7 @@ def on_close(window): # load session persistent settings - session_settings = shelve.open(os.path.join(g_pool.user_dir,'user_settings_world'),protocol=2) + session_settings = Persistent_Dict(os.path.join(g_pool.user_dir,'user_settings_world')) def load(var_name,default): return session_settings.get(var_name,default) def save(var_name,var): diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index 89fab51041..ef9e1e1010 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -72,7 +72,7 @@ logging.getLogger("OpenGL").addHandler(logging.NullHandler()) logger = logging.getLogger(__name__) -import shelve +from file_methods import Persistent_Dict from time import time,sleep from ctypes import c_int,c_bool,c_float,create_string_buffer import numpy as np @@ -210,7 +210,7 @@ def on_close(window): # load session persistent settings - session_settings = shelve.open(os.path.join(user_dir,"user_settings"),protocol=2) + session_settings = Persistent_Dict(os.path.join(user_dir,"user_settings")) def load(var_name,default): return session_settings.get(var_name,default) def save(var_name,var): diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index 0f832ff9b6..afdffa71a8 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -11,8 +11,7 @@ import sys, os,platform import cv2 import numpy as np -import shelve - +from file_methods import Persistent_Dict from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture from methods import normalize,denormalize from glfw import * @@ -41,7 +40,7 @@ def __init__(self,g_pool,atb_pos=(320,220)): self.markers = [] # all registered surfaces - self.surface_definitions = shelve.open(os.path.join(g_pool.user_dir,'surface_definitions'),protocol=2) + self.surface_definitions = Persistent_Dict(os.path.join(g_pool.user_dir,'surface_definitions') ) self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] # edit surfaces diff --git a/pupil_src/shared_modules/offline_marker_detector.py b/pupil_src/shared_modules/offline_marker_detector.py index 8a7e34998d..1d98115a06 100644 --- a/pupil_src/shared_modules/offline_marker_detector.py +++ b/pupil_src/shared_modules/offline_marker_detector.py @@ -11,7 +11,6 @@ import sys, os,platform import cv2 import numpy as np -import shelve if platform.system() == 'Darwin': @@ -27,6 +26,8 @@ from OpenGL.GL import * from OpenGL.GLU import gluOrtho2D from methods import normalize,denormalize,Cache_List +from file_methods import Persistent_Dict + from glfw import * import atb from ctypes import c_int,c_bool,create_string_buffer @@ -64,7 +65,7 @@ def __init__(self,g_pool,gui_settings={'pos':(220,200),'size':(300,300),'iconifi if g_pool.app == 'capture': raise Exception('For Player only.') #in player we load from the rec_dir: but we have a couple options: - self.surface_definitions = shelve.open(os.path.join(g_pool.rec_dir,'surface_definitions'),protocol=2) + self.surface_definitions = Persistent_Dict(os.path.join(g_pool.rec_dir,'surface_definitions')) if self.load('offline_square_marker_surfaces',[]) != []: logger.debug("Found ref surfaces defined or copied in previous session.") self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)] @@ -85,7 +86,7 @@ def __init__(self,g_pool,gui_settings={'pos':(220,200),'size':(300,300),'iconifi self.min_marker_perimeter = 80 #check if marker cache is available from last session - self.persistent_cache = shelve.open(os.path.join(g_pool.rec_dir,'square_marker_cache'),protocol=2) + self.persistent_cache = Persistent_Dict(os.path.join(g_pool.rec_dir,'square_marker_cache')) self.cache = Cache_List(self.persistent_cache.get('marker_cache',[False for _ in g_pool.timestamps])) logger.debug("Loaded marker cache %s / %s frames had been searched before"%(len(self.cache)-self.cache.count(False),len(self.cache)) ) self.init_marker_cacher() From 1652edd2d552127c1ce0289779259080821a9e6d Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Fri, 9 May 2014 19:45:59 +0200 Subject: [PATCH 27/41] added missing file. --- pupil_src/shared_modules/file_methods.py | 51 ++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 pupil_src/shared_modules/file_methods.py diff --git a/pupil_src/shared_modules/file_methods.py b/pupil_src/shared_modules/file_methods.py new file mode 100644 index 0000000000..2c35781a02 --- /dev/null +++ b/pupil_src/shared_modules/file_methods.py @@ -0,0 +1,51 @@ + +import cPickle as pickle +import os +import logging +logger = logging.getLogger(__name__) + +class Persistent_Dict(dict): + """a dict class that uses pickle to save inself to file""" + def __init__(self, file_path): + super(Persistent_Dict, self).__init__() + self.file_path = os.path.expanduser(file_path) + try: + with open(self.file_path,'rb') as fh: + try: + self.update(pickle.load(fh)) + except: #KeyError,EOFError + logger.warning("Session settings file '%s'could not be read. Will overwrite on exit."%self.file_path) + except IOError: + logger.debug("Session settings file '%s' not found. Will make new one on exit."%self.file_path) + + + def close(self): + d = {} + d.update(self) + try: + with open(self.file_path,'wb') as fh: + pickle.dump(d,fh,-1) + except IOError: + logger.warning("Could not save session settings to '%s'"%self.file_path) + + + +def load_object(file_path): + file_path = os.path.expanduser(file_path) + with open(file_path,'rb') as fh: + return pickle.load(fh) + +def save_object(object,file_path): + file_path = os.path.expanduser(file_path) + with open(file_path,'wb') as fh: + pickle.dump(object,fh,-1) + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + settings = Persistent_Dict("~/Desktop/test") + settings['f'] = "this is a test" + settings['list'] = ["list 1","list2"] + settings.close() + + save_object("string",'test') + print load_object('test') \ No newline at end of file From a373f15c4248155963a92b8c24b1e738987d7c55 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Tue, 13 May 2014 12:30:28 +0200 Subject: [PATCH 28/41] made pupil result confidence parameter accesible to player plugins --- pupil_src/player/player_methods.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pupil_src/player/player_methods.py b/pupil_src/player/player_methods.py index 2fd957f2fb..09256255ed 100644 --- a/pupil_src/player/player_methods.py +++ b/pupil_src/player/player_methods.py @@ -48,7 +48,7 @@ def correlate_gaze(gaze_list,timestamps): except IndexError: break if gaze_timestamp <= t_between_frames: - positions_by_frame[frame_idx].append({'norm_gaze':(data_point[0],data_point[1]),'norm_pupil': (data_point[2],data_point[3]), 'timestamp':gaze_timestamp}) + positions_by_frame[frame_idx].append({'norm_gaze':(data_point[0],data_point[1]),'norm_pupil': (data_point[2],data_point[3]), 'timestamp':data_point[4],'confidence':data_point[5]}) data_point = gaze_list.pop(0) gaze_timestamp = data_point[4] else: From 7dc7dac40e6b2f2baff7eceef6c196564ae0b9fe Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Tue, 13 May 2014 14:16:42 +0200 Subject: [PATCH 29/41] more work on offline amrker detector --- pupil_src/capture/recorder.py | 16 +-- pupil_src/player/main.py | 6 +- pupil_src/shared_modules/file_methods.py | 7 +- pupil_src/shared_modules/marker_detector.py | 8 +- pupil_src/shared_modules/methods.py | 91 ------------- .../shared_modules/offline_marker_detector.py | 39 +++--- pupil_src/shared_modules/reference_surface.py | 126 +++++------------- 7 files changed, 68 insertions(+), 225 deletions(-) diff --git a/pupil_src/capture/recorder.py b/pupil_src/capture/recorder.py index 6fcc059785..e2520397ea 100644 --- a/pupil_src/capture/recorder.py +++ b/pupil_src/capture/recorder.py @@ -111,6 +111,7 @@ def stop_and_destruct(self): self.eye_tx.send(None) except: logger.warning("Could not stop eye-recording. Please report this bug!") + gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy") np.save(gaze_list_path,np.asarray(self.gaze_list)) @@ -118,25 +119,18 @@ def stop_and_destruct(self): np.save(timestamps_path,np.array(self.timestamps)) try: - surface_definitions_file = glob(os.path.join(self.g_pool.user_dir,"surface_definitions*"))[0].rsplit(os.path.sep,1)[-1] - copy2(os.path.join(self.g_pool.user_dir,surface_definitions_file),os.path.join(self.rec_path,surface_definitions_file)) + copy2(os.path.join(self.g_pool.user_dir,"surface_definitions"),os.path.join(self.rec_path,"surface_definitions")) except: logger.info("No surface_definitions data found. You may want this if you do marker tracking.") try: - cal_pt_cloud = np.load(os.path.join(self.g_pool.user_dir,"cal_pt_cloud.npy")) - cal_pt_cloud_path = os.path.join(self.rec_path, "cal_pt_cloud.npy") - np.save(cal_pt_cloud_path, cal_pt_cloud) + copy2(os.path.join(self.g_pool.user_dir,"cal_pt_cloud.npy"),os.path.join(self.rec_path,"cal_pt_cloud.npy")) except: logger.warning("No calibration data found. Please calibrate first.") try: - camera_matrix = np.load(os.path.join(self.g_pool.user_dir,"camera_matrix.npy")) - dist_coefs = np.load(os.path.join(self.g_pool.user_dir,"dist_coefs.npy")) - cam_path = os.path.join(self.rec_path, "camera_matrix.npy") - dist_path = os.path.join(self.rec_path, "dist_coefs.npy") - np.save(cam_path, camera_matrix) - np.save(dist_path, dist_coefs) + copy2(os.path.join(self.g_pool.user_dir,"camera_matrix.npy"),os.path.join(self.rec_path,"camera_matrix.npy")) + copy2(os.path.join(self.g_pool.user_dir,"dist_coefs.npy"),os.path.join(self.rec_path,"dist_coefs.npy")) except: logger.info("No camera intrinsics found.") diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index ef9e1e1010..6b65b7162e 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -167,7 +167,7 @@ def on_close(window): rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: - rec_dir = '/home/mkassner/Desktop/003' + rec_dir = '/home/mkassner/Desktop/002' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: @@ -242,7 +242,7 @@ def save(var_name,var): glfwSetScrollCallback(main_window,on_scroll) - # create container for globally scoped vars (within world) + # create container for globally scoped varfs (within world) g = Temp() g.plugins = [] g.play = False @@ -405,7 +405,7 @@ def get_from_data(data): g.new_seek = False frame = new_frame.copy() - #new positons and events we make a deepcopy just like the image should be a copy. + #new positons and events we make a deepcopy just like the image is a copy. current_pupil_positions = deepcopy(positions_by_frame[frame.index]) events = [] diff --git a/pupil_src/shared_modules/file_methods.py b/pupil_src/shared_modules/file_methods.py index 2c35781a02..6aab06bc9b 100644 --- a/pupil_src/shared_modules/file_methods.py +++ b/pupil_src/shared_modules/file_methods.py @@ -19,7 +19,7 @@ def __init__(self, file_path): logger.debug("Session settings file '%s' not found. Will make new one on exit."%self.file_path) - def close(self): + def save(self): d = {} d.update(self) try: @@ -30,6 +30,11 @@ def close(self): + def close(self): + self.save() + + + def load_object(file_path): file_path = os.path.expanduser(file_path) with open(file_path,'rb') as fh: diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index afdffa71a8..e53f6fcdf4 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -29,7 +29,6 @@ class Marker_Detector(Plugin): """docstring - """ def __init__(self,g_pool,atb_pos=(320,220)): super(Marker_Detector, self).__init__() @@ -123,7 +122,6 @@ def update(self,frame,recent_pupil_positions,events): img = frame.img self.img_shape = frame.img.shape - if self.robust_detection.value: self.markers = detect_markers_robust(img, grid_size = 5, @@ -139,7 +137,6 @@ def update(self,frame,recent_pupil_positions,events): aperture=self.aperture.value, visualize=0) - # locate surfaces for s in self.surfaces: s.locate(self.markers) @@ -165,12 +162,12 @@ def update(self,frame,recent_pupil_positions,events): #map recent gaze onto detected surfaces used for pupil server for s in self.surfaces: if s.detected: - s.recent_gaze = [] + s.gaze_on_srf = [] for p in recent_pupil_positions: if p['norm_pupil'] is not None: gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_gaze']))) p['realtime gaze on '+s.name] = gp_on_s - s.recent_gaze.append(gp_on_s) + s.gaze_on_srf.append(gp_on_s) #allow surfaces to open/close windows @@ -185,7 +182,6 @@ def gl_display(self): """ Display marker and surface info inside world screen """ - for m in self.markers: hat = np.array([[[0,0],[0,1],[.5,1.3],[1,1],[1,0],[0,0]]],dtype=np.float32) hat = cv2.perspectiveTransform(hat,m_marker_to_screen(m)) diff --git a/pupil_src/shared_modules/methods.py b/pupil_src/shared_modules/methods.py index 4107adc87c..81079285f9 100644 --- a/pupil_src/shared_modules/methods.py +++ b/pupil_src/shared_modules/methods.py @@ -16,7 +16,6 @@ import cv2 import logging logger = logging.getLogger(__name__) - class Temp(object): """Temp class to make objects""" def __init__(self): @@ -88,96 +87,6 @@ def set(self,vals): def get(self): return self.lX,self.lY,self.uX,self.uY,self.array_shape -import itertools - -class Cache_List(list): - """Cache list is a list of False - [False,False,False] - with update() 'False' can be overwritten with a result (anything not 'False') - self.ranges show ranges where the cache does not evaluate as 'False' using eval_fn - this allows to use ranges a a way of showing where no caching has happed (default) or whatever you do with eval_fn - Warning: a positve result cannot be overwritten by a negative one in this implementation - self.complete indicated that the cache list has no unknowns eval_fn(n) == False - """ - - def __init__(self, init_list,eval_fn=lambda x: x!=False): - super(Cache_List, self).__init__(init_list) - self.eval_fn = eval_fn - self._ranges = self.init_ranges() - self._togo = self.count(False) - self.length = len(self) - - - @property - def ranges(self): - return self._ranges - - @ranges.setter - def ranges(self, value): - raise Exception("Read only") - - @property - def complete(self): - return self._togo == 0 - - @complete.setter - def complete(self, value): - raise Exception("Read only") - - - def update(self,key,item): - if self[key] != False: - logger.warning("You are overwriting a precached result.") - self[key] = item - self._ranges = self.init_ranges() - else: - #unvisited - self[key] = item - if self.eval_fn(item): #need to update ranges - self.update_ranges(key) - self._togo -= 1 - - - def init_ranges(self): - l = self - i = -1 - ranges = [] - for t,g in itertools.groupby(l,self.eval_fn): - l = i + 1 - i += len(list(g)) - if t: - ranges.append([l,i]) - return ranges - - def merge_ranges(self): - l = self._ranges - for i in range(len(l)-1): - if l[i][1] == l[i+1][0]-1: - #merge touching fields - l[i] = ([l[i][0],l[i+1][1]]) - #del second field - del l[i+1] - return - return - - def update_ranges(self,i): - l = self._ranges - for _range in l: - #most common case: extend a range - if i == _range[0]-1: - _range[0] = i - self.merge_ranges() - return - elif i == _range[1]+1: - _range[1] = i - self.merge_ranges() - return - #somewhere outside of range proximity - l.append([i,i]) - l.sort(key=lambda x: x[0]) - - def to_list(self): - return list(self) def bin_thresholding(image, image_lower=0, image_upper=256): binary_img = cv2.inRange(image, np.asarray(image_lower), diff --git a/pupil_src/shared_modules/offline_marker_detector.py b/pupil_src/shared_modules/offline_marker_detector.py index 1d98115a06..68b0d1604b 100644 --- a/pupil_src/shared_modules/offline_marker_detector.py +++ b/pupil_src/shared_modules/offline_marker_detector.py @@ -25,12 +25,12 @@ from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture from OpenGL.GL import * from OpenGL.GLU import gluOrtho2D -from methods import normalize,denormalize,Cache_List +from methods import normalize,denormalize from file_methods import Persistent_Dict - +from cache_list import Cache_List from glfw import * import atb -from ctypes import c_int,c_bool,create_string_buffer +from ctypes import c_int,c_bool,c_float,create_string_buffer from plugin import Plugin #logging @@ -38,7 +38,7 @@ logger = logging.getLogger(__name__) from square_marker_detect import detect_markers_robust,detect_markers_simple, draw_markers,m_marker_to_screen -from reference_surface import Reference_Surface +from offline_reference_surface import Offline_Reference_Surface from math import sqrt @@ -57,6 +57,7 @@ def __init__(self,g_pool,gui_settings={'pos':(220,200),'size':(300,300),'iconifi self.g_pool = g_pool self.gui_settings = gui_settings self.order = .2 + # all markers that are detected in the most recent frame self.markers = [] @@ -68,10 +69,10 @@ def __init__(self,g_pool,gui_settings={'pos':(220,200),'size':(300,300),'iconifi self.surface_definitions = Persistent_Dict(os.path.join(g_pool.rec_dir,'surface_definitions')) if self.load('offline_square_marker_surfaces',[]) != []: logger.debug("Found ref surfaces defined or copied in previous session.") - self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)] + self.surfaces = [Offline_Reference_Surface(saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)] elif self.load('offline_square_marker_surfaces',[]) != []: logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.") - self.surfaces = [Reference_Surface(saved_definition=d) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] + self.surfaces = [Offline_Reference_Surface(saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] else: logger.debug("No surface defs found. Please define using GUI.") self.surfaces = [] @@ -140,7 +141,7 @@ def advance(self): pass def add_surface(self): - self.surfaces.append(Reference_Surface()) + self.surfaces.append(Offline_Reference_Surface(gaze_positions_by_frame=self.g_pool.positions_by_frame)) self.update_bar_markers() def remove_surface(self,i): @@ -158,6 +159,9 @@ def update_bar_markers(self): self._bar.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window') self._bar.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') self._bar.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) + self._bar.add_var("%s_x_scale"%i,vtype=c_float, getter=s.atb_get_scale_x, min=1,setter=s.atb_set_scale_x,group=str(i),label='scale factor x', help='the scale factor is used to adjust the coordinate space for your needs (think photo pixels or mm or whatever)' ) + self._bar.add_var("%s_y_scale"%i,vtype=c_float, getter=s.atb_get_scale_y,min=1,setter=s.atb_set_scale_y,group=str(i),label='scale factor y',help='defining x and y scale factor you atumatically set the correct aspect ratio.' ) + self._bar.add_button("%s_export"%i, s.save_surface_positions_to_file,label='export surface data',group=str(i)) self._bar.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) def update(self,frame,recent_pupil_positions,events): @@ -174,7 +178,7 @@ def update(self,frame,recent_pupil_positions,events): if not s.locate_from_cache(frame.index): s.locate(self.markers) if s.detected: - events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp}) + events.append({'type':'marker_ref_surface','name':s.name,'uid':s.uid,'m_to_screen':s.m_to_screen,'m_from_screen':s.m_from_screen, 'timestamp':frame.timestamp,'gaze_on_srf':s.gaze_on_srf}) if self.draw_markers.value: draw_markers(frame.img,self.markers) @@ -191,22 +195,14 @@ def update(self,frame,recent_pupil_positions,events): pos = normalize(pos,(self.img_shape[1],self.img_shape[0]),flip_y=True) new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) + s.cache = None + s.gaze_cache = None else: # update srf with no or invald cache: for s in self.surfaces: if s.cache == None: s.init_cache(self.cache) - #map recent gaze onto detected surfaces used for pupil server - for s in self.surfaces: - if s.detected: - s.recent_gaze = [] - for p in recent_pupil_positions: - if p['norm_pupil'] is not None: - gp_on_s = tuple(s.img_to_ref_surface(np.array(p['norm_gaze']))) - p['realtime gaze on '+s.name] = gp_on_s - s.recent_gaze.append(gp_on_s) - #allow surfaces to open/close windows for s in self.surfaces: @@ -215,6 +211,9 @@ def update(self,frame,recent_pupil_positions,events): if s.window_should_open: s.open_window() + + + def init_marker_cacher(self): forking_enable(0) #for MacOs only from marker_detector_cacher import fill_cache @@ -268,7 +267,7 @@ def gl_display_cache_bars(self): # Lines for areas that have be cached cached_ranges = [] - for r in self.cache.ranges: # [[0,1],[3,4]] + for r in self.cache.visited_ranges: # [[0,1],[3,4]] cached_ranges += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)] # Lines where surfaces have been found in video @@ -276,7 +275,7 @@ def gl_display_cache_bars(self): for s in self.surfaces: found_at = [] if s.cache is not None: - for r in s.cache.ranges: # [[0,1],[3,4]] + for r in s.cache.positive_ranges: # [[0,1],[3,4]] found_at += (r[0],0),(r[1],0) #[(0,0),(1,0),(3,0),(4,0)] cached_surfaces.append(found_at) diff --git a/pupil_src/shared_modules/reference_surface.py b/pupil_src/shared_modules/reference_surface.py index e5dfb4e5fc..0aed6252d6 100644 --- a/pupil_src/shared_modules/reference_surface.py +++ b/pupil_src/shared_modules/reference_surface.py @@ -15,7 +15,7 @@ from OpenGL.GL import * from OpenGL.GLU import gluOrtho2D -from methods import GetAnglesPolyline,normalize,Cache_List +from methods import GetAnglesPolyline,normalize #ctypes import for atb_vars: from ctypes import c_int,c_bool,create_string_buffer @@ -35,7 +35,6 @@ def m_verts_from_screen(verts): return cv2.getPerspectiveTransform(verts,mapped_space_one) - class Reference_Surface(object): """docstring for Reference Surface @@ -72,12 +71,12 @@ def __init__(self,name="unnamed",saved_definition=None): self.detected = False self.m_to_screen = None self.m_from_screen = None - self.cache = None + self.uid = str(time()) + self.scale_factor = [1.,1.] + if saved_definition is not None: self.load_from_dict(saved_definition) - else: - self.uid = str(time()) ###window and gui vars self._window = None @@ -96,14 +95,14 @@ def __init__(self,name="unnamed",saved_definition=None): # monitor_enum = atb.enum("Monitor",dict(((key,val) for val,key in enumerate(self.monitor_names)))) #primary_monitor = glfwGetPrimaryMonitor() - self.recent_gaze = [] # points on surface for realtime feedback display + self.gaze_on_srf = [] # points on surface for realtime feedback display def save_to_dict(self): """ save all markers and name of this surface to a dict. """ markers = dict([(m_id,m.uv_coords) for m_id,m in self.markers.iteritems()]) - return {'name':self.name,'uid':self.uid,'markers':markers} + return {'name':self.name,'uid':self.uid,'markers':markers,'scale_factor':self.scale_factor} def load_from_dict(self,d): @@ -112,6 +111,8 @@ def load_from_dict(self,d): """ self.name = d['name'] self.uid = d['uid'] + self.scale_factor = d['scale_factor'] + marker_dict = d['markers'] for m_id,uv_coords in marker_dict.iteritems(): self.markers[m_id] = Support_Marker(m_id) @@ -248,6 +249,17 @@ def ref_surface_to_img(self,pos): else: return None + def gaze_on_srf(self,gaze_positions,m_from_screen): + if self.m_from_screen is None: + return None + gaze_on_src = [] + for g_p in gaze_positions: + gaze_points = np.array([g_p['norm_gaze']]).reshape(1,1,2) + gaze_points_on_srf = cv2.perspectiveTransform(gaze_points , m_from_screen ) + gaze_points_on_srf.shape = (2) + gaze_on_src.append( {'norm_gaze_on_srf':(gaze_points_on_srf[0],gaze_points_on_srf[1]),'timestamp':g_p['timestamp'] } ) + return gaze_on_src + def move_vertex(self,vert_idx,new_pos): """ @@ -264,7 +276,6 @@ def move_vertex(self,vert_idx,new_pos): for m in self.markers.values(): m.uv_coords = cv2.perspectiveTransform(m.uv_coords,transform) - self.cache = None #purge cache as it is not valid anymore def atb_marker_status(self): return create_string_buffer("%s / %s" %(self.detected_markers,len(self.markers)),512) @@ -275,6 +286,16 @@ def atb_get_name(self): def atb_set_name(self,name): self.name = name.value + def atb_set_scale_x(self,val): + self.scale_factor[0]=val + def atb_set_scale_y(self,val): + self.scale_factor[1]=val + def atb_get_scale_x(self): + return self.scale_factor[0] + def atb_get_scale_y(self): + return self.scale_factor[1] + + def gl_draw_frame(self): """ draw surface and markers @@ -330,10 +351,9 @@ def gl_display_in_window(self,world_tex_id): glMatrixMode(GL_MODELVIEW) glPopMatrix() - - #now lets get recent pupil positions on this surface: - draw_gl_points_norm(self.recent_gaze,color=(0.,8.,.5,.8), size=80) - + # now lets get recent pupil positions on this surface: + draw_gl_points_norm(self.gaze_on_srf,color=(0.,8.,.5,.8), size=80) + glfwSwapBuffers(self._window) glfwMakeContextCurrent(active_window) @@ -355,7 +375,7 @@ def open_window(self): height,width= mode[0],mode[1] else: monitor = None - height,width= 640,640 + height,width= 640,int(640./(self.scale_factor[0]/self.scale_factor[1])) #open with same aspect ratio as surface self._window = glfwCreateWindow(height, width, "Reference Surface: " + self.name, monitor=monitor, share=glfwGetCurrentContext()) if not self.fullscreen.value: @@ -407,86 +427,6 @@ def cleanup(self): self.close_window() - #cache fn for offline marker - def locate_from_cache(self,frame_idx): - if self.cache == None: - #no cache available cannot update from cache - return False - cache_result = self.cache[frame_idx] - if cache_result == False: - #cached data not avaible for this frame - return False - elif cache_result == None: - #cached data shows surface not found: - self.detected = False - self.m_from_screen = None - self.m_to_screen = None - return True - else: - self.detected = True - self.m_from_screen = cache_result['m_from_screen'] - self.m_to_screen = cache_result['m_to_screen'] - self.detected_markers = cache_result['detected_markers'] - return True - raise Exception("Invalid cache entry. Please report Bug.") - - - def update_cache(self,marker_cache,idx=None): - ''' - compute surface m's and gaze points from cached marker data - entries are: - - False: when marker cache entry was False (not yet searched) - - None: when surface was not found - - {'m_to_screen':,'m_from_screen':,'detected_markers':} - ''' - - # iterations = 0 - - if self.cache == None: - self.init_cache(marker_cache) - elif idx != None: - #update single data pt - self.cache.update(idx,self.answer_caching_request(marker_cache[idx])) - else: - # update where markercache is not False but surface cache is still false - # this happens when the markercache was incomplete when this fn was run before - for i in range(len(marker_cache)): - if self.cache[i] == False and marker_cache[i] != False: - self.cache.update(i,self.answer_caching_request(marker_cache[i])) - # iterations +=1 - # return iterations - - def init_cache(self,marker_cache): - if self.defined: - logger.debug("Full update of surface '%s' positons cache"%self.name) - self.cache = Cache_List([self.answer_caching_request(c_m) for c_m in marker_cache],eval_fn=lambda x: (x!=False) and (x!=None)) - - - def answer_caching_request(self,visible_markers): - # cache point had not been visited - if visible_markers == False: - return False - # cache point had been visited - marker_by_id = dict([(m['id'],m) for m in visible_markers]) - visible_ids = set(marker_by_id.keys()) - requested_ids = set(self.markers.keys()) - overlap = visible_ids & requested_ids - detected_markers = len(overlap) - if len(overlap)>=min(2,len(requested_ids)): - yx = np.array( [marker_by_id[i]['verts_norm'] for i in overlap] ) - uv = np.array( [self.markers[i].uv_coords for i in overlap] ) - yx.shape=(-1,1,2) - uv.shape=(-1,1,2) - # print 'uv',uv - # print 'yx',yx - m_to_screen,mask = cv2.findHomography(uv,yx) - m_from_screen,mask = cv2.findHomography(yx,uv) - - return {'m_to_screen':m_to_screen,'m_from_screen':m_from_screen,'detected_markers':len(overlap)} - else: - #surface not found - return None - class Support_Marker(object): ''' From f4d2055bce0495cb80c39d0450960ebf5bcf5e3a Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Tue, 13 May 2014 14:18:21 +0200 Subject: [PATCH 30/41] cache file class now in its onw module. offline ref surface is a class derived from ref surface that utilizes caching. It is used by offline amrker tracker. --- pupil_src/shared_modules/cache_list.py | 124 ++++++++++++ .../offline_reference_surface.py | 188 ++++++++++++++++++ 2 files changed, 312 insertions(+) create mode 100644 pupil_src/shared_modules/cache_list.py create mode 100644 pupil_src/shared_modules/offline_reference_surface.py diff --git a/pupil_src/shared_modules/cache_list.py b/pupil_src/shared_modules/cache_list.py new file mode 100644 index 0000000000..44dd32f9f3 --- /dev/null +++ b/pupil_src/shared_modules/cache_list.py @@ -0,0 +1,124 @@ + +import logging +logger = logging.getLogger(__name__) +import itertools + +class Cache_List(list): + """Cache list is a list of False + [False,False,False] + with update() 'False' can be overwritten with a result (anything not 'False') + self.visited_ranges show ranges where the cache contect is False + self.positive_ranges show ranges where the cache does not evaluate as 'False' using eval_fn + this allows to use ranges a a way of showing where no caching has happed (default) or whatever you do with eval_fn + self.complete indicated that the cache list has no unknowns aka False + """ + + def __init__(self, init_list,positive_eval_fn=None): + super(Cache_List, self).__init__(init_list) + + self.visited_eval_fn = lambda x: x!=False + self._visited_ranges = init_ranges(l = self,eval_fn = self.visited_eval_fn ) + self._togo = self.count(False) + self.length = len(self) + + if positive_eval_fn == None: + self.positive_eval_fn = lambda x: False + self._positive_ranges = [] + else: + self.positive_eval_fn = positive_eval_fn + self._positive_ranges = init_ranges(l = self,eval_fn = self.positive_eval_fn ) + + @property + def visited_ranges(self): + return self._visited_ranges + + @visited_ranges.setter + def visited_ranges(self, value): + raise Exception("Read only") + + @property + def positive_ranges(self): + return self._positive_ranges + + @positive_ranges.setter + def positive_ranges(self, value): + raise Exception("Read only") + + + @property + def complete(self): + return self._togo == 0 + + @complete.setter + def complete(self, value): + raise Exception("Read only") + + + def update(self,key,item): + if self[key] != False: + logger.warning("You are overwriting a precached result.") + self[key] = item + self._visited_ranges = init_ranges(l = self,eval_fn = self.visited_eval_fn ) + self._positive_ranges = init_ranges(l = self,eval_fn = self.positive_eval_fn ) + + elif item != False: + #unvisited + self._togo -= 1 + self[key] = item + + update_ranges(self._visited_ranges,key) + if self.positive_eval_fn(item): + update_ranges(self._positive_ranges,key) + else: + #writing False to list entry already false, do nothing + pass + + + def to_list(self): + return list(self) + + + +def init_ranges(l,eval_fn): + i = -1 + ranges = [] + for t,g in itertools.groupby(l,eval_fn): + l = i + 1 + i += len(list(g)) + if t: + ranges.append([l,i]) + return ranges + +def update_ranges(l,i): + for _range in l: + #most common case: extend a range + if i == _range[0]-1: + _range[0] = i + merge_ranges(l) + return + elif i == _range[1]+1: + _range[1] = i + merge_ranges(l) + return + #somewhere outside of range proximity + l.append([i,i]) + l.sort(key=lambda x: x[0]) + +def merge_ranges(l): + for i in range(len(l)-1): + if l[i][1] == l[i+1][0]-1: + #merge touching fields + l[i] = ([l[i][0],l[i+1][1]]) + #del second field + del l[i+1] + return + return + + +if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG) + cl = Cache_List(range(100),positive_eval_fn = lambda x: bool(x%2)) + cl.update(0,1) + cl.update(4,1) + print cl.positive_ranges + print cl \ No newline at end of file diff --git a/pupil_src/shared_modules/offline_reference_surface.py b/pupil_src/shared_modules/offline_reference_surface.py new file mode 100644 index 0000000000..85730d928a --- /dev/null +++ b/pupil_src/shared_modules/offline_reference_surface.py @@ -0,0 +1,188 @@ +''' +(*)~---------------------------------------------------------------------------------- + Pupil - eye tracking platform + Copyright (C) 2012-2014 Pupil Labs + + Distributed under the terms of the CC BY-NC-SA License. + License details are in the file license.txt, distributed as part of this software. +----------------------------------------------------------------------------------~(*) +''' + +import numpy as np +import cv2 +from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture +from glfw import * +from OpenGL.GL import * +from OpenGL.GLU import gluOrtho2D + +from methods import GetAnglesPolyline,normalize +from cache_list import Cache_List + +#ctypes import for atb_vars: +from ctypes import c_int,c_bool,create_string_buffer +from time import time + +import logging +logger = logging.getLogger(__name__) + +from reference_surface import Reference_Surface + +class Offline_Reference_Surface(Reference_Surface): + """docstring for Offline_Reference_Surface""" + def __init__(self,name="unnamed",saved_definition=None, gaze_positions_by_frame = None): + super(Offline_Reference_Surface, self).__init__(name,saved_definition) + self.gaze_positions_by_frame = gaze_positions_by_frame + self.cache = None + + + #cache fn for offline marker + def locate_from_cache(self,frame_idx): + if self.cache == None: + #no cache available cannot update from cache + return False + cache_result = self.cache[frame_idx] + if cache_result == False: + #cached data not avaible for this frame + return False + elif cache_result == None: + #cached data shows surface not found: + self.detected = False + self.m_from_screen = None + self.m_to_screen = None + self.gaze_on_srf = [] + return True + else: + self.detected = True + self.m_from_screen = cache_result['m_from_screen'] + self.m_to_screen = cache_result['m_to_screen'] + self.detected_markers = cache_result['detected_markers'] + self.gaze_on_srf = [gp['norm_gaze_on_srf'] for gp in cache_result['gaze_on_srf'] ] + return True + raise Exception("Invalid cache entry. Please report Bug.") + + + def update_cache(self,marker_cache,idx=None): + ''' + compute surface m's and gaze points from cached marker data + entries are: + - False: when marker cache entry was False (not yet searched) + - None: when surface was not found + - {'m_to_screen':,'m_from_screen':,'detected_markers':,gaze_on_srf} + ''' + + # iterations = 0 + + if self.cache == None: + self.init_cache(marker_cache) + elif idx != None: + #update single data pt + self.cache.update(idx,self.answer_caching_request(marker_cache,idx)) + else: + # update where markercache is not False but surface cache is still false + # this happens when the markercache was incomplete when this fn was run before + for i in range(len(marker_cache)): + if self.cache[i] == False and marker_cache[i] != False: + self.cache.update(i,self.answer_caching_request(marker_cache,i)) + # iterations +=1 + # return iterations + + + + def init_cache(self,marker_cache): + if self.defined: + logger.debug("Full update of surface '%s' positons cache"%self.name) + self.cache = Cache_List([self.answer_caching_request(marker_cache,i) for i in xrange(len(marker_cache))],positive_eval_fn=lambda x: (x!=False) and (x!=None)) + + + def answer_caching_request(self,marker_cache,frame_index): + visible_markers = marker_cache[frame_index] + # cache point had not been visited + if visible_markers == False: + return False + # cache point had been visited + marker_by_id = dict( [ (m['id'],m) for m in visible_markers] ) + visible_ids = set(marker_by_id.keys()) + requested_ids = set(self.markers.keys()) + overlap = visible_ids & requested_ids + detected_markers = len(overlap) + if len(overlap)>=min(2,len(requested_ids)): + yx = np.array( [marker_by_id[i]['verts_norm'] for i in overlap] ) + uv = np.array( [self.markers[i].uv_coords for i in overlap] ) + yx.shape=(-1,1,2) + uv.shape=(-1,1,2) + m_to_screen,mask = cv2.findHomography(uv,yx) + m_from_screen,mask = cv2.findHomography(yx,uv) + + return {'m_to_screen':m_to_screen, + 'm_from_screen':m_from_screen, + 'detected_markers':len(overlap), + 'gaze_on_srf':self.gaze_on_srf_by_frame_idx(frame_index,m_from_screen)} + else: + #surface not found + return None + + + def gaze_on_srf_by_frame_idx(self,frame_index,m_from_screen): + gaze_positions = self.gaze_positions_by_frame[frame_index] + gaze_on_src = [] + for g_p in gaze_positions: + gaze_points = np.array([g_p['norm_gaze']]).reshape(1,1,2) + gaze_points_on_srf = cv2.perspectiveTransform(gaze_points , m_from_screen ) + gaze_points_on_srf.shape = (2) + gaze_on_src.append( {'norm_gaze_on_srf':(gaze_points_on_srf[0],gaze_points_on_srf[1]),'timestamp':g_p['timestamp'] } ) + return gaze_on_src + + + # def map_gaze_by_frame_onto_srf(self,gaze_by_frame): + # ''' + # this fn is not cached and can be slow, dont call every frame... + # ''' + + # gaze_on_srf_by_frame = [[] for i in gaze_by_frame] + + # if self.cache: + # for surface_data,gaze_points,result in zip(self.cache,gaze_by_frame,gaze_on_srf_by_frame): + # if surface_data and gaze_points: + # gaze_points = np.array([d['norm_gaze'] for d in gaze_points if d is not None]) + # gaze_points.shape = (-1,1,2) + # gaze_points_on_srf = cv2.perspectiveTransform(gaze_points , surface_data['m_from_screen'] ) + # gaze_points_on_srf.shape = (-1,2) + # result.extend(gaze_points_on_srf.tolist()) + # return gaze_on_srf_by_frame + + + def save_surface_positions_to_file(self,s): + if s.cache == None: + logger.warning("The surface is not cached. Please wait for the cacher to collect data.") + return + + srf_dir = os.path.join(self.g_pool.rec_dir,'surface_data',s.name.replace('/',''),s.uid) + logger.info("exporting surface gaze data to %s"%srf_dir) + if os.path.isdir(srf_dir): + logger.info("Will overwrite previous export for this referece surface") + else: + try: + os.mkdir(srf_dir) + except: + logger.warning("Could name make export dir %s"%srf_dir) + return + + # logger.info("Saving surface positon data and gaze on surface data for '%s' with uid:'%'"%(s.name,s.uid)) + # #save surface_positions as pickle file + # save_object(s.cache,os.path.join(srf_dir,'srf_positons_by_frame')) + # #save surface_positions as csv + # with open(os.path.join(srf_dir,'srf_positons_by_frame.csv','wb')) as csvfile: + # csw_writer =csv.writer(csvfile, delimiter=' ',quotechar='|', quoting=csv.QUOTE_MINIMAL) + # csw_writer.writerow(('frame_idx','timestamp','m_to_screen','m_from_screen','detected_markers')) + # for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache) + # if ref_srf_data is not None: + # csw_writer.writerow( (idx,ts,ref_srf_data['m_to_screen'],ref_srf_data['m_from_screen'],ref_srf_data['detected_markers']) ) + # #save gaze on srf as csv and pickle file + # gaze_on_ref + # with open(os.path.join(srf_dir,'gaze_positions_on_surface','wb')) as csvfile: + # csw_writer = csv.writer(csvfile, delimiter=' ',quotechar='|', quoting=csv.QUOTE_MINIMAL) + # csw_writer.writerow(('world_frame_idx','world_timestamp','eye_timestamp','x_norm','y_norm','x_scaled','y_scaled')) + # for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache) + # if ref_srf_data is not None: + # csw_writer.writerow((idx,ts,ref_srf_data['m_to_screen'],ref_srf_data['m_from_screen'],ref_srf_data['detected_markers']) + # #save gaze on srf as csv and pickle file \ No newline at end of file From 7e9885fea760d23ab48ae4aada853920b53d8a6c Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Wed, 14 May 2014 10:23:30 +0200 Subject: [PATCH 31/41] more work on heatmaps, various bugfixes --- pupil_src/capture/world.py | 6 +- pupil_src/player/exporter.py | 2 +- pupil_src/player/main.py | 6 +- pupil_src/player/seek_bar.py | 16 ++- pupil_src/player/vis_polyline.py | 2 +- pupil_src/shared_modules/atb/raw.py | 1 + .../shared_modules/offline_marker_detector.py | 73 ++++++++-- .../offline_reference_surface.py | 130 +++++++++++------- pupil_src/shared_modules/plugin.py | 9 ++ pupil_src/shared_modules/reference_surface.py | 17 +-- 10 files changed, 179 insertions(+), 83 deletions(-) diff --git a/pupil_src/capture/world.py b/pupil_src/capture/world.py index 3a861c7cde..d61d820106 100644 --- a/pupil_src/capture/world.py +++ b/pupil_src/capture/world.py @@ -64,6 +64,8 @@ def on_resize(window,w, h): atb.TwWindowSize(*map(int,fb_size)) adjust_gl_view(w,h,window) glfwMakeContextCurrent(active_window) + for p in g.plugins: + p.on_window_resize(window,w,h) def on_iconify(window,iconfied): if not isinstance(cap,FakeCapture): @@ -123,8 +125,8 @@ def map_pupil(vector): """ return vector - # any object we attach to the g_pool object now will only be visible to this process! - # vars should be declared here to make them visible to the reader. + # any object we attach to the g_pool object *from now on* will only be visible to this process! + # vars should be declared here to make them visible to the code reader. g_pool.plugins = [] g_pool.map_pupil = map_pupil g_pool.update_textures = c_bool(1) diff --git a/pupil_src/player/exporter.py b/pupil_src/player/exporter.py index d9ac65f3de..3f658d92b0 100644 --- a/pupil_src/player/exporter.py +++ b/pupil_src/player/exporter.py @@ -146,7 +146,7 @@ def export(should_terminate,frames_to_export,current_frame, data_dir,start_frame p = plugin_by_name[name](g,**args) plugins.append(p) except: - logger.warning("Plugin '%s' failed to load." %name) + logger.warning("Plugin '%s' could not be loaded in exporter." %name) while frames_to_export.value - current_frame.value > 0: diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index 6b65b7162e..15e1dce26d 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -130,6 +130,8 @@ def on_resize(window,w, h): fb_size = denormalize(norm_size,glfwGetFramebufferSize(window)) atb.TwWindowSize(*map(int,fb_size)) glfwMakeContextCurrent(active_window) + for p in g.plugins: + p.on_window_resize(window,w,h) def on_key(window, key, scancode, action, mods): if not atb.TwEventKeyboardGLFW(key,action): @@ -160,14 +162,14 @@ def on_scroll(window,x,y): def on_close(window): glfwSetWindowShouldClose(main_window,True) - logger.info('Process closing from window') + logger.debug('Process closing from window') try: rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: - rec_dir = '/home/mkassner/Desktop/002' + rec_dir = '/home/mkassner/Desktop/007' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: diff --git a/pupil_src/player/seek_bar.py b/pupil_src/player/seek_bar.py index bbb09047f7..e5f56422c8 100644 --- a/pupil_src/player/seek_bar.py +++ b/pupil_src/player/seek_bar.py @@ -37,7 +37,16 @@ def __init__(self, g_pool,capture): self.drag_mode = False self.was_playing = True #display layout - self.padding = 20. + self.padding = 20. #in sceen pixel + + + def init_gui(self): + self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext())) + + def on_window_resize(self,window,w,h): + width,height = w,h + self.h_pad = self.padding/width + self.v_pad = self.padding/height def update(self,frame,recent_pupil_positions,events): self.current_frame_index = frame.index @@ -105,10 +114,7 @@ def gl_display(self): glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() - width,height = glfwGetWindowSize(glfwGetCurrentContext()) - h_pad = self.padding/width - v_pad = self.padding/height - gluOrtho2D(-h_pad, 1+h_pad, -v_pad, 1+v_pad) # gl coord convention + gluOrtho2D(-self.h_pad, 1+self.h_pad, -self.v_pad, 1+self.v_pad) # gl coord convention glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() diff --git a/pupil_src/player/vis_polyline.py b/pupil_src/player/vis_polyline.py index 250f2f1d9f..3d6f42ef38 100644 --- a/pupil_src/player/vis_polyline.py +++ b/pupil_src/player/vis_polyline.py @@ -48,7 +48,7 @@ def init_gui(self,pos=None): self._bar = atb.Bar(name = self.__class__.__name__+str(id(self)), label=atb_label, help="polyline", color=(50, 50, 50), alpha=100, text='light', position=pos,refresh=.1, size=self.gui_settings['size']) - + self._bar.iconified = self.gui_settings['iconified'] self._bar.add_var('color',self.color) self._bar.add_var('thickness',self.thickness,min=1) self._bar.add_button('remove',self.unset_alive) diff --git a/pupil_src/shared_modules/atb/raw.py b/pupil_src/shared_modules/atb/raw.py index 073ba94621..38c912370e 100644 --- a/pupil_src/shared_modules/atb/raw.py +++ b/pupil_src/shared_modules/atb/raw.py @@ -65,6 +65,7 @@ TwMouseMotion = __dll__.TwMouseMotion TwWindowSize = __dll__.TwWindowSize TwMouseWheel = __dll__.TwMouseWheel +TwSetCurrentWindow = __dll__.TwSetCurrentWindow TwEventMouseButtonGLFW = __dll__.TwEventMouseButtonGLFW # TwEventMouseMotionGLUT = __dll__.TwEventMousePosGLFW diff --git a/pupil_src/shared_modules/offline_marker_detector.py b/pupil_src/shared_modules/offline_marker_detector.py index 68b0d1604b..91550b58c9 100644 --- a/pupil_src/shared_modules/offline_marker_detector.py +++ b/pupil_src/shared_modules/offline_marker_detector.py @@ -11,6 +11,7 @@ import sys, os,platform import cv2 import numpy as np +import csv if platform.system() == 'Darwin': @@ -26,7 +27,7 @@ from OpenGL.GL import * from OpenGL.GLU import gluOrtho2D from methods import normalize,denormalize -from file_methods import Persistent_Dict +from file_methods import Persistent_Dict,save_object from cache_list import Cache_List from glfw import * import atb @@ -105,11 +106,14 @@ def init_gui(self): pos = self.gui_settings['pos'] atb_label = "Marker Detector" self._bar = atb.Bar(name =self.__class__.__name__+str(id(self)), label=atb_label, - help="circle", color=(150, 150, 50), alpha=50, + help="circle", color=(50, 150, 50), alpha=50, text='light', position=pos,refresh=.1, size=self.gui_settings['size']) self._bar.iconified = self.gui_settings['iconified'] self.update_bar_markers() + #set up bar display padding + self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext())) + def unset_alive(self): self.alive = False @@ -119,6 +123,9 @@ def load(self, var_name, default): def save(self, var_name, var): self.surface_definitions[var_name] = var + def on_window_resize(self,window,w,h): + self.win_size = w,h + def on_click(self,pos,button,action): if self.surface_edit_mode.value: @@ -156,12 +163,13 @@ def update_bar_markers(self): self._bar.add_button(" add surface ", self.add_surface, key='a') self._bar.add_var(" edit mode ", self.surface_edit_mode ) for s,i in zip(self.surfaces,range(len(self.surfaces)))[::-1]: - self._bar.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window') self._bar.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') self._bar.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) self._bar.add_var("%s_x_scale"%i,vtype=c_float, getter=s.atb_get_scale_x, min=1,setter=s.atb_set_scale_x,group=str(i),label='scale factor x', help='the scale factor is used to adjust the coordinate space for your needs (think photo pixels or mm or whatever)' ) self._bar.add_var("%s_y_scale"%i,vtype=c_float, getter=s.atb_get_scale_y,min=1,setter=s.atb_set_scale_y,group=str(i),label='scale factor y',help='defining x and y scale factor you atumatically set the correct aspect ratio.' ) - self._bar.add_button("%s_export"%i, s.save_surface_positions_to_file,label='export surface data',group=str(i)) + self._bar.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window') + self._bar.add_button("%s_hm"%i, s.generate_heatmap, label='generate_heatmap',group=str(i)) + self._bar.add_button("%s_export"%i, self.save_surface_positions_to_file,data=i, label='export surface data',group=str(i)) self._bar.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) def update(self,frame,recent_pupil_positions,events): @@ -196,7 +204,6 @@ def update(self,frame,recent_pupil_positions,events): new_pos = s.img_to_ref_surface(np.array(pos)) s.move_vertex(v_idx,new_pos) s.cache = None - s.gaze_cache = None else: # update srf with no or invald cache: for s in self.surfaces: @@ -282,10 +289,12 @@ def gl_display_cache_bars(self): glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() - width,height = glfwGetWindowSize(glfwGetCurrentContext()) + width,height = self.win_size h_pad = padding * (self.cache.length-2)/float(width) v_pad = padding* 1./(height-2) gluOrtho2D(-h_pad, (self.cache.length-1)+h_pad, -v_pad, 1+v_pad) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) + + glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() @@ -305,6 +314,56 @@ def gl_display_cache_bars(self): glPopMatrix() + def save_surface_positions_to_file(self,i): + s = self.surfaces[i] + + if s.cache == None: + logger.warning("The surface is not cached. Please wait for the cacher to collect data.") + return + + srf_dir = os.path.join(self.g_pool.rec_dir,'surface_data'+'_'+s.name.replace('/','')+'_'+s.uid) + logger.info("exporting surface gaze data to %s"%srf_dir) + if os.path.isdir(srf_dir): + logger.info("Will overwrite previous export for this referece surface") + else: + try: + os.mkdir(srf_dir) + except: + logger.warning("Could name make export dir %s"%srf_dir) + return + + #save surface_positions as pickle file + save_object(s.cache.to_list(),os.path.join(srf_dir,'srf_positons')) + + + #save surface_positions as csv + with open(os.path.join(srf_dir,'srf_positons.csv'),'wb') as csvfile: + csw_writer =csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL) + csw_writer.writerow(('frame_idx','timestamp','m_to_screen','m_from_screen','detected_markers')) + for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache): + if ref_srf_data is not None and ref_srf_data is not False: + csw_writer.writerow( (idx,ts,ref_srf_data['m_to_screen'],ref_srf_data['m_from_screen'],ref_srf_data['detected_markers']) ) + + + #save gaze on srf as csv. + with open(os.path.join(srf_dir,'gaze_positions_on_surface.csv'),'wb') as csvfile: + csw_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL) + csw_writer.writerow(('world_frame_idx','world_timestamp','eye_timestamp','x_norm','y_norm','x_scaled','y_scaled','on_srf')) + for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache): + if ref_srf_data is not None and ref_srf_data is not False: + for gp in ref_srf_data['gaze_on_srf']: + gp_x,gp_y = gp['norm_gaze_on_srf'] + on_srf = (0 <= gp_x <= 1) and (0 <= gp_y <= 1) + csw_writer.writerow( (idx,ts,gp['timestamp'],gp_x,gp_y,gp_x*s.scale_factor[0],gp_x*s.scale_factor[1],on_srf) ) + + logger.info("Saved surface positon data and gaze on surface data for '%s' with uid:'%s'"%(s.name,s.uid)) + + if s.heatmap is not None: + logger.info("Saved Heatmap as .png file.") + cv2.imwrite(os.path.join(srf_dir,'heatmap.png'),s.heatmap) + + + def get_init_dict(self): d = {} @@ -314,7 +373,6 @@ def get_init_dict(self): return d - def cleanup(self): """ called when the plugin gets terminated. This happends either voluntary or forced. @@ -331,4 +389,3 @@ def cleanup(self): for s in self.surfaces: s.close_window() self._bar.destroy() - self._bar.destroy() diff --git a/pupil_src/shared_modules/offline_reference_surface.py b/pupil_src/shared_modules/offline_reference_surface.py index 85730d928a..792b06a663 100644 --- a/pupil_src/shared_modules/offline_reference_surface.py +++ b/pupil_src/shared_modules/offline_reference_surface.py @@ -10,7 +10,7 @@ import numpy as np import cv2 -from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture +from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture,create_named_texture from glfw import * from OpenGL.GL import * from OpenGL.GLU import gluOrtho2D @@ -33,8 +33,11 @@ def __init__(self,name="unnamed",saved_definition=None, gaze_positions_by_frame super(Offline_Reference_Surface, self).__init__(name,saved_definition) self.gaze_positions_by_frame = gaze_positions_by_frame self.cache = None + self.gaze_on_srf = [] # points on surface for realtime feedback display - + self.heatmap_detail = .1 + self.heatmap = None + self.heatmap_texture = None #cache fn for offline marker def locate_from_cache(self,frame_idx): if self.cache == None: @@ -56,7 +59,7 @@ def locate_from_cache(self,frame_idx): self.m_from_screen = cache_result['m_from_screen'] self.m_to_screen = cache_result['m_to_screen'] self.detected_markers = cache_result['detected_markers'] - self.gaze_on_srf = [gp['norm_gaze_on_srf'] for gp in cache_result['gaze_on_srf'] ] + self.gaze_on_srf = cache_result['gaze_on_srf'] return True raise Exception("Invalid cache entry. Please report Bug.") @@ -133,56 +136,81 @@ def gaze_on_srf_by_frame_idx(self,frame_index,m_from_screen): return gaze_on_src - # def map_gaze_by_frame_onto_srf(self,gaze_by_frame): - # ''' - # this fn is not cached and can be slow, dont call every frame... - # ''' - # gaze_on_srf_by_frame = [[] for i in gaze_by_frame] + #### fns to draw surface in seperate window + def gl_display_in_window(self,world_tex_id): + """ + here we map a selected surface onto a seperate window. + """ + if self._window and self.detected: + active_window = glfwGetCurrentContext() + glfwMakeContextCurrent(self._window) + clear_gl_screen() - # if self.cache: - # for surface_data,gaze_points,result in zip(self.cache,gaze_by_frame,gaze_on_srf_by_frame): - # if surface_data and gaze_points: - # gaze_points = np.array([d['norm_gaze'] for d in gaze_points if d is not None]) - # gaze_points.shape = (-1,1,2) - # gaze_points_on_srf = cv2.perspectiveTransform(gaze_points , surface_data['m_from_screen'] ) - # gaze_points_on_srf.shape = (-1,2) - # result.extend(gaze_points_on_srf.tolist()) - # return gaze_on_srf_by_frame + # cv uses 3x3 gl uses 4x4 tranformation matricies + m = cvmat_to_glmat(self.m_from_screen) + glMatrixMode(GL_PROJECTION) + glPushMatrix() + glLoadIdentity() + gluOrtho2D(0, 1, 0, 1) # gl coord convention - def save_surface_positions_to_file(self,s): - if s.cache == None: - logger.warning("The surface is not cached. Please wait for the cacher to collect data.") - return + glMatrixMode(GL_MODELVIEW) + glPushMatrix() + #apply m to our quad - this will stretch the quad such that the ref suface will span the window extends + glLoadMatrixf(m) - srf_dir = os.path.join(self.g_pool.rec_dir,'surface_data',s.name.replace('/',''),s.uid) - logger.info("exporting surface gaze data to %s"%srf_dir) - if os.path.isdir(srf_dir): - logger.info("Will overwrite previous export for this referece surface") - else: - try: - os.mkdir(srf_dir) - except: - logger.warning("Could name make export dir %s"%srf_dir) - return - - # logger.info("Saving surface positon data and gaze on surface data for '%s' with uid:'%'"%(s.name,s.uid)) - # #save surface_positions as pickle file - # save_object(s.cache,os.path.join(srf_dir,'srf_positons_by_frame')) - # #save surface_positions as csv - # with open(os.path.join(srf_dir,'srf_positons_by_frame.csv','wb')) as csvfile: - # csw_writer =csv.writer(csvfile, delimiter=' ',quotechar='|', quoting=csv.QUOTE_MINIMAL) - # csw_writer.writerow(('frame_idx','timestamp','m_to_screen','m_from_screen','detected_markers')) - # for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache) - # if ref_srf_data is not None: - # csw_writer.writerow( (idx,ts,ref_srf_data['m_to_screen'],ref_srf_data['m_from_screen'],ref_srf_data['detected_markers']) ) - # #save gaze on srf as csv and pickle file - # gaze_on_ref - # with open(os.path.join(srf_dir,'gaze_positions_on_surface','wb')) as csvfile: - # csw_writer = csv.writer(csvfile, delimiter=' ',quotechar='|', quoting=csv.QUOTE_MINIMAL) - # csw_writer.writerow(('world_frame_idx','world_timestamp','eye_timestamp','x_norm','y_norm','x_scaled','y_scaled')) - # for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache) - # if ref_srf_data is not None: - # csw_writer.writerow((idx,ts,ref_srf_data['m_to_screen'],ref_srf_data['m_from_screen'],ref_srf_data['detected_markers']) - # #save gaze on srf as csv and pickle file \ No newline at end of file + draw_named_texture(world_tex_id) + + glMatrixMode(GL_PROJECTION) + glPopMatrix() + glMatrixMode(GL_MODELVIEW) + glPopMatrix() + + + if self.heatmap_texture: + draw_named_texture(self.heatmap_texture) + + # now lets get recent pupil positions on this surface: + for gp in self.gaze_on_srf: + draw_gl_points_norm([gp['norm_gaze_on_srf']],color=(0.,8.,.5,.8), size=80) + + glfwSwapBuffers(self._window) + glfwMakeContextCurrent(active_window) + + + def generate_heatmap(self): + + x,y = self.scale_factor + x = max(1,int(x)) + y = max(1,int(y)) + + filter_size = (int(self.heatmap_detail * x)/2)*2 +1 + std_dev = filter_size /6. + self.heatmap = np.ones((y,x,4),dtype=np.uint8) + all_gaze = [] + for c_e in self.cache: + if c_e: + for gp in c_e['gaze_on_srf']: + all_gaze.append(gp['norm_gaze_on_srf']) + all_gaze = np.array(all_gaze) + all_gaze *= self.scale_factor + hist,xedge,yedge = np.histogram2d(all_gaze[:,0], all_gaze[:,1], + bins=[x,y], + range=[[0, self.scale_factor[0]], [0,self.scale_factor[1]]], + normed=False, + weights=None) + + + hist = np.rot90(hist) + + #smoothing.. + hist = cv2.GaussianBlur(hist, (filter_size,filter_size),std_dev) + hist = np.uint8( hist*(255./np.amax(hist) ) ) + + #colormapping + c_map = cv2.applyColorMap(hist, cv2.COLORMAP_JET) + + self.heatmap[:,:,:3] = c_map + self.heatmap[:,:,3] = 125 + self.heatmap_texture = create_named_texture(self.heatmap) diff --git a/pupil_src/shared_modules/plugin.py b/pupil_src/shared_modules/plugin.py index 9ea003f4f5..4bd067e9f2 100644 --- a/pupil_src/shared_modules/plugin.py +++ b/pupil_src/shared_modules/plugin.py @@ -53,6 +53,13 @@ def on_click(self,pos,button,action): """ pass + def on_window_resize(self,window,w,h): + ''' + gets called when user resizes window. + window is the glfw window handle of the resized window. + ''' + pass + def update(self,frame,recent_pupil_positions,events): """ gets called once every frame @@ -61,6 +68,8 @@ def update(self,frame,recent_pupil_positions,events): """ pass + + def gl_display(self): """ gets called once every frame when its time to draw onto the gl canvas. diff --git a/pupil_src/shared_modules/reference_surface.py b/pupil_src/shared_modules/reference_surface.py index 0aed6252d6..b7148c8f43 100644 --- a/pupil_src/shared_modules/reference_surface.py +++ b/pupil_src/shared_modules/reference_surface.py @@ -10,7 +10,7 @@ import numpy as np import cv2 -from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture +from gl_utils import draw_gl_polyline,adjust_gl_view,draw_gl_polyline_norm,clear_gl_screen,draw_gl_point,draw_gl_points,draw_gl_point_norm,draw_gl_points_norm,basic_gl_setup,cvmat_to_glmat, draw_named_texture,make_coord_system_norm_based from glfw import * from OpenGL.GL import * from OpenGL.GLU import gluOrtho2D @@ -249,17 +249,6 @@ def ref_surface_to_img(self,pos): else: return None - def gaze_on_srf(self,gaze_positions,m_from_screen): - if self.m_from_screen is None: - return None - gaze_on_src = [] - for g_p in gaze_positions: - gaze_points = np.array([g_p['norm_gaze']]).reshape(1,1,2) - gaze_points_on_srf = cv2.perspectiveTransform(gaze_points , m_from_screen ) - gaze_points_on_srf.shape = (2) - gaze_on_src.append( {'norm_gaze_on_srf':(gaze_points_on_srf[0],gaze_points_on_srf[1]),'timestamp':g_p['timestamp'] } ) - return gaze_on_src - def move_vertex(self,vert_idx,new_pos): """ @@ -321,7 +310,7 @@ def gl_draw_corners(self): - #### fns to draw surface in seperate window + #### fns to draw surface in separate window def gl_display_in_window(self,world_tex_id): """ here we map a selected surface onto a seperate window. @@ -392,6 +381,8 @@ def open_window(self): active_window = glfwGetCurrentContext() glfwMakeContextCurrent(self._window) basic_gl_setup() + make_coord_system_norm_based() + # refresh speed settings glfwSwapInterval(0) From 1bfe591240f3e6713a90c344ee75be5883b60437 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Wed, 14 May 2014 12:23:35 +0200 Subject: [PATCH 32/41] adding image exports to sruface export function. --- .../shared_modules/offline_marker_detector.py | 20 +++++++++++++++++++ pupil_src/shared_modules/reference_surface.py | 4 +++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/offline_marker_detector.py b/pupil_src/shared_modules/offline_marker_detector.py index 91550b58c9..92882ecf0c 100644 --- a/pupil_src/shared_modules/offline_marker_detector.py +++ b/pupil_src/shared_modules/offline_marker_detector.py @@ -99,6 +99,7 @@ def __init__(self,g_pool,gui_settings={'pos':(220,200),'size':(300,300),'iconifi self.recent_pupil_positions = [] self.img_shape = None + self.img = None def init_gui(self): @@ -173,6 +174,7 @@ def update_bar_markers(self): self._bar.add_button("%s_remove"%i, self.remove_surface,data=i,label='remove',group=str(i)) def update(self,frame,recent_pupil_positions,events): + self.img = frame.img self.img_shape = frame.img.shape self.update_marker_cache() self.markers = self.cache[frame.index] @@ -362,6 +364,24 @@ def save_surface_positions_to_file(self,i): logger.info("Saved Heatmap as .png file.") cv2.imwrite(os.path.join(srf_dir,'heatmap.png'),s.heatmap) + if s.detected and self.img is not None: + #let save out the current surface image found in video + + #here we get the verts of the surface quad in norm_coords + mapped_space_one = np.array(((0,0),(1,0),(1,1),(0,1)),dtype=np.float32).reshape(-1,1,2) + screen_space = cv2.perspectiveTransform(mapped_space_one,s.m_to_screen).reshape(-1,2) + #now we convert to image pixel coods + screen_space[:,1] = 1-screen_space[:,1] + screen_space[:,1] *= self.img.shape[0] + screen_space[:,0] *= self.img.shape[1] + s_0,s_1 = s.scale_factor + #no we need to flip vertically again by setting the mapped_space verts accordingly. + mapped_space_scaled = np.array(((0,s_1),(s_0,s_1),(s_0,0),(0,0)),dtype=np.float32) + M = cv2.getPerspectiveTransform(screen_space,mapped_space_scaled) + #here we do the actual perspactive transform of the image. + srf_in_video = cv2.warpPerspective(self.img,M, (int(s.scale_factor[0]),int(s.scale_factor[1])) ) + cv2.imwrite(os.path.join(srf_dir,'surface.png'),srf_in_video) + logger.info("Saved current image as .png file.") diff --git a/pupil_src/shared_modules/reference_surface.py b/pupil_src/shared_modules/reference_surface.py index b7148c8f43..c3d95ddbd1 100644 --- a/pupil_src/shared_modules/reference_surface.py +++ b/pupil_src/shared_modules/reference_surface.py @@ -35,6 +35,7 @@ def m_verts_from_screen(verts): return cv2.getPerspectiveTransform(verts,mapped_space_one) + class Reference_Surface(object): """docstring for Reference Surface @@ -97,6 +98,7 @@ def __init__(self,name="unnamed",saved_definition=None): self.gaze_on_srf = [] # points on surface for realtime feedback display + def save_to_dict(self): """ save all markers and name of this surface to a dict. @@ -226,7 +228,6 @@ def locate(self, visible_markers): self.m_to_screen = None - def img_to_ref_surface(self,pos): if self.m_from_screen is not None: #convenience lines to allow 'simple' vectors (x,y) to be used @@ -250,6 +251,7 @@ def ref_surface_to_img(self,pos): return None + def move_vertex(self,vert_idx,new_pos): """ this fn is used to manipulate the surface boundary (coordinate system) From a4d7c6c3000bc5418134e48ebc718b629bb6b8b7 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Thu, 15 May 2014 18:54:46 +0200 Subject: [PATCH 33/41] adding trim marks --- pupil_src/player/export_launcher.py | 10 +- pupil_src/player/main.py | 6 +- pupil_src/player/seek_bar.py | 51 +++--- pupil_src/player/trim_marks.py | 147 ++++++++++++++++++ .../shared_modules/offline_marker_detector.py | 26 ++-- .../offline_reference_surface.py | 27 +++- 6 files changed, 214 insertions(+), 53 deletions(-) create mode 100644 pupil_src/player/trim_marks.py diff --git a/pupil_src/player/export_launcher.py b/pupil_src/player/export_launcher.py index cd95adfd1b..f2933068ec 100644 --- a/pupil_src/player/export_launcher.py +++ b/pupil_src/player/export_launcher.py @@ -74,8 +74,6 @@ def __init__(self, g_pool,data_dir,frame_count): default_path = "world_viz.avi" self.rec_name = create_string_buffer(default_path,512) - self.start_frame = c_int(0) - self.end_frame = c_int(frame_count) def init_gui(self): @@ -96,8 +94,8 @@ def update_bar(self): self._bar.clear() self._bar.add_var('export name',self.rec_name, help="Supply export video recording name. The export will be in the recording dir. If you give a path the export will end up there instead.") - self._bar.add_var('start frame',self.start_frame,help="Supply start frame no. Negative numbers will count from the end. The behaves like python list indexing") - self._bar.add_var('end frame',self.end_frame,help="Supply end frame no. Negative numbers will count from the end. The behaves like python list indexing") + self._bar.add_var('start frame',vtype=c_int,getter=self.g_pool.trim_marks.atb_get_in_mark,setter= self.g_pool.trim_marks.atb_set_in_mark, help="Supply start frame no. Negative numbers will count from the end. The behaves like python list indexing") + self._bar.add_var('end frame',vtype=c_int,getter=self.g_pool.trim_marks.atb_get_out_mark,setter= self.g_pool.trim_marks.atb_set_out_mark,help="Supply end frame no. Negative numbers will count from the end. The behaves like python list indexing") self._bar.add_button('new export',self.add_export) for job,i in zip(self.exports,range(len(self.exports)))[::-1]: @@ -134,8 +132,8 @@ def add_export(self): current_frame = RawValue(c_int,0) data_dir = self.data_dir - start_frame= self.start_frame.value - end_frame= self.end_frame.value + start_frame= self.g_pool.trim_marks.in_mark + end_frame= self.g_pool.trim_marks.out_mark+1 #end_frame is exclusive plugins = [] # Here we make clones of every plugin that supports it. diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index 15e1dce26d..50476d0c59 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -105,6 +105,7 @@ from display_gaze import Display_Gaze from vis_light_points import Vis_Light_Points from seek_bar import Seek_Bar +from trim_marks import Trim_Marks from export_launcher import Export_Launcher from scan_path import Scan_Path from offline_marker_detector import Offline_Marker_Detector @@ -249,6 +250,7 @@ def save(var_name,var): g.plugins = [] g.play = False g.new_seek = True + g.trim_marks = [0,cap.get_frame_count()] g.user_dir = user_dir g.rec_dir = rec_dir g.app = 'player' @@ -275,7 +277,7 @@ def get_from_data(data): """ helper for atb getter and setter use """ - return data.value + return data.valuescale def get_play(): return g.play @@ -359,6 +361,8 @@ def get_from_data(data): #we always load these plugins g.plugins.append(Export_Launcher(g,data_dir=rec_dir,frame_count=len(timestamps))) g.plugins.append(Seek_Bar(g,capture=cap)) + g.trim_marks = Trim_Marks(g,capture=cap) + g.plugins.append(g.trim_marks) #these are loaded based on user settings for initializer in load('plugins',[]): diff --git a/pupil_src/player/seek_bar.py b/pupil_src/player/seek_bar.py index e5f56422c8..a79b831889 100644 --- a/pupil_src/player/seek_bar.py +++ b/pupil_src/player/seek_bar.py @@ -14,9 +14,7 @@ from glfw import glfwGetWindowSize,glfwGetCurrentContext,glfwGetCursorPos,GLFW_RELEASE,GLFW_PRESS from plugin import Plugin -import numpy as np -from methods import denormalize import logging logger = logging.getLogger(__name__) @@ -24,7 +22,6 @@ class Seek_Bar(Plugin): """docstring for Seek_Bar seek bar displays a bar at the bottom of the screen when you hover close to it. it will show the current positon and allow you to drag to any postion in the video file. - """ def __init__(self, g_pool,capture): super(Seek_Bar, self).__init__() @@ -33,7 +30,6 @@ def __init__(self, g_pool,capture): self.current_frame_index = self.cap.get_frame_index() self.frame_count = self.cap.get_frame_count() - self.norm_seek_pos = self.current_frame_index/float(self.frame_count) self.drag_mode = False self.was_playing = True #display layout @@ -44,22 +40,22 @@ def init_gui(self): self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext())) def on_window_resize(self,window,w,h): - width,height = w,h - self.h_pad = self.padding/width - self.v_pad = self.padding/height + self.window_size = w,h + self.h_pad = self.padding * self.frame_count/float(w) + self.v_pad = self.padding * 1./h def update(self,frame,recent_pupil_positions,events): self.current_frame_index = frame.index - self.norm_seek_pos = self.current_frame_index/float(self.frame_count) if self.drag_mode: - pos = glfwGetCursorPos(glfwGetCurrentContext()) - norm_seek_pos, _ = self.screen_to_seek_bar(pos) - norm_seek_pos = min(1,max(0,norm_seek_pos)) - if abs(norm_seek_pos-self.norm_seek_pos) >=.01: - seek_pos = min(int(norm_seek_pos*self.frame_count),self.frame_count-5) #the last frames can be problematic to seek to + x,y = glfwGetCursorPos(glfwGetCurrentContext()) + x,_ = self.screen_to_seek_bar((x,y)) + seek_pos = min(self.frame_count,max(0,x)) + if abs(seek_pos-self.current_frame_index) >=.002*self.frame_count: + seek_pos = int(min(seek_pos,self.frame_count-5)) #the last frames can be problematic to seek to try: self.cap.seek_to_frame(seek_pos) + self.current_frame_index = seek_pos except: pass self.g_pool.new_seek = True @@ -72,7 +68,7 @@ def on_click(self,img_pos,button,action): pos = glfwGetCursorPos(glfwGetCurrentContext()) #drag the seek point if action == GLFW_PRESS: - screen_seek_pos = self.seek_bar_to_screen((self.norm_seek_pos,0)) + screen_seek_pos = self.seek_bar_to_screen((self.current_frame_index,0)) dist = abs(pos[0]-screen_seek_pos[0])+abs(pos[1]-screen_seek_pos[1]) if dist < 20: self.drag_mode=True @@ -81,11 +77,10 @@ def on_click(self,img_pos,button,action): elif action == GLFW_RELEASE: if self.drag_mode: - norm_seek_pos, _ = self.screen_to_seek_bar(pos) - norm_seek_pos = min(1,max(0,norm_seek_pos)) - seek_pos = min(int(norm_seek_pos*self.frame_count),self.frame_count-5) + x, _ = self.screen_to_seek_bar(pos) + x = int(min(self.frame_count-5,max(0,x))) try: - self.cap.seek_to_frame(seek_pos) + self.cap.seek_to_frame(x) except: pass self.g_pool.new_seek = True @@ -94,10 +89,10 @@ def on_click(self,img_pos,button,action): def seek_bar_to_screen(self,pos): - width,height = glfwGetWindowSize(glfwGetCurrentContext()) + width,height = self.window_size x,y=pos y = 1-y - x = x*(width-2*self.padding)+self.padding + x = (x/float(self.frame_count))*(width-self.padding*2) +self.padding y = y*(height-2*self.padding)+self.padding return x,y @@ -105,7 +100,7 @@ def seek_bar_to_screen(self,pos): def screen_to_seek_bar(self,pos): width,height = glfwGetWindowSize(glfwGetCurrentContext()) x,y=pos - x = (x-self.padding)/(width-2*self.padding) + x = (x-self.padding)/(width-2*self.padding)*self.frame_count y = (y-self.padding)/(height-2*self.padding) return x,1-y @@ -114,7 +109,7 @@ def gl_display(self): glMatrixMode(GL_PROJECTION) glPushMatrix() glLoadIdentity() - gluOrtho2D(-self.h_pad, 1+self.h_pad, -self.v_pad, 1+self.v_pad) # gl coord convention + gluOrtho2D(-self.h_pad, (self.frame_count)+self.h_pad, -self.v_pad, 1+self.v_pad) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) glMatrixMode(GL_MODELVIEW) glPushMatrix() glLoadIdentity() @@ -126,14 +121,12 @@ def gl_display(self): color1 = (.25,.8,.8,.5) color2 = (.25,.8,.8,1.) - draw_gl_polyline( [(0,0),(self.norm_seek_pos,0)],color=color1) - draw_gl_polyline( [(self.norm_seek_pos,0),(1,0)],color=(.5,.5,.5,.5)) - draw_gl_point((self.norm_seek_pos,0),color=color1,size=40) - draw_gl_point((self.norm_seek_pos,0),color=color2,size=10) + draw_gl_polyline( [(0,0),(self.current_frame_index,0)],color=color1) + draw_gl_polyline( [(self.current_frame_index,0),(self.frame_count,0)],color=(.5,.5,.5,.5)) + draw_gl_point((self.current_frame_index,0),color=color1,size=40) + draw_gl_point((self.current_frame_index,0),color=color2,size=10) glMatrixMode(GL_PROJECTION) glPopMatrix() glMatrixMode(GL_MODELVIEW) - glPopMatrix() - - + glPopMatrix() \ No newline at end of file diff --git a/pupil_src/player/trim_marks.py b/pupil_src/player/trim_marks.py new file mode 100644 index 0000000000..b620f01387 --- /dev/null +++ b/pupil_src/player/trim_marks.py @@ -0,0 +1,147 @@ +''' +(*)~---------------------------------------------------------------------------------- + Pupil - eye tracking platform + Copyright (C) 2012-2014 Pupil Labs + + Distributed under the terms of the CC BY-NC-SA License. + License details are in the file license.txt, distributed as part of this software. +----------------------------------------------------------------------------------~(*) +''' + +from gl_utils import draw_gl_polyline,draw_gl_point +from OpenGL.GL import * +from OpenGL.GLU import gluOrtho2D + +from glfw import glfwGetWindowSize,glfwGetCurrentContext,glfwGetCursorPos,GLFW_RELEASE,GLFW_PRESS +from plugin import Plugin + +import logging +logger = logging.getLogger(__name__) + +class Trim_Marks(Plugin): + """docstring for Trim_Mark + """ + def __init__(self, g_pool,capture): + super(Trim_Marks, self).__init__() + self.order = .8 + self.g_pool = g_pool + self.frame_count = capture.get_frame_count() + self._in_mark = 0 + self._out_mark = self.frame_count + self.drag_in = False + self.drag_out = False + #display layout + self.padding = 20. #in sceen pixel + + @property + def in_mark(self): + return self._in_mark + + @in_mark.setter + def in_mark(self, value): + self._in_mark = int(min(self._out_mark,max(0,value))) + + @property + def out_mark(self): + return self._out_mark + + @out_mark.setter + def out_mark(self, value): + self._out_mark = int(min(self.frame_count,max(self.in_mark,value))) + + + def init_gui(self): + self.on_window_resize(glfwGetCurrentContext(),*glfwGetWindowSize(glfwGetCurrentContext())) + + def on_window_resize(self,window,w,h): + self.window_size = w,h + self.h_pad = self.padding * self.frame_count/float(w) + self.v_pad = self.padding * 1./h + + def update(self,frame,recent_pupil_positions,events): + + if frame.index == self.out_mark or frame.index == self.in_mark: + self.g_pool.play=False + + if self.drag_in: + x,y = glfwGetCursorPos(glfwGetCurrentContext()) + x,_ = self.screen_to_bar_space((x,y)) + self.in_mark = x + + elif self.drag_out: + x,y = glfwGetCursorPos(glfwGetCurrentContext()) + x,_ = self.screen_to_bar_space((x,y)) + self.out_mark = x + + + def on_click(self,img_pos,button,action): + """ + gets called when the user clicks in the window screen + """ + pos = glfwGetCursorPos(glfwGetCurrentContext()) + #drag the seek point + if action == GLFW_PRESS: + screen_in_mark_pos = self.bar_space_to_screen((self.in_mark,0)) + screen_out_mark_pos = self.bar_space_to_screen((self.out_mark,0)) + + #in mark + dist = abs(pos[0]-screen_in_mark_pos[0])+abs(pos[1]-screen_in_mark_pos[1]) + if dist < 10: + self.drag_in=True + return + #out mark + dist = abs(pos[0]-screen_out_mark_pos[0])+abs(pos[1]-screen_out_mark_pos[1]) + if dist < 10: + self.drag_out=True + + elif action == GLFW_RELEASE: + self.drag_out = False + self.drag_in = False + + def atb_get_in_mark(self): + return self.in_mark + def atb_get_out_mark(self): + return self.out_mark + def atb_set_in_mark(self,val): + self.in_mark = val + def atb_set_out_mark(self,val): + self.out_mark = val + + def bar_space_to_screen(self,pos): + width,height = self.window_size + x,y=pos + y = 1-y + x = (x/float(self.frame_count))*(width-self.padding*2) +self.padding + y = y*(height-2*self.padding)+self.padding + return x,y + + + def screen_to_bar_space(self,pos): + width,height = glfwGetWindowSize(glfwGetCurrentContext()) + x,y=pos + x = (x-self.padding)/(width-2*self.padding)*self.frame_count + y = (y-self.padding)/(height-2*self.padding) + return x,1-y + + def gl_display(self): + + glMatrixMode(GL_PROJECTION) + glPushMatrix() + glLoadIdentity() + gluOrtho2D(-self.h_pad, (self.frame_count)+self.h_pad, -self.v_pad, 1+self.v_pad) # ranging from 0 to cache_len-1 (horizontal) and 0 to 1 (vertical) + glMatrixMode(GL_MODELVIEW) + glPushMatrix() + glLoadIdentity() + + color1 = (.1,.9,.2,.5) + color2 = (.1,.9,.2,.5) + + if self.in_mark != 0 or self.out_mark != self.frame_count: + draw_gl_polyline( [(self.in_mark,0),(self.out_mark,0)],color=(.1,.9,.2,.5),thickness=2) + draw_gl_point((self.in_mark,0),color=color2,size=10) + draw_gl_point((self.out_mark,0),color=color2,size=10) + + glMatrixMode(GL_PROJECTION) + glPopMatrix() + glMatrixMode(GL_MODELVIEW) + glPopMatrix() \ No newline at end of file diff --git a/pupil_src/shared_modules/offline_marker_detector.py b/pupil_src/shared_modules/offline_marker_detector.py index 92882ecf0c..bea7589f1c 100644 --- a/pupil_src/shared_modules/offline_marker_detector.py +++ b/pupil_src/shared_modules/offline_marker_detector.py @@ -70,10 +70,10 @@ def __init__(self,g_pool,gui_settings={'pos':(220,200),'size':(300,300),'iconifi self.surface_definitions = Persistent_Dict(os.path.join(g_pool.rec_dir,'surface_definitions')) if self.load('offline_square_marker_surfaces',[]) != []: logger.debug("Found ref surfaces defined or copied in previous session.") - self.surfaces = [Offline_Reference_Surface(saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)] + self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)] elif self.load('offline_square_marker_surfaces',[]) != []: logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.") - self.surfaces = [Offline_Reference_Surface(saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] + self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] else: logger.debug("No surface defs found. Please define using GUI.") self.surfaces = [] @@ -149,7 +149,7 @@ def advance(self): pass def add_surface(self): - self.surfaces.append(Offline_Reference_Surface(gaze_positions_by_frame=self.g_pool.positions_by_frame)) + self.surfaces.append(Offline_Reference_Surface(self.g_pool,gaze_positions_by_frame=self.g_pool.positions_by_frame)) self.update_bar_markers() def remove_surface(self,i): @@ -319,6 +319,9 @@ def gl_display_cache_bars(self): def save_surface_positions_to_file(self,i): s = self.surfaces[i] + in_mark = self.g_pool.trim_marks.in_mark + out_mark = self.g_pool.trim_marks.out_mark + if s.cache == None: logger.warning("The surface is not cached. Please wait for the cacher to collect data.") return @@ -337,14 +340,14 @@ def save_surface_positions_to_file(self,i): #save surface_positions as pickle file save_object(s.cache.to_list(),os.path.join(srf_dir,'srf_positons')) - #save surface_positions as csv with open(os.path.join(srf_dir,'srf_positons.csv'),'wb') as csvfile: csw_writer =csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL) csw_writer.writerow(('frame_idx','timestamp','m_to_screen','m_from_screen','detected_markers')) for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache): - if ref_srf_data is not None and ref_srf_data is not False: - csw_writer.writerow( (idx,ts,ref_srf_data['m_to_screen'],ref_srf_data['m_from_screen'],ref_srf_data['detected_markers']) ) + if in_mark <= idx <= out_mark: + if ref_srf_data is not None and ref_srf_data is not False: + csw_writer.writerow( (idx,ts,ref_srf_data['m_to_screen'],ref_srf_data['m_from_screen'],ref_srf_data['detected_markers']) ) #save gaze on srf as csv. @@ -352,11 +355,12 @@ def save_surface_positions_to_file(self,i): csw_writer = csv.writer(csvfile, delimiter='\t',quotechar='|', quoting=csv.QUOTE_MINIMAL) csw_writer.writerow(('world_frame_idx','world_timestamp','eye_timestamp','x_norm','y_norm','x_scaled','y_scaled','on_srf')) for idx,ts,ref_srf_data in zip(range(len(self.g_pool.timestamps)),self.g_pool.timestamps,s.cache): - if ref_srf_data is not None and ref_srf_data is not False: - for gp in ref_srf_data['gaze_on_srf']: - gp_x,gp_y = gp['norm_gaze_on_srf'] - on_srf = (0 <= gp_x <= 1) and (0 <= gp_y <= 1) - csw_writer.writerow( (idx,ts,gp['timestamp'],gp_x,gp_y,gp_x*s.scale_factor[0],gp_x*s.scale_factor[1],on_srf) ) + if in_mark <= idx <= out_mark: + if ref_srf_data is not None and ref_srf_data is not False: + for gp in ref_srf_data['gaze_on_srf']: + gp_x,gp_y = gp['norm_gaze_on_srf'] + on_srf = (0 <= gp_x <= 1) and (0 <= gp_y <= 1) + csw_writer.writerow( (idx,ts,gp['timestamp'],gp_x,gp_y,gp_x*s.scale_factor[0],gp_x*s.scale_factor[1],on_srf) ) logger.info("Saved surface positon data and gaze on surface data for '%s' with uid:'%s'"%(s.name,s.uid)) diff --git a/pupil_src/shared_modules/offline_reference_surface.py b/pupil_src/shared_modules/offline_reference_surface.py index 792b06a663..629007b0f2 100644 --- a/pupil_src/shared_modules/offline_reference_surface.py +++ b/pupil_src/shared_modules/offline_reference_surface.py @@ -29,8 +29,9 @@ class Offline_Reference_Surface(Reference_Surface): """docstring for Offline_Reference_Surface""" - def __init__(self,name="unnamed",saved_definition=None, gaze_positions_by_frame = None): + def __init__(self,g_pool,name="unnamed",saved_definition=None, gaze_positions_by_frame = None): super(Offline_Reference_Surface, self).__init__(name,saved_definition) + self.g_pool = g_pool self.gaze_positions_by_frame = gaze_positions_by_frame self.cache = None self.gaze_on_srf = [] # points on surface for realtime feedback display @@ -181,6 +182,9 @@ def gl_display_in_window(self,world_tex_id): def generate_heatmap(self): + in_mark = self.g_pool.trim_marks.in_mark + out_mark = self.g_pool.trim_marks.out_mark + x,y = self.scale_factor x = max(1,int(x)) y = max(1,int(y)) @@ -189,10 +193,15 @@ def generate_heatmap(self): std_dev = filter_size /6. self.heatmap = np.ones((y,x,4),dtype=np.uint8) all_gaze = [] - for c_e in self.cache: - if c_e: - for gp in c_e['gaze_on_srf']: - all_gaze.append(gp['norm_gaze_on_srf']) + for idx,c_e in enumerate(self.cache): + if in_mark <= idx <= out_mark: + if c_e: + for gp in c_e['gaze_on_srf']: + all_gaze.append(gp['norm_gaze_on_srf']) + + if not all_gaze: + logger.warning("No gaze data on surface for heatmap found.") + all_gaze.append((-1.,-1.)) all_gaze = np.array(all_gaze) all_gaze *= self.scale_factor hist,xedge,yedge = np.histogram2d(all_gaze[:,0], all_gaze[:,1], @@ -206,7 +215,13 @@ def generate_heatmap(self): #smoothing.. hist = cv2.GaussianBlur(hist, (filter_size,filter_size),std_dev) - hist = np.uint8( hist*(255./np.amax(hist) ) ) + maxval = np.amax(hist) + if maxval: + scale = 255./maxval + else: + scale = 0 + + hist = np.uint8( hist*(scale) ) #colormapping c_map = cv2.applyColorMap(hist, cv2.COLORMAP_JET) From 8e980dbff60167c3a8d2dad745fe781fc1b3c46e Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Thu, 15 May 2014 18:57:02 +0200 Subject: [PATCH 34/41] typo fix --- pupil_src/player/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index 50476d0c59..eece769ffc 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -250,7 +250,6 @@ def save(var_name,var): g.plugins = [] g.play = False g.new_seek = True - g.trim_marks = [0,cap.get_frame_count()] g.user_dir = user_dir g.rec_dir = rec_dir g.app = 'player' @@ -277,7 +276,7 @@ def get_from_data(data): """ helper for atb getter and setter use """ - return data.valuescale + return data.value def get_play(): return g.play From 1b3ee086d054effac0ba21de90c051221c885550 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Fri, 16 May 2014 11:12:29 +0200 Subject: [PATCH 35/41] small gui tweak --- pupil_src/player/trim_marks.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/pupil_src/player/trim_marks.py b/pupil_src/player/trim_marks.py index b620f01387..c357318435 100644 --- a/pupil_src/player/trim_marks.py +++ b/pupil_src/player/trim_marks.py @@ -25,6 +25,7 @@ def __init__(self, g_pool,capture): super(Trim_Marks, self).__init__() self.order = .8 self.g_pool = g_pool + self.capture = capture self.frame_count = capture.get_frame_count() self._in_mark = 0 self._out_mark = self.frame_count @@ -87,12 +88,14 @@ def on_click(self,img_pos,button,action): #in mark dist = abs(pos[0]-screen_in_mark_pos[0])+abs(pos[1]-screen_in_mark_pos[1]) if dist < 10: - self.drag_in=True - return + if self.distance_in_pix(self.in_mark,self.capture.get_frame_index()) > 20: + self.drag_in=True + return #out mark dist = abs(pos[0]-screen_out_mark_pos[0])+abs(pos[1]-screen_out_mark_pos[1]) if dist < 10: - self.drag_out=True + if self.distance_in_pix(self.out_mark,self.capture.get_frame_index()) > 20: + self.drag_out=True elif action == GLFW_RELEASE: self.drag_out = False @@ -107,6 +110,12 @@ def atb_set_in_mark(self,val): def atb_set_out_mark(self,val): self.out_mark = val + def distance_in_pix(self,frame_pos_0,frame_pos_1): + fr0_screen_x,_ = self.bar_space_to_screen((frame_pos_0,0)) + fr1_screen_x,_ = self.bar_space_to_screen((frame_pos_1,0)) + return abs(fr0_screen_x-fr1_screen_x) + + def bar_space_to_screen(self,pos): width,height = self.window_size x,y=pos From 00866324bbaa9ff6085d9ce45792b4c97f022f2a Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Mon, 19 May 2014 16:43:35 +0200 Subject: [PATCH 36/41] typo fix --- pupil_src/capture/world.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pupil_src/capture/world.py b/pupil_src/capture/world.py index d61d820106..7f8d1e2f59 100644 --- a/pupil_src/capture/world.py +++ b/pupil_src/capture/world.py @@ -64,7 +64,7 @@ def on_resize(window,w, h): atb.TwWindowSize(*map(int,fb_size)) adjust_gl_view(w,h,window) glfwMakeContextCurrent(active_window) - for p in g.plugins: + for p in g_pool.plugins: p.on_window_resize(window,w,h) def on_iconify(window,iconfied): From 3524ef3290301d313af5e3abe3a836bea8b4365f Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Tue, 20 May 2014 11:29:50 +0200 Subject: [PATCH 37/41] small fixes to loading definitions --- pupil_src/shared_modules/marker_detector.py | 3 +-- pupil_src/shared_modules/offline_marker_detector.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pupil_src/shared_modules/marker_detector.py b/pupil_src/shared_modules/marker_detector.py index e53f6fcdf4..66706403c6 100644 --- a/pupil_src/shared_modules/marker_detector.py +++ b/pupil_src/shared_modules/marker_detector.py @@ -201,8 +201,7 @@ def cleanup(self): This happends either voluntary or forced. if you have an atb bar or glfw window destroy it here. """ - if self.g_pool.app == 'capture': - self.save("realtime_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) + self.save("realtime_square_marker_surfaces",[rs.save_to_dict() for rs in self.surfaces if rs.defined]) self.surface_definitions.close() diff --git a/pupil_src/shared_modules/offline_marker_detector.py b/pupil_src/shared_modules/offline_marker_detector.py index bea7589f1c..8c39192647 100644 --- a/pupil_src/shared_modules/offline_marker_detector.py +++ b/pupil_src/shared_modules/offline_marker_detector.py @@ -71,7 +71,7 @@ def __init__(self,g_pool,gui_settings={'pos':(220,200),'size':(300,300),'iconifi if self.load('offline_square_marker_surfaces',[]) != []: logger.debug("Found ref surfaces defined or copied in previous session.") self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('offline_square_marker_surfaces',[]) if isinstance(d,dict)] - elif self.load('offline_square_marker_surfaces',[]) != []: + elif self.load('realtime_square_marker_surfaces',[]) != []: logger.debug("Did not find ref surfaces def created or used by the user in player from earlier session. Loading surfaces defined during capture.") self.surfaces = [Offline_Reference_Surface(self.g_pool,saved_definition=d,gaze_positions_by_frame=self.g_pool.positions_by_frame) for d in self.load('realtime_square_marker_surfaces',[]) if isinstance(d,dict)] else: From 0a3b22753470b9744a3325cf6b5031658f56c514 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Tue, 20 May 2014 14:27:39 +0200 Subject: [PATCH 38/41] small tweak to increase responsivness during edit mode --- pupil_src/shared_modules/offline_reference_surface.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/offline_reference_surface.py b/pupil_src/shared_modules/offline_reference_surface.py index 629007b0f2..f8cede054c 100644 --- a/pupil_src/shared_modules/offline_reference_surface.py +++ b/pupil_src/shared_modules/offline_reference_surface.py @@ -36,7 +36,7 @@ def __init__(self,g_pool,name="unnamed",saved_definition=None, gaze_positions_by self.cache = None self.gaze_on_srf = [] # points on surface for realtime feedback display - self.heatmap_detail = .1 + self.heatmap_detail = .2 self.heatmap = None self.heatmap_texture = None #cache fn for offline marker @@ -77,7 +77,8 @@ def update_cache(self,marker_cache,idx=None): # iterations = 0 if self.cache == None: - self.init_cache(marker_cache) + pass + # self.init_cache(marker_cache) elif idx != None: #update single data pt self.cache.update(idx,self.answer_caching_request(marker_cache,idx)) From 42d5e44d96c3261b385ca2ce08c7fda872d84037 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Tue, 20 May 2014 20:22:19 +0200 Subject: [PATCH 39/41] fixed bug that prevented loading of old calibration. --- pupil_src/capture/world.py | 40 +++++++++++++++++++------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/pupil_src/capture/world.py b/pupil_src/capture/world.py index 7f8d1e2f59..f9b48c3d32 100644 --- a/pupil_src/capture/world.py +++ b/pupil_src/capture/world.py @@ -113,41 +113,42 @@ def save(var_name,var): session_settings[var_name] = var + + + # Initialize capture + cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase) + + # Get an image from the grabber + try: + frame = cap.get_frame() + except CameraCaptureError: + logger.error("Could not retrieve image from capture") + cap.close() + return + height,width = frame.img.shape[:2] + # load last calibration data try: pt_cloud = np.load(os.path.join(g_pool.user_dir,'cal_pt_cloud.npy')) - logger.info("Using calibration found in %s" %g_pool.user_dir) + logger.debug("Using calibration found in %s" %g_pool.user_dir) map_pupil = calibrate.get_map_from_cloud(pt_cloud,(width,height)) - except: - logger.info("No calibration found.") + except : + logger.debug("No calibration found.") def map_pupil(vector): - """ 1 to 1 mapping - """ + """ 1 to 1 mapping """ return vector + # any object we attach to the g_pool object *from now on* will only be visible to this process! # vars should be declared here to make them visible to the code reader. g_pool.plugins = [] g_pool.map_pupil = map_pupil g_pool.update_textures = c_bool(1) - - # Initialize capture - cap = autoCreateCapture(cap_src, cap_size, 24, timebase=g_pool.timebase) - if isinstance(cap,FakeCapture): g_pool.update_textures.value = False - - # Get an image from the grabber - try: - frame = cap.get_frame() - except CameraCaptureError: - logger.error("Could not retrieve image from capture") - cap.close() - return - - height,width = frame.img.shape[:2] g_pool.capture = cap + # helpers called by the main atb bar def update_fps(): old_time, bar.timestamp = bar.timestamp, time() @@ -338,7 +339,6 @@ def reset_timebase(): #check if a plugin need to be destroyed g_pool.plugins = [p for p in g_pool.plugins if p.alive] - # render camera image glfwMakeContextCurrent(world_window) From 0b746c97432658f791d74796f3fbfca049f53c23 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Wed, 21 May 2014 11:19:56 +0200 Subject: [PATCH 40/41] renamed gui fields, small bugfix --- pupil_src/shared_modules/offline_marker_detector.py | 4 ++-- pupil_src/shared_modules/offline_reference_surface.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/offline_marker_detector.py b/pupil_src/shared_modules/offline_marker_detector.py index 8c39192647..4185e89555 100644 --- a/pupil_src/shared_modules/offline_marker_detector.py +++ b/pupil_src/shared_modules/offline_marker_detector.py @@ -166,8 +166,8 @@ def update_bar_markers(self): for s,i in zip(self.surfaces,range(len(self.surfaces)))[::-1]: self._bar.add_var("%s_name"%i,create_string_buffer(512),getter=s.atb_get_name,setter=s.atb_set_name,group=str(i),label='name') self._bar.add_var("%s_markers"%i,create_string_buffer(512), getter=s.atb_marker_status,group=str(i),label='found/registered markers' ) - self._bar.add_var("%s_x_scale"%i,vtype=c_float, getter=s.atb_get_scale_x, min=1,setter=s.atb_set_scale_x,group=str(i),label='scale factor x', help='the scale factor is used to adjust the coordinate space for your needs (think photo pixels or mm or whatever)' ) - self._bar.add_var("%s_y_scale"%i,vtype=c_float, getter=s.atb_get_scale_y,min=1,setter=s.atb_set_scale_y,group=str(i),label='scale factor y',help='defining x and y scale factor you atumatically set the correct aspect ratio.' ) + self._bar.add_var("%s_x_scale"%i,vtype=c_float, getter=s.atb_get_scale_x, min=1,setter=s.atb_set_scale_x,group=str(i),label='real width', help='this scale factor is used to adjust the coordinate space for your needs (think photo pixels or mm or whatever)' ) + self._bar.add_var("%s_y_scale"%i,vtype=c_float, getter=s.atb_get_scale_y,min=1,setter=s.atb_set_scale_y,group=str(i),label='real height',help='defining x and y scale factor you atumatically set the correct aspect ratio.' ) self._bar.add_var("%s_window"%i,setter=s.toggle_window,getter=s.window_open,group=str(i),label='open in window') self._bar.add_button("%s_hm"%i, s.generate_heatmap, label='generate_heatmap',group=str(i)) self._bar.add_button("%s_export"%i, self.save_surface_positions_to_file,data=i, label='export surface data',group=str(i)) diff --git a/pupil_src/shared_modules/offline_reference_surface.py b/pupil_src/shared_modules/offline_reference_surface.py index f8cede054c..c951dbd902 100644 --- a/pupil_src/shared_modules/offline_reference_surface.py +++ b/pupil_src/shared_modules/offline_reference_surface.py @@ -54,6 +54,7 @@ def locate_from_cache(self,frame_idx): self.m_from_screen = None self.m_to_screen = None self.gaze_on_srf = [] + self.detected_markers = 0 return True else: self.detected = True From b78b0e00ffb56e537a69098cfd11ea42fd49d263 Mon Sep 17 00:00:00 2001 From: Moritz Kassner Date: Wed, 21 May 2014 12:46:52 +0200 Subject: [PATCH 41/41] setting logger levels --- pupil_src/capture/main.py | 2 +- pupil_src/player/main.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pupil_src/capture/main.py b/pupil_src/capture/main.py index 21049b922c..ca33ade22d 100644 --- a/pupil_src/capture/main.py +++ b/pupil_src/capture/main.py @@ -103,7 +103,7 @@ def main(): # to use a pre-recorded video. # Use a string to specify the path to your video file as demonstrated below # eye_src = '/Users/mkassner/Pupil/datasets/p1-left/frames/test.avi' - # world_src = "~/Desktop/000" + # world_src = "/Users/mkassner/Desktop/2014_01_21/000/world.avi" # Camera video size in pixels (width,height) eye_size = (640,360) diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index eece769ffc..2f1ddc0544 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -46,7 +46,7 @@ import logging #set up root logger before other imports logger = logging.getLogger() -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.WARNING) #since we are not using OS.fork on MacOS we need to do a few extra things to log our exports correctly. if platform.system() == 'Darwin': if __name__ == '__main__': #clear log if main @@ -58,7 +58,7 @@ fh.setLevel(logging.DEBUG) # create console handler with a higher log level ch = logging.StreamHandler() -ch.setLevel(logging.INFO) +ch.setLevel(logging.WARNING) # create formatter and add it to the handlers formatter = logging.Formatter('Player: %(asctime)s - %(name)s - %(levelname)s - %(message)s') fh.setFormatter(formatter) @@ -170,7 +170,7 @@ def on_close(window): rec_dir = sys.argv[1] except: #for dev, supply hardcoded dir: - rec_dir = '/home/mkassner/Desktop/007' + rec_dir = '/home/mkassner/Desktop/003' if os.path.isdir(rec_dir): logger.debug("Dev option: Using hadcoded data dir.") else: