diff --git a/pupil_src/capture/recorder.py b/pupil_src/capture/recorder.py index 0e39394e5e..aef20afad7 100644 --- a/pupil_src/capture/recorder.py +++ b/pupil_src/capture/recorder.py @@ -18,6 +18,7 @@ from shutil import copy2 from glob import glob from audio import Audio_Capture,Audio_Input_Dict +from file_methods import save_object #logging import logging logger = logging.getLogger(__name__) @@ -166,7 +167,6 @@ def get_rec_time_str(self): def start(self): self.timestamps = [] self.pupil_list = [] - self.gaze_list = [] self.frame_count = 0 self.running = True self.menu.read_only = True @@ -242,16 +242,7 @@ def close_info_menu(self): def update(self,frame,events): if self.running: - - # cv2.putText(frame.img, "Frame %s"%self.frame_count,(200,200), cv2.FONT_HERSHEY_SIMPLEX,1,(255,100,100)) - for p in events['pupil_positions']: - pupil_pos = p['timestamp'],p['confidence'],p['id'],p['norm_pos'][0],p['norm_pos'][1],p['diameter'] - self.pupil_list.append(pupil_pos) - - for g in events.get('gaze_positions',[]): - gaze_pos = g['timestamp'],g['confidence'],g['norm_pos'][0],g['norm_pos'][1] - self.gaze_list.append(gaze_pos) - + self.pupil_list += events['pupil_positions'] self.timestamps.append(frame.timestamp) self.writer.write(frame.img) self.frame_count += 1 @@ -270,11 +261,14 @@ def stop(self): except: logger.warning("Could not stop eye-recording. Please report this bug!") - gaze_list_path = os.path.join(self.rec_path, "gaze_positions.npy") - np.save(gaze_list_path,np.asarray(self.gaze_list)) - pupil_list_path = os.path.join(self.rec_path, "pupil_positions.npy") - np.save(pupil_list_path,np.asarray(self.pupil_list)) + save_object(self.pupil_list,os.path.join(self.rec_path, "pupil_positions")) + + for p in g_pool.plugins: + if p.base_class_name == 'Calibration_Plugin': + calibration_plugin = p + break + save_object(p.get_init_dict(),os.path.join(self.rec_path,'active_calibration')) timestamps_path = os.path.join(self.rec_path, "world_timestamps.npy") ts = sanitize_timestamps(np.array(self.timestamps)) diff --git a/pupil_src/capture/world.py b/pupil_src/capture/world.py index 33fc778e37..2b6d1a0278 100644 --- a/pupil_src/capture/world.py +++ b/pupil_src/capture/world.py @@ -51,10 +51,9 @@ from pupil_server import Pupil_Server from pupil_remote import Pupil_Remote from marker_detector import Marker_Detector -from fixation_detector import Fixation_Detector, Dispersion_Fixation_Detector #manage plugins -user_launchable_plugins = [Show_Calibration,Pupil_Server,Pupil_Remote,Marker_Detector] # TODO: Dispersion_Fixation_Detector +user_launchable_plugins = [Show_Calibration,Pupil_Server,Pupil_Remote,Marker_Detector] system_plugins = [Display_Recent_Gaze,Recorder] plugin_by_index = user_launchable_plugins+system_plugins+calibration_plugins+gaze_mapping_plugins name_by_index = [p.__name__ for p in plugin_by_index] diff --git a/pupil_src/player/main.py b/pupil_src/player/main.py index a6d9719b27..344b9bda24 100644 --- a/pupil_src/player/main.py +++ b/pupil_src/player/main.py @@ -77,7 +77,7 @@ y_scroll_factor = 1.0 #imports -from file_methods import Persistent_Dict +from file_methods import Persistent_Dict,load_object import numpy as np #display @@ -116,10 +116,12 @@ from show_calibration import Show_Calibration from batch_exporter import Batch_Exporter from eye_video_overlay import Eye_Video_Overlay +from gaze_mappers import Dummy_Gaze_Mapper,Simple_Gaze_Mapper,Volumetric_Gaze_Mapper,Bilateral_Gaze_Mapper system_plugins = Seek_Bar,Trim_Marks +gaze_mapper_plugins = Dummy_Gaze_Mapper,Simple_Gaze_Mapper,Volumetric_Gaze_Mapper,Bilateral_Gaze_Mapper user_launchable_plugins = Export_Launcher, Vis_Circle,Vis_Cross, Vis_Polyline, Vis_Light_Points,Scan_Path,Dispersion_Duration_Fixation_Detector,Vis_Watermark, Manual_Gaze_Correction, Show_Calibration, Offline_Marker_Detector,Pupil_Server,Batch_Exporter,Eye_Video_Overlay #,Marker_Auto_Trim_Marks -available_plugins = system_plugins + user_launchable_plugins +available_plugins = system_plugins + user_launchable_plugins + gaze_mapper_plugins name_by_index = [p.__name__ for p in available_plugins] index_by_name = dict(zip(name_by_index,range(len(name_by_index)))) plugin_by_name = dict(zip(name_by_index,available_plugins)) @@ -203,32 +205,18 @@ def on_close(window): rec_version = read_rec_version(meta_info) - if rec_version < VersionFormat('0.4'): - video_path = rec_dir + "world.avi" - timestamps_path = rec_dir + "timestamps.npy" - else: - video_path = rec_dir + "world.mkv" - timestamps_path = rec_dir + "world_timestamps.npy" - + if rec_version < VersionFormat('0.5'): + logger.Error("This version is to old. Please upgrade recording format.") + return - gaze_positions_path = rec_dir + "gaze_positions.npy" - pupil_positions_path = rec_dir + "pupil_positions.npy" - #load gaze information - gaze_list = np.load(gaze_positions_path) - timestamps = np.load(timestamps_path) - #correlate data - if rec_version < VersionFormat('0.4'): - gaze_positions_by_frame = correlate_gaze_legacy(gaze_list,timestamps) - pupil_positions_by_frame = [[]for x in range(len(timestamps))] - else: - pupil_list = np.load(pupil_positions_path) - gaze_positions_by_frame = correlate_gaze(gaze_list,timestamps) - pupil_positions_by_frame = correlate_pupil_data(pupil_list,timestamps) + video_path = rec_dir + "world.mkv" + timestamps_path = rec_dir + "world_timestamps.npy" + pupil_positions_path = rec_dir + "pupil_positions" + gaze_mapper_path = rec_dir + 'active_gaze_mapper' # Initialize capture cap = autoCreateCapture(video_path,timestamps=timestamps_path) - if isinstance(cap,FakeCapture): logger.error("could not start capture.") return @@ -264,15 +252,12 @@ def on_close(window): glfwSetScrollCallback(main_window,on_scroll) - # create container for globally scoped vars (within world) + # create container for globally scoped vars g_pool = Global_Container() g_pool.app = 'player' g_pool.version = get_version(version_file) g_pool.capture = cap g_pool.timestamps = timestamps - g_pool.pupil_positions_by_frame = pupil_positions_by_frame - g_pool.gaze_positions_by_frame = gaze_positions_by_frame - # g_pool.fixations_by_frame = [[] for x in timestamps] #let this be filled by the fixation detector plugin g_pool.play = False g_pool.new_seek = True g_pool.user_dir = user_dir @@ -281,6 +266,22 @@ def on_close(window): g_pool.meta_info = meta_info g_pool.pupil_confidence_threshold = session_settings.get('pupil_confidence_threshold',.6) + + # load calibration and map gaze + pupil_list = load_object(pupil_positions_path) + timestamps = np.load(timestamps_path) + gaze_mapper_init_dict = load_object(gaze_mapper_path) + name, args = gaze_mapper_init_dict + logger.info("Loading gaze mapper: %s with settings %s"%(name, args)) + gaze_mapper = gaze_mapper_plugins[name](g_pool,**args) + gaze_list = gaze_mapper.map_gaze_offline(pupil_list) + + + #add new data to g_pool + g_pool.pupil_positions_by_frame = correlate_data(pupil_list,timestamps) + g_pool.gaze_positions_by_frame = correlate_data(gaze_list,timestamps) + g_pool.fixations_by_frame = [[] for x in timestamps] #let this be filled by the fixation detector plugin + def next_frame(_): try: cap.seek_to_frame(cap.get_frame_index()) diff --git a/pupil_src/player/player_methods.py b/pupil_src/player/player_methods.py index 612e1c5685..8513c5e3d0 100644 --- a/pupil_src/player/player_methods.py +++ b/pupil_src/player/player_methods.py @@ -15,131 +15,42 @@ logger = logging.getLogger(__name__) - - -def correlate_pupil_data(pupil_list,timestamps): +def correlate_data(data,timestamps): ''' - pupil_list: timestamp | confidence | id | norm_pos x | norm_pos y | diameter | other data ... + data: dict of data : + will have at least: + timestamp: float - timestamps timestamps to correlate gaze data to + timestamps: timestamps list to correlate data to - - this takes a pupil positions list and a timestamps list and makes a new list - with the length of the number of recorded frames. - Each slot conains a list that will have 0, 1 or more assosiated pupil data points. + this takes a data list and a timestamps list and makes a new list + with the length of the number of timestamps. + Each slot conains a list that will have 0, 1 or more assosiated data points. ''' - pupil_list = list(pupil_list) timestamps = list(timestamps) - - positions_by_frame = [[] for i in timestamps] + data_by_frame = [[] for i in timestamps] frame_idx = 0 - try: - data_point = list(pupil_list.pop(0)) - except: - logger.warning("No gaze positons in this recording.") - return positions_by_frame + data_index = 0 - gaze_timestamp = data_point[0] - while pupil_list: - # if the current gaze point is before the mean of the current world frame timestamp and the next worldframe timestamp + while True: try: + datum = data[data_index] t_between_frames = ( timestamps[frame_idx]+timestamps[frame_idx+1] ) / 2. except IndexError: + # we might loose a data point at the end but we dont care break - if gaze_timestamp <= t_between_frames: - timestamp, confidence, id, x, y, diameter = data_point[:6] - other_data = data_point[6:] #use python slicing to generate empty list is case no other_data is recorded. - positions_by_frame[frame_idx].append({'norm_pos':(x,y), 'confidence':confidence, 'timestamp':timestamp,'id':id,'diameter':diameter,'other_data':other_data}) - data_point = list(pupil_list.pop(0)) - gaze_timestamp = data_point[0] - else: - frame_idx+=1 - - return positions_by_frame - - -def correlate_gaze(gaze_list,timestamps): - ''' - gaze_list: timestamp | confidence | gaze x | gaze y | - timestamps timestamps to correlate gaze data to - - - this takes a gaze positions list and a timestamps list and makes a new list - with the length of the number of recorded frames. - Each slot conains a list that will have 0, 1 or more assosiated gaze postions. - ''' - gaze_list = list(gaze_list) - timestamps = list(timestamps) - - positions_by_frame = [[] for i in timestamps] - - frame_idx = 0 - try: - data_point = gaze_list.pop(0) - except: - logger.warning("No gaze positons in this recording.") - return positions_by_frame - gaze_timestamp = data_point[0] - - while gaze_list: - # if the current gaze point is before the mean of the current world frame timestamp and the next worldframe timestamp - try: - t_between_frames = ( timestamps[frame_idx]+timestamps[frame_idx+1] ) / 2. - except IndexError: - break - if gaze_timestamp <= t_between_frames: - ts,confidence,x,y, = data_point - positions_by_frame[frame_idx].append({'norm_pos':(x,y), 'confidence':confidence, 'timestamp':ts}) - data_point = gaze_list.pop(0) - gaze_timestamp = data_point[0] + if datum['timestamp'] <= t_between_frames: + data_by_frame[frame_idx].append(datum) + data_index +=1 else: frame_idx+=1 - return positions_by_frame - - -def correlate_gaze_legacy(gaze_list,timestamps): - ''' - gaze_list: gaze x | gaze y | pupil x | pupil y | timestamp - timestamps timestamps to correlate gaze data to - - - this takes a gaze positions list and a timestamps list and makes a new list - with the length of the number of recorded frames. - Each slot conains a list that will have 0, 1 or more assosiated gaze postions. - load gaze information - ''' - gaze_list = list(gaze_list) - timestamps = list(timestamps) - - positions_by_frame = [[] for i in timestamps] - - frame_idx = 0 - try: - data_point = gaze_list.pop(0) - except: - logger.warning("No gaze positons in this recording.") - return positions_by_frame - - gaze_timestamp = data_point[4] + return data_by_frame - while gaze_list: - # if the current gaze point is before the mean of the current world frame timestamp and the next worldframe timestamp - try: - t_between_frames = ( timestamps[frame_idx]+timestamps[frame_idx+1] ) / 2. - except IndexError: - break - if gaze_timestamp <= t_between_frames: - positions_by_frame[frame_idx].append({'norm_pos':(data_point[0],data_point[1]), 'timestamp':data_point[4],'confidence':data_point[5]}) - data_point = gaze_list.pop(0) - gaze_timestamp = data_point[4] - else: - frame_idx+=1 - return positions_by_frame @@ -156,69 +67,7 @@ def is_pupil_rec_dir(data_dir): logger.debug("%s contains %s and is therefore considered a valid rec dir."%(data_dir,required_files)) return True -# backwards compatibility tools: - -def patch_meta_info(rec_dir): - #parse info.csv file - - ''' - This is how we need it: - - Recording Name 2014_01_21 - Start Date 21.01.2014 - Start Time 11:42:24 - Duration Time 00:00:29 - World Camera Frames 710 - World Camera Resolution 1280x720 - Capture Software Version v0.3.7 - User testing - Platform Linux - Machine brosnan - Release 3.5.0-45-generic - Version #68~precise1-Ubuntu SMP Wed Dec 4 16:18:46 UTC 2013 - ''' - proper_names = ['Recording Name', - 'Start Date', - 'Start Time', - 'Duration Time', - 'World Camera Frames', - 'World Camera Resolution', - 'Capture Software Version', - 'User', - 'Platform', - 'Release', - 'Version'] - - with open(rec_dir + "/info.csv") as info: - meta_info = [line.strip().split('\t') for line in info.readlines() ] - - for entry in meta_info: - for proper_name in proper_names: - if proper_name == entry[0]: - break - elif proper_name in entry[0]: - logger.info("Permanently updated info.csv field name: '%s' to '%s'."%(entry[0],proper_name)) - entry[0]=proper_name - break - - new_info = '' - for e in meta_info: - new_info += e[0]+'\t'+e[1]+'\n' - with open(rec_dir + "/info.csv",'w') as info: - info.write(new_info) - -def convert_gaze_pos(gaze_list,capture_version): - ''' - util fn to update old gaze pos files to new coordsystem. UNTESTED! - ''' - #lets make a copy here so that we are not making inplace edits of the passed array - gaze_list = gaze_list.copy() - if capture_version < .36: - logger.info("Gaze list is from old Recoding, I will update the data to work with new coordinate system.") - gaze_list[:,:4] += 1. #broadcasting - gaze_list[:,:4] /= 2. #broadcasting - return gaze_list def transparent_circle(img,center,radius,color,thickness): diff --git a/pupil_src/shared_modules/calibration_routines/gaze_mappers.py b/pupil_src/shared_modules/calibration_routines/gaze_mappers.py index 247f8c1caf..d6f2d77d16 100644 --- a/pupil_src/shared_modules/calibration_routines/gaze_mappers.py +++ b/pupil_src/shared_modules/calibration_routines/gaze_mappers.py @@ -20,7 +20,7 @@ def update(self,frame,events): gaze_pts = [] for p in events['pupil_positions']: if p['confidence'] > self.g_pool.pupil_confidence_threshold: - gaze_pts.append({'norm_pos':p['norm_pos'][:],'confidence':p['confidence'],'timestamp':p['timestamp']}) + gaze_pts.append({'norm_pos':p['norm_pos'][:],'confidence':p['confidence'],'timestamp':p['timestamp'],'base':[p]}) events['gaze_positions'] = gaze_pts @@ -41,7 +41,7 @@ def update(self,frame,events): for p in events['pupil_positions']: if p['confidence'] > self.g_pool.pupil_confidence_threshold: gaze_point = self.map_fn(p['norm_pos']) - gaze_pts.append({'norm_pos':gaze_point,'confidence':p['confidence'],'timestamp':p['timestamp']}) + gaze_pts.append({'norm_pos':gaze_point,'confidence':p['confidence'],'timestamp':p['timestamp'],'base':[p]]}) events['gaze_positions'] = gaze_pts diff --git a/pupil_src/shared_modules/fixation_detector.py b/pupil_src/shared_modules/fixation_detector.py index 309f6d082b..3c12bb3040 100644 --- a/pupil_src/shared_modules/fixation_detector.py +++ b/pupil_src/shared_modules/fixation_detector.py @@ -43,7 +43,6 @@ class Dispersion_Duration_Fixation_Detector(Fixation_Detector): Terms + dispersion (spatial) = how much spatial movement is allowed within one fixation (in visual angular degrees or pixels) + duration (temporal) = what is the minimum time required for gaze data to be within dispersion threshold? - + cohesion (spatial+temporal) = is the cluster of fixations close together ''' def __init__(self,g_pool,max_dispersion = 1.0,min_duration = 0.15,h_fov=78, v_fov=50,show_fixations = False): @@ -100,71 +99,96 @@ def _classify(self): ''' classify fixations ''' - gaze_data = chain(*self.g_pool.gaze_positions_by_frame) + gaze_data = list(chain(*self.g_pool.gaze_positions_by_frame)) #filter out below threshold confidence mesurements - gaze_data = filter(lambda g: g['confidence'] > self.g_pool.pupil_confidence_threshold, gaze_data) - + # gaze_data = filter(lambda g: g['confidence'] > self.g_pool.pupil_confidence_threshold, gaze_data) sample_threshold = self.min_duration * 3 *.3 #lets assume we need data for at least 30% of the duration dispersion_threshold = self.max_dispersion duration_threshold = self.min_duration - + def get_next_sample(gaze_data,support,low_confidence): + while gaze_data: + gp = gaze_data.pop(0) + if gp['confidence'] < self.g_pool.pupil_confidence_threshold: + low_confidence.append(gp) + else: + support.append(gp) + return def dist_deg(p1,p2): return sqrt(((p1[0]-p2[0])*self.h_fov)**2+((p1[1]-p2[1])*self.v_fov)**2) fixations = [] fixation_support = [gaze_data.pop(0)] - - - while gaze_data: + low_confidence_samples = [] + while fixation_support and gaze_data: fixation_centroid = sum([p['norm_pos'][0] for p in fixation_support])/len(fixation_support),sum([p['norm_pos'][1] for p in fixation_support])/len(fixation_support) dispersion = max([dist_deg(fixation_centroid,p['norm_pos']) for p in fixation_support]) - if dispersion < dispersion_threshold: + if dispersion < dispersion_threshold and gaze_data: #so far all samples inside the threshold, lets add a new canditate - fixation_support.append(gaze_data.pop(0)) + get_next_sample(gaze_data,fixation_support,low_confidence_samples) else: #last added point will break dispersion threshold for current candite fixation. So we conclude sampling for this fixation last_sample = fixation_support.pop(-1) - duration = fixation_support[-1]['timestamp'] - fixation_support[0]['timestamp'] - if duration > duration_threshold and len(fixation_support) > sample_threshold: - #long enough for fixation: we classifiy this fixation canditae as fixation - fixation_centroid = sum([p['norm_pos'][0] for p in fixation_support])/len(fixation_support),sum([p['norm_pos'][1] for p in fixation_support])/len(fixation_support) - dispersion = max([dist_deg(fixation_centroid,p['norm_pos']) for p in fixation_support]) - new_fixation = {'id': len(fixations),'norm_pos':fixation_centroid,'gaze':fixation_support, 'duration':duration,'dispersion':dispersion, 'pix_dispersion':dispersion*self.pix_per_degree, 'start_timestamp':fixation_support[0]['timestamp']} - fixations.append(new_fixation) + if fixation_support: + duration = fixation_support[-1]['timestamp'] - fixation_support[0]['timestamp'] + if duration > duration_threshold and len(fixation_support) > sample_threshold: + #long enough for fixation: we classifiy this fixation canditae as fixation + fixation_centroid = sum([p['norm_pos'][0] for p in fixation_support])/len(fixation_support),sum([p['norm_pos'][1] for p in fixation_support])/len(fixation_support) + dispersion = max([dist_deg(fixation_centroid,p['norm_pos']) for p in fixation_support]) + confidence = sum(g['confidence'] for g in fixation_support+low_confidence_samples)/(len(fixation_support)+len(low_confidence_samples)) + new_fixation = {'id': len(fixations), + 'norm_pos':fixation_centroid, + 'gaze':fixation_support, + 'low_confidece_gaze':low_confidence_samples, + 'duration':duration, + 'dispersion':dispersion, + 'pix_dispersion':dispersion*self.pix_per_degree, + 'timestamp':fixation_support[0]['timestamp'], + 'confidence':confidence} + fixations.append(new_fixation) #start a new fixation candite fixation_support = [last_sample] + low_confidence_samples = [] logger.debug("detected %s Fixations"%len(fixations)) - self.fixations = fixations[:] #keep a copy because we destroy our list below. + + # now lets bin fixations into frames. Fixations may be repeated this way as they span muliple frames + self.fixations = fixations[:] #keep a copy because we destroy our list below. fixations_by_frame = [[] for x in self.g_pool.timestamps] + if not fixations: + logger.error('No fixations detected.') + return + index = 0 f = fixations.pop(0) - while fixations: + while True: try: t = self.g_pool.timestamps[index] except IndexError: #reached end of ts list break - if f['start_timestamp'] > t: + if f['timestamp'] > t: #fixation in the future, lets move forward in time. index += 1 - elif f['start_timestamp']+f['duration'] > t: + elif f['timestamp']+f['duration'] > t: # fixation during this frame fixations_by_frame[index].append(f) index += 1 else: #fixation in the past, get new one and check again - f = fixations.pop(0) + try: + f = fixations.pop(0) + except: + break self.fixations_by_frame = fixations_by_frame - + self.g_pool.fixations_by_frame = fixations_by_frame def update(self,frame,events): events['fixations'] = self.fixations_by_frame[frame.index]