From 1642564260b2e4e7db2468ec16d24fe70853e1ce Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 27 Feb 2018 09:18:01 +0100 Subject: [PATCH 01/11] Service: Add Fake_Source as loadable plugin + cleanup --- pupil_src/launchables/service.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pupil_src/launchables/service.py b/pupil_src/launchables/service.py index 35491eec1f..963dad87c1 100644 --- a/pupil_src/launchables/service.py +++ b/pupil_src/launchables/service.py @@ -95,6 +95,7 @@ def stop_eye_process(eye_id): from pupil_groups import Pupil_Groups from frame_publisher import Frame_Publisher from service_ui import Service_UI + from video_capture import Fake_Source logger.info('Application Version: {}'.format(version)) logger.info('System Info: {}'.format(get_system_info())) @@ -118,12 +119,14 @@ def get_timestamp(): g_pool.get_timestamp = get_timestamp # manage plugins - runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, 'plugins')) - user_launchable_plugins = [Service_UI, Pupil_Groups, Pupil_Remote, Frame_Publisher]+runtime_plugins - plugin_by_index = runtime_plugins+calibration_plugins+gaze_mapping_plugins+user_launchable_plugins - name_by_index = [p.__name__ for p in plugin_by_index] - plugin_by_name = dict(zip(name_by_index, plugin_by_index)) - default_plugins = [('Service_UI', {}), ('Dummy_Gaze_Mapper', {}), ('HMD_Calibration', {}), ('Pupil_Remote', {})] + plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, 'plugins')) + plugins += [Service_UI, Pupil_Groups, Pupil_Remote, Frame_Publisher, Fake_Source] + plugins += calibration_plugins + gaze_mapping_plugins + plugin_by_name = {p.__name__: p for p in plugins} + default_plugins = [('Service_UI', {}), + ('Dummy_Gaze_Mapper', {}), + ('HMD_Calibration', {}), + ('Pupil_Remote', {})] g_pool.plugin_by_name = plugin_by_name tick = delta_t() From 7e3becc6eb22d0c7bd25c9c032ddc053705d48b7 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 27 Feb 2018 10:39:38 +0100 Subject: [PATCH 02/11] Revert "Service: Add Fake_Source as loadable plugin + cleanup" This reverts commit 1642564260b2e4e7db2468ec16d24fe70853e1ce. --- pupil_src/launchables/service.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/pupil_src/launchables/service.py b/pupil_src/launchables/service.py index 963dad87c1..35491eec1f 100644 --- a/pupil_src/launchables/service.py +++ b/pupil_src/launchables/service.py @@ -95,7 +95,6 @@ def stop_eye_process(eye_id): from pupil_groups import Pupil_Groups from frame_publisher import Frame_Publisher from service_ui import Service_UI - from video_capture import Fake_Source logger.info('Application Version: {}'.format(version)) logger.info('System Info: {}'.format(get_system_info())) @@ -119,14 +118,12 @@ def get_timestamp(): g_pool.get_timestamp = get_timestamp # manage plugins - plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, 'plugins')) - plugins += [Service_UI, Pupil_Groups, Pupil_Remote, Frame_Publisher, Fake_Source] - plugins += calibration_plugins + gaze_mapping_plugins - plugin_by_name = {p.__name__: p for p in plugins} - default_plugins = [('Service_UI', {}), - ('Dummy_Gaze_Mapper', {}), - ('HMD_Calibration', {}), - ('Pupil_Remote', {})] + runtime_plugins = import_runtime_plugins(os.path.join(g_pool.user_dir, 'plugins')) + user_launchable_plugins = [Service_UI, Pupil_Groups, Pupil_Remote, Frame_Publisher]+runtime_plugins + plugin_by_index = runtime_plugins+calibration_plugins+gaze_mapping_plugins+user_launchable_plugins + name_by_index = [p.__name__ for p in plugin_by_index] + plugin_by_name = dict(zip(name_by_index, plugin_by_index)) + default_plugins = [('Service_UI', {}), ('Dummy_Gaze_Mapper', {}), ('HMD_Calibration', {}), ('Pupil_Remote', {})] g_pool.plugin_by_name = plugin_by_name tick = delta_t() From 3532e07d8b14f591d67e5ce41c38d61cdd0b423a Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 27 Feb 2018 14:20:16 +0100 Subject: [PATCH 03/11] UVC Source: Correctly restore uvc controls after disconnect --- .../shared_modules/video_capture/uvc_backend.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index aef23732ab..8bbc02fe37 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -88,6 +88,7 @@ def __init__(self, g_pool, frame_size, frame_rate, name=None, preferred_names=() self.name_backup = (self.name,) self.frame_size_backup = frame_size self.frame_rate_backup = frame_rate + self.backup_uvc_controls = {} def verify_drivers(self): import os @@ -202,12 +203,12 @@ def _re_init_capture(self, uid): self.configure_capture(current_size, current_fps, current_uvc_controls) self.update_menu() - def _init_capture(self, uid): + def _init_capture(self, uid, backup_uvc_controls={}): self.uvc_capture = uvc.Capture(uid) - self.configure_capture(self.frame_size_backup, self.frame_rate_backup, self._get_uvc_controls()) + self.configure_capture(self.frame_size_backup, self.frame_rate_backup, backup_uvc_controls) self.update_menu() - def _re_init_capture_by_names(self, names): + def _re_init_capture_by_names(self, names, backup_uvc_controls={}): # burn-in test specific. Do not change text! self.devices.update() for d in self.devices: @@ -217,7 +218,7 @@ def _re_init_capture_by_names(self, names): if self.uvc_capture: self._re_init_capture(d['uid']) else: - self._init_capture(d['uid']) + self._init_capture(d['uid'], backup_uvc_controls) return raise InitialisationError('Could not find Camera {} during re initilization.'.format(names)) @@ -226,9 +227,10 @@ def _restart_logic(self): if self.uvc_capture: logger.warning("Capture failed to provide frames. Attempting to reinit.") self.name_backup = (self.uvc_capture.name,) + self.backup_uvc_controls = self._get_uvc_controls() self.uvc_capture = None try: - self._re_init_capture_by_names(self.name_backup) + self._re_init_capture_by_names(self.name_backup, self.backup_uvc_controls) except (InitialisationError, uvc.InitError): time.sleep(0.02) self.update_menu() From abb886ce8e26591da6977d634adf4ea694d04f26 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 27 Feb 2018 14:20:54 +0100 Subject: [PATCH 04/11] Seek Control: Avoid chrash on recordings with less than 10 world frames --- pupil_src/shared_modules/seek_control.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/seek_control.py b/pupil_src/shared_modules/seek_control.py index 72580933b8..f4a0560656 100644 --- a/pupil_src/shared_modules/seek_control.py +++ b/pupil_src/shared_modules/seek_control.py @@ -68,7 +68,11 @@ def play(self, new_state): if new_state and self.current_ts == self.trim_right_ts: self.g_pool.capture.seek_to_frame(self.trim_left) self.g_pool.new_seek = True - elif new_state and self.current_ts >= self.g_pool.timestamps[-10]: + + # Sometimes there is less video frames than timestamps. The rewind + # logic needs to catch these cases but work for recordings with less + # than 10 frames + elif new_state and self.current_ts >= self.g_pool.timestamps[-10:][0]: self.g_pool.capture.seek_to_frame(0) self.g_pool.new_seek = True logger.warning("End of video - restart at beginning.") From 04ddbaf21c27c290cda72e2836d693de0fa936d8 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 27 Feb 2018 14:21:26 +0100 Subject: [PATCH 05/11] Detector 3D: Consistent types in projected_sphere field --- pupil_src/shared_modules/pupil_detectors/detector_utils.pxd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/pupil_detectors/detector_utils.pxd b/pupil_src/shared_modules/pupil_detectors/detector_utils.pxd index b3fd212c47..20e4da1229 100644 --- a/pupil_src/shared_modules/pupil_detectors/detector_utils.pxd +++ b/pupil_src/shared_modules/pupil_detectors/detector_utils.pxd @@ -72,7 +72,7 @@ cdef inline convertTo3DPythonResult( Detector3DResult& result, object frame ) py_result['sphere'] = sphere if str(result.projectedSphere.center[0]) == 'nan': - projectedSphere = {'axes': (0,0), 'angle': 90.0, 'center': (0,0)} + projectedSphere = {'axes': (0.,0.), 'angle': 90.0, 'center': (0.,0.)} else: projectedSphere = {} projectedSphere['center'] = (result.projectedSphere.center[0] + frame.width / 2.0 ,frame.height / 2.0 - result.projectedSphere.center[1]) From 993f7374bddaf8759ff938b1a4a934f304e21857 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 27 Feb 2018 14:22:08 +0100 Subject: [PATCH 06/11] Player methods: Always check for worldless recordings Instead of only on upgrading from v1.3 to v1.4 --- pupil_src/shared_modules/player_methods.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/pupil_src/shared_modules/player_methods.py b/pupil_src/shared_modules/player_methods.py index cca9dca2f6..00939890b4 100644 --- a/pupil_src/shared_modules/player_methods.py +++ b/pupil_src/shared_modules/player_methods.py @@ -124,6 +124,9 @@ def update_recording_to_recent(rec_dir): if rec_version < VersionFormat('1.4'): update_recording_v13_v14(rec_dir) + # Do this independent of rec_version + check_for_worldless_recording(rec_dir) + # How to extend: # if rec_version < VersionFormat('FUTURE FORMAT'): # update_recording_v081_to_FUTURE(rec_dir) @@ -460,6 +463,15 @@ def update_recording_v0915_v13(rec_dir): def update_recording_v13_v14(rec_dir): logger.info("Updating recording from v1.3 to v1.4") + meta_info_path = os.path.join(rec_dir, "info.csv") + with open(meta_info_path, 'r', encoding='utf-8') as csvfile: + meta_info = csv_utils.read_key_value_file(csvfile) + meta_info['Data Format Version'] = 'v1.4' + update_meta_info(rec_dir, meta_info) + + +def check_for_worldless_recording(rec_dir): + logger.info("Checking for world-less recording") valid_ext = ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg') existing_videos = [f for f in glob.glob(os.path.join(rec_dir, 'world.*')) if os.path.splitext(f)[1] in valid_ext] @@ -488,12 +500,6 @@ def update_recording_v13_v14(rec_dir): save_object({'frame_rate': frame_rate, 'frame_size': (1280, 720), 'version': 0}, os.path.join(rec_dir, 'world.fake')) - meta_info_path = os.path.join(rec_dir, "info.csv") - with open(meta_info_path, 'r', encoding='utf-8') as csvfile: - meta_info = csv_utils.read_key_value_file(csvfile) - meta_info['Data Format Version'] = 'v1.4' - update_meta_info(rec_dir, meta_info) - def update_recording_bytes_to_unicode(rec_dir): logger.info("Updating recording from bytes to unicode.") From 78491247055f2545f6cddcb439ccd412ebeef6c0 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 27 Feb 2018 14:22:49 +0100 Subject: [PATCH 07/11] Capture: Allow creation of world-less recordings --- pupil_src/shared_modules/av_writer.py | 14 ++++---------- pupil_src/shared_modules/recorder.py | 17 ++++++++++------- .../video_capture/ndsi_backend.py | 2 +- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/pupil_src/shared_modules/av_writer.py b/pupil_src/shared_modules/av_writer.py index c6a6f02690..7011490204 100644 --- a/pupil_src/shared_modules/av_writer.py +++ b/pupil_src/shared_modules/av_writer.py @@ -179,16 +179,14 @@ def write_video_frame(self, input_frame): break # wait for next image def close(self): - # flush encoder - while 1: + # only flush encoder if there has been at least one frame + while self.configured: packet = self.video_stream.encode() if packet: self.container.mux(packet) else: break - - self.container.close() - logger.debug("Closed media container") + self.container.close() # throws RuntimeError if no frames were written write_timestamps(self.file_loc, self.timestamps) def release(self): @@ -239,11 +237,7 @@ def write_video_frame(self, input_frame): self.timestamps.append(input_frame.timestamp) def close(self): - try: - self.container.close() - except(RuntimeError): - logger.error("Media file does not contain any frames.") - logger.debug("Closed media container") + self.container.close() # throws RuntimeError if no frames were written write_timestamps(self.file_loc, self.timestamps) def release(self): diff --git a/pupil_src/shared_modules/recorder.py b/pupil_src/shared_modules/recorder.py index 17c7372357..785e6d8168 100644 --- a/pupil_src/shared_modules/recorder.py +++ b/pupil_src/shared_modules/recorder.py @@ -188,8 +188,6 @@ def on_notify(self, notification): elif notification['subject'] == 'recording.should_start': if self.running: logger.info('Recording already running!') - elif not self.g_pool.capture.online: - logger.error("Current world capture is offline. Please reconnect or switch to fake capture") else: if notification.get("session_name", ""): self.set_session_name(notification["session_name"]) @@ -295,7 +293,7 @@ def close_info_menu(self): self.g_pool.gui.remove(self.info_menu) self.info_menu = None - def recent_events(self,events): + def recent_events(self, events): if self.running: for key, data in events.items(): if key not in ('dt', 'frame', 'depth_frame'): @@ -316,8 +314,15 @@ def recent_events(self,events): def stop(self): # explicit release of VideoWriter - self.writer.release() - self.writer = None + try: + self.writer.release() + except RuntimeError: + logger.error("No world video recorded") + else: + logger.debug("Closed media container") + self.g_pool.capture.intrinsics.save(self.rec_path, custom_name='world') + finally: + self.writer = None save_object(self.data, os.path.join(self.rec_path, "pupil_data")) @@ -327,8 +332,6 @@ def stop(self): except: logger.info("No surface_definitions data found. You may want this if you do marker tracking.") - self.g_pool.capture.intrinsics.save(self.rec_path, custom_name='world') - try: with open(self.meta_info_path, 'a', newline='') as csvfile: csv_utils.write_key_value_file(csvfile, { diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index 566cb3db45..c256670505 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -19,7 +19,7 @@ try: from ndsi import __version__ - assert __version__ >= '0.3.4' + assert __version__ >= '0.3.5' from ndsi import __protocol_version__ except (ImportError, AssertionError): raise Exception("pyndsi version is to old. Please upgrade") From 8a025474ce113c0dc33b9c894883f274361cbdce Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 27 Feb 2018 14:24:51 +0100 Subject: [PATCH 08/11] Recorder: Enable eye video recording by default --- pupil_src/shared_modules/recorder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/recorder.py b/pupil_src/shared_modules/recorder.py index 785e6d8168..72a1a9ddaf 100644 --- a/pupil_src/shared_modules/recorder.py +++ b/pupil_src/shared_modules/recorder.py @@ -85,7 +85,7 @@ class Recorder(System_Plugin_Base): def __init__(self, g_pool, session_name=get_auto_name(), rec_dir=None, user_info={'name': '', 'additional_field': 'change_me'}, - info_menu_conf={}, show_info_menu=False, record_eye=False, + info_menu_conf={}, show_info_menu=False, record_eye=True, raw_jpeg=True): super().__init__(g_pool) # update name if it was autogenerated. From a306c8b6b34bf0b0716e5c36e15e0730ce8f2ec6 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 27 Feb 2018 14:58:49 +0100 Subject: [PATCH 09/11] Recorder: Enable/disable eye recording via record_eye notification field --- pupil_src/shared_modules/recorder.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/recorder.py b/pupil_src/shared_modules/recorder.py index 72a1a9ddaf..66eac010d8 100644 --- a/pupil_src/shared_modules/recorder.py +++ b/pupil_src/shared_modules/recorder.py @@ -167,9 +167,10 @@ def on_notify(self, notification): Reacts to notifications: ``recording.should_start``: Starts a new recording session. - fields: 'session_name' change session name - use `/` to att dirs. + fields: + - 'session_name' change session name start with `/` to ingore the rec base dir and start from root instead. + - `record_eye` boolean that indicates recording of the eyes, defaults to current setting ``recording.should_stop``: Stops current recording session Emits notifications: @@ -189,6 +190,7 @@ def on_notify(self, notification): if self.running: logger.info('Recording already running!') else: + self.record_eye = notification.get('record_eye', self.record_eye) if notification.get("session_name", ""): self.set_session_name(notification["session_name"]) self.start() From 991633586b1decf840b71a11ac9aee7f3faf00b8 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 27 Feb 2018 15:27:24 +0100 Subject: [PATCH 10/11] Gaze topics: Switch format from 0/1/2 to 0./1./01. --- .../calibration_routines/gaze_mappers.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pupil_src/shared_modules/calibration_routines/gaze_mappers.py b/pupil_src/shared_modules/calibration_routines/gaze_mappers.py index a0dee1ab19..c7e55609cb 100644 --- a/pupil_src/shared_modules/calibration_routines/gaze_mappers.py +++ b/pupil_src/shared_modules/calibration_routines/gaze_mappers.py @@ -134,7 +134,7 @@ def __init__(self, g_pool): super().__init__(g_pool) def _map_monocular(self, p): - return {'topic': 'gaze.2d.{}'.format(p['id']), + return {'topic': 'gaze.2d.{}.'.format(p['id']), 'norm_pos': p['norm_pos'], 'confidence': p['confidence'], 'timestamp': p['timestamp'], @@ -158,7 +158,7 @@ def __init__(self, g_pool, params): def _map_monocular(self, p): gaze_point = self.map_fn(p['norm_pos']) - return {'topic': 'gaze.2d.{}'.format(p['id']), + return {'topic': 'gaze.2d.{}.'.format(p['id']), 'norm_pos': gaze_point, 'confidence': p['confidence'], 'id': p['id'], @@ -180,7 +180,7 @@ def __init__(self, g_pool, params0, params1): def _map_monocular(self, p): gaze_point = self.map_fns[p['id']](p['norm_pos']) - return {'topic': 'gaze.2d.{}'.format(p['id']), + return {'topic': 'gaze.2d.{}.'.format(p['id']), 'norm_pos': gaze_point, 'confidence': p['confidence'], 'id': p['id'], @@ -221,7 +221,7 @@ def _map_binocular(self, p0, p1): (gaze_point_eye0[1] + gaze_point_eye1[1])/2.) confidence = (p0['confidence'] + p1['confidence'])/2. ts = (p0['timestamp'] + p1['timestamp'])/2. - return {'topic': 'gaze.2d.2', + return {'topic': 'gaze.2d.01.', 'norm_pos': gaze_point, 'confidence': confidence, 'timestamp': ts, @@ -229,7 +229,7 @@ def _map_binocular(self, p0, p1): def _map_monocular(self, p): gaze_point = self.map_fn_fallback[p['id']](p['norm_pos']) - return {'topic': 'gaze.2d.2', + return {'topic': 'gaze.2d.{}.'.format(p['id']), 'norm_pos': gaze_point, 'confidence': p['confidence'], 'timestamp': p['timestamp'], @@ -294,7 +294,7 @@ def _map_monocular(self,p): gaze_3d = self.toWorld(gaze_point) normal_3d = np.dot( self.rotation_matrix, np.array( p['circle_3d']['normal'] ) ) - g = { 'topic': 'gaze.2d.2', + g = { 'topic': 'gaze.3d.{}.'.format(p['id']), 'norm_pos': image_point, 'eye_center_3d': eye_center.tolist(), 'gaze_normal_3d': normal_3d.tolist(), @@ -407,7 +407,7 @@ def _map_monocular(self, p): normal_3d = self.rotation_matricies[p_id] @ np.array(p['circle_3d']['normal']) - g = {'topic': 'gaze.3d.2', + g = {'topic': 'gaze.3d.{}.'.format(p_id), 'eye_centers_3d': {p['id']: eye_center.tolist()}, 'gaze_normals_3d': {p['id']: normal_3d.tolist()}, 'gaze_point_3d': gaze_3d.tolist(), @@ -490,7 +490,7 @@ def _map_binocular(self, p0, p1): confidence = min(p0['confidence'],p1['confidence']) ts = (p0['timestamp'] + p1['timestamp'])/2. - g = {'topic': 'gaze.3d.2', + g = {'topic': 'gaze.3d.01.', 'eye_centers_3d': {0: s0_center.tolist(), 1: s1_center.tolist()}, 'gaze_normals_3d': {0: s0_normal.tolist(), 1: s1_normal.tolist()}, 'gaze_point_3d': nearest_intersection_point.tolist(), From 5c5f4d6bf608b4c8b2c9ec194207ef11974ee11f Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 27 Feb 2018 15:57:24 +0100 Subject: [PATCH 11/11] Eye: Catch RuntimeErrors when no eye video were recorded --- pupil_src/launchables/eye.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index 0b29397670..89d8a4e2ff 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -401,7 +401,10 @@ def replace_source(source_class_name, source_settings): g_pool.capture.init_ui() if g_pool.writer: logger.info("Done recording.") - g_pool.writer.release() + try: + g_pool.writer.release() + except RuntimeError: + logger.error('No eye video recorded') g_pool.writer = None g_pool.replace_source = replace_source # for ndsi capture @@ -494,7 +497,10 @@ def window_should_update(): elif subject == 'recording.stopped': if g_pool.writer: logger.info("Done recording.") - g_pool.writer.release() + try: + g_pool.writer.release() + except RuntimeError: + logger.error('No eye video recorded') g_pool.writer = None elif subject.startswith('meta.should_doc'): ipc_socket.notify({