Merge pull request #1071 from papr/world_less

Support Pupil Mobile recordings without world video

pupil_src/launchables/marker_detectors.py

@@ -34,11 +34,12 @@ def circle_detector(ipc_push_url, pair_url, source_path, batch_size=20):
 
     # imports
     from time import sleep
-    from video_capture import File_Source, EndofVideoFileError
+    from video_capture import init_playback_source, EndofVideoError
     from circle_detector import CircleTracker
 
     try:
-        src = File_Source(Empty(), source_path, timed_playback=False)
+        src = init_playback_source(Empty(), source_path, timed_playback=False)
+
         frame = src.get_frame()
         logger.info('Starting calibration marker detection...')
         frame_count = src.get_frame_count()
@@ -77,7 +78,7 @@ def circle_detector(ipc_push_url, pair_url, source_path, batch_size=20):
 
             frame = src.get_frame()
 
-    except EndofVideoFileError:
+    except EndofVideoError:
         process_pipe.send(topic='progress', payload={'data': queue})
         process_pipe.send(topic='finished', payload={})
         logger.debug("Process finished")

pupil_src/launchables/player.py

@@ -70,7 +70,7 @@ def player(rec_dir, ipc_pub_url, ipc_sub_url,
         from pyglui.cygl.utils import Named_Texture, RGBA
         import gl_utils
         # capture
-        from video_capture import File_Source, EndofVideoFileError
+        from video_capture import init_playback_source, EndofVideoError
 
         # helpers/utils
         from version_utils import VersionFormat
@@ -183,8 +183,6 @@ def on_drop(window, count, paths):
         def get_dt():
             return next(tick)
 
-        video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
-                      if os.path.splitext(f)[1] in ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')][0]
         pupil_data_path = os.path.join(rec_dir, "pupil_data")
 
         meta_info = load_meta_info(rec_dir)
@@ -208,8 +206,10 @@ def get_dt():
         g_pool.plugin_by_name = {p.__name__: p for p in plugins}
         g_pool.camera_render_size = None
 
-        # sets itself to g_pool.capture
-        File_Source(g_pool, video_path)
+        valid_ext = ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg', '.fake')
+        video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
+                      if os.path.splitext(f)[1] in valid_ext][0]
+        init_playback_source(g_pool, source_path=video_path)
 
         # load session persistent settings
         session_settings = Persistent_Dict(os.path.join(user_dir, "user_settings_player"))
@@ -319,7 +319,7 @@ def toggle_general_settings(collapsed):
         g_pool.gui = ui.UI()
         g_pool.gui_user_scale = session_settings.get('gui_scale', 1.)
         g_pool.menubar = ui.Scrolling_Menu("Settings", pos=(-500, 0), size=(-icon_bar_width, 0), header_pos='left')
-        g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width,0),size=(0,0),header_pos='hidden')
+        g_pool.iconbar = ui.Scrolling_Menu("Icons", pos=(-icon_bar_width, 0), size=(0, 0), header_pos='hidden')
         g_pool.timelines = ui.Container((0, 0), (0, 0), (0, 0))
         g_pool.timelines.horizontal_constraint = g_pool.menubar
         g_pool.user_timelines = ui.Timeline_Menu('User Timelines', pos=(0., -150.),
@@ -437,10 +437,10 @@ def handle_notifications(n):
                 g_pool.new_seek = False
                 try:
                     new_frame = g_pool.capture.get_frame()
-                except EndofVideoFileError:
+                except EndofVideoError:
                     # end of video logic: pause at last frame.
                     g_pool.capture.play = False
-                    logger.warning("end of video")
+                    logger.warning("End of video")
 
             frame = new_frame.copy()
             events = {}
@@ -636,8 +636,12 @@ def on_drop(window, count, paths):
             glfw.glfwSwapBuffers(window)
 
             if rec_dir:
-                update_recording_to_recent(rec_dir)
-                glfw.glfwSetWindowShouldClose(window, True)
+                try:
+                    update_recording_to_recent(rec_dir)
+                except AssertionError as err:
+                    logger.error(str(err))
+                else:
+                    glfw.glfwSetWindowShouldClose(window, True)
 
             glfw.glfwPollEvents()
 

pupil_src/shared_modules/background_helper.py

@@ -57,6 +57,9 @@ def _wrapper(self, pipe, _should_terminate_flag, generator, *args, **kwargs):
 
     def fetch(self):
         '''Fetches progress and available results from background'''
+        if self.completed or self.canceled:
+            return
+
         while self.pipe.poll(0):
             try:
                 datum = self.pipe.recv()

pupil_src/shared_modules/exporter.py

@@ -20,8 +20,7 @@
 import os
 from time import time
 from glob import glob
-import numpy as np
-from video_capture import File_Source, EndofVideoFileError
+from video_capture import init_playback_source, EndofVideoError
 from player_methods import update_recording_to_recent, load_meta_info
 from av_writer import AV_Writer
 from file_methods import load_object
@@ -74,8 +73,6 @@ def export(rec_dir, user_dir, min_data_confidence, start_frame=None, end_frame=N
 
         update_recording_to_recent(rec_dir)
 
-        video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
-                      if os.path.splitext(f)[-1] in ('.mp4', '.mkv', '.avi', '.mjpeg')][0]
         pupil_data_path = os.path.join(rec_dir, "pupil_data")
         audio_path = os.path.join(rec_dir, "audio.mp4")
 
@@ -84,7 +81,12 @@ def export(rec_dir, user_dir, min_data_confidence, start_frame=None, end_frame=N
         g_pool = Global_Container()
         g_pool.app = 'exporter'
         g_pool.min_data_confidence = min_data_confidence
-        cap = File_Source(g_pool, video_path)
+
+        valid_ext = ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg', '.fake')
+        video_path = [f for f in glob(os.path.join(rec_dir, "world.*"))
+                      if os.path.splitext(f)[1] in valid_ext][0]
+        cap = init_playback_source(g_pool, source_path=video_path)
+
         timestamps = cap.timestamps
 
         # Out file path verification, we do this before but if one uses a separate tool, this will kick in.
@@ -155,7 +157,7 @@ def export(rec_dir, user_dir, min_data_confidence, start_frame=None, end_frame=N
         while frames_to_export > current_frame:
             try:
                 frame = cap.get_frame()
-            except EndofVideoFileError:
+            except EndofVideoError:
                 break
 
             events = {'frame': frame}

pupil_src/shared_modules/gaze_producers.py

@@ -347,7 +347,7 @@ def trim(format_only=False):
                     minutes = ts // 60
                     seconds = ts - (minutes * 60.)
                     time_fmt += ' {:02.0f}:{:02.0f} -'.format(abs(minutes), seconds)
-                button.outer_label = time_fmt[:-2]  # remove final ' - '
+                button.outer_label = time_fmt[:-2]  # remove final ' -'
             button.function = trim
 
         section_menu.append(ui.Text_Input('label', sec, label='Label'))
@@ -569,9 +569,9 @@ def toggle_marker_detection(self):
             self.start_marker_detection()
 
     def start_marker_detection(self):
-        self.process_pipe = zmq_tools.Msg_Pair_Server(self.g_pool.zmq_ctx)
         self.circle_marker_positions = []
         source_path = self.g_pool.capture.source_path
+        self.process_pipe = zmq_tools.Msg_Pair_Server(self.g_pool.zmq_ctx)
         self.notify_all({'subject': 'circle_detector_process.should_start',
                          'source_path': source_path, "pair_url": self.process_pipe.url})
 

pupil_src/shared_modules/marker_detector_cacher.py

@@ -9,11 +9,12 @@
 ---------------------------------------------------------------------------~(*)
 '''
 
+
 class Global_Container(object):
     pass
 
 
-def fill_cache(visited_list, video_file_path, q, seek_idx, run,min_marker_perimeter, invert_image):
+def fill_cache(visited_list, video_file_path, q, seek_idx, run, min_marker_perimeter, invert_image):
     '''
     this function is part of marker_detector it is run as a seperate process.
     it must be kept in a seperate file for namespace sanatisation
@@ -22,11 +23,11 @@ def fill_cache(visited_list, video_file_path, q, seek_idx, run,min_marker_perime
     import logging
     logger = logging.getLogger(__name__+' with pid: '+str(os.getpid()))
     logger.debug('Started cacher process for Marker Detector')
-    from video_capture import File_Source, EndofVideoFileError, FileSeekError
+    from video_capture import init_playback_source, EndofVideoError, FileSeekError
     from square_marker_detect import detect_markers_robust
     aperture = 9
     markers = []
-    cap = File_Source(Global_Container(), video_file_path)
+    cap = init_playback_source(Global_Container(), video_file_path)
 
     def next_unvisited_idx(frame_idx):
         try:
@@ -67,7 +68,7 @@ def handle_frame(next_frame):
 
         try:
             frame = cap.get_frame()
-        except EndofVideoFileError:
+        except EndofVideoError:
             logger.debug("Video File's last frame(s) not accesible")
 
             # could not read frame

pupil_src/shared_modules/offline_surface_tracker.py

@@ -283,19 +283,24 @@ def invalidate_marker_cache(self):
         self.init_marker_cacher()
 
     def init_marker_cacher(self):
+
         from marker_detector_cacher import fill_cache
         visited_list = [False if x is False else True for x in self.cache]
-        video_file_path =  self.g_pool.capture.source_path
         self.cache_queue = mp.Queue()
-        self.cacher_seek_idx = mp.Value('i',0)
-        self.cacher_run = mp.Value(c_bool,True)
-        self.cacher = mp.Process(target=fill_cache, args=(visited_list,video_file_path,self.cache_queue,self.cacher_seek_idx,self.cacher_run,self.min_marker_perimeter_cacher,self.invert_image))
+        self.cacher_seek_idx = mp.Value('i', 0)
+        self.cacher_run = mp.Value(c_bool, True)
+
+        video_file_path = self.g_pool.capture.source_path
+        args = (visited_list, video_file_path, self.cache_queue,
+                self.cacher_seek_idx, self.cacher_run,
+                self.min_marker_perimeter_cacher, self.invert_image)
+        self.cacher = mp.Process(target=fill_cache, args=args)
         self.cacher.start()
 
     def update_marker_cache(self):
         while not self.cache_queue.empty():
-            idx,c_m = self.cache_queue.get()
-            self.cache.update(idx,c_m)
+            idx, c_m = self.cache_queue.get()
+            self.cache.update(idx, c_m)
 
             for s in self.surfaces:
                 s.update_cache(self.cache, min_marker_perimeter=self.min_marker_perimeter,

pupil_src/shared_modules/player_methods.py

@@ -121,6 +121,8 @@ def update_recording_to_recent(rec_dir):
         update_recording_v0913_to_v0915(rec_dir)
     if rec_version < VersionFormat('1.3'):
         update_recording_v0915_v13(rec_dir)
+    if rec_version < VersionFormat('1.4'):
+        update_recording_v13_v14(rec_dir)
 
     # How to extend:
     # if rec_version < VersionFormat('FUTURE FORMAT'):
@@ -456,6 +458,43 @@ def update_recording_v0915_v13(rec_dir):
     update_meta_info(rec_dir, meta_info)
 
 
+def update_recording_v13_v14(rec_dir):
+    logger.info("Updating recording from v1.3 to v1.4")
+    valid_ext = ('.mp4', '.mkv', '.avi', '.h264', '.mjpeg')
+    existing_videos = [f for f in glob.glob(os.path.join(rec_dir, 'world.*'))
+                       if os.path.splitext(f)[1] in valid_ext]
+
+    if not existing_videos:
+        min_ts = np.inf
+        max_ts = -np.inf
+        for f in glob.glob(os.path.join(rec_dir, "eye*_timestamps.npy")):
+            try:
+                eye_ts = np.load(f)
+                assert len(eye_ts.shape) == 1
+                assert eye_ts.shape[0] > 1
+                min_ts = min(min_ts, eye_ts[0])
+                max_ts = max(max_ts, eye_ts[-1])
+            except (FileNotFoundError, AssertionError):
+                pass
+
+        error_msg = 'Could not generate world timestamps from eye timestamps. This is an invalid recording.'
+        assert -np.inf < min_ts < max_ts < np.inf, error_msg
+
+        logger.warning('No world video found. Constructing an artificial replacement.')
+
+        frame_rate = 30
+        timestamps = np.arange(min_ts, max_ts, 1/frame_rate)
+        np.save(os.path.join(rec_dir, 'world_timestamps'), timestamps)
+        save_object({'frame_rate': frame_rate, 'frame_size': (1280, 720), 'version': 0},
+                    os.path.join(rec_dir, 'world.fake'))
+
+    meta_info_path = os.path.join(rec_dir, "info.csv")
+    with open(meta_info_path, 'r', encoding='utf-8') as csvfile:
+        meta_info = csv_utils.read_key_value_file(csvfile)
+        meta_info['Data Format Version'] = 'v1.4'
+    update_meta_info(rec_dir, meta_info)
+
+
 def update_recording_bytes_to_unicode(rec_dir):
     logger.info("Updating recording from bytes to unicode.")
 

pupil_src/shared_modules/seek_control.py

@@ -67,10 +67,13 @@ def play(self):
     def play(self, new_state):
         if new_state and self.current_ts == self.trim_right_ts:
             self.g_pool.capture.seek_to_frame(self.trim_left)
+            self.g_pool.new_seek = True
         elif new_state and self.current_ts >= self.g_pool.timestamps[-10]:
-            self.g_pool.capture.seek_to_frame(0)  # avoid pause set by hitting trimmark pause.
+            self.g_pool.capture.seek_to_frame(0)
+            self.g_pool.new_seek = True
             logger.warning("End of video - restart at beginning.")
-        self.g_pool.capture.play = new_state
+        else:
+            self.g_pool.capture.play = new_state
 
     @property
     def trim_left_ts(self):
@@ -119,7 +122,6 @@ def forwards(self, x):
             self.g_pool.capture.playback_speed = speeds[new_idx]
         else:
             # frame-by-frame mode, seek one frame forward
-            self.g_pool.capture.seek_to_next_frame()
             self.g_pool.new_seek = True
 
     @property

pupil_src/shared_modules/video_capture/__init__.py

@@ -18,17 +18,22 @@
 These backends are available:
 - UVC: Local USB sources
 - NDSI: Remote Pupil Mobile sources
-- Fake: Fallback, static random image
+- Fake: Fallback, static grid image
 - File: Loads video from file
 '''
 
+import os
+import numpy as np
+from glob import glob
+from camera_models import load_intrinsics
+
 import logging
 logger = logging.getLogger(__name__)
 
-from .base_backend import InitialisationError, StreamError
+from .base_backend import InitialisationError, StreamError, EndofVideoError
 from .base_backend import Base_Source, Base_Manager
 from .fake_backend import Fake_Source, Fake_Manager
-from .file_backend import FileCaptureError, EndofVideoFileError, FileSeekError
+from .file_backend import FileCaptureError, FileSeekError
 from .file_backend import File_Source, File_Manager
 from .uvc_backend import UVC_Source,  UVC_Manager
 
@@ -50,3 +55,10 @@
 else:
     source_classes.append(Realsense_Source)
     manager_classes.append(Realsense_Manager)
+
+
+def init_playback_source(g_pool, source_path=None, *args, **kwargs):
+        if source_path is None or os.path.splitext(source_path)[1] == '.fake':
+            return Fake_Source(g_pool, source_path=source_path, *args, **kwargs)
+        else:
+            return File_Source(g_pool, source_path=source_path, *args, **kwargs)