From c6e8c243937143d67ff9b52b810e96077fec3865 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 20 Jan 2020 16:01:54 +0100 Subject: [PATCH 01/65] Remove FakeSource and FakeBackend --- .../shared_modules/video_capture/__init__.py | 5 +- .../video_capture/base_backend.py | 2 +- .../video_capture/fake_backend.py | 384 ------------------ 3 files changed, 3 insertions(+), 388 deletions(-) delete mode 100644 pupil_src/shared_modules/video_capture/fake_backend.py diff --git a/pupil_src/shared_modules/video_capture/__init__.py b/pupil_src/shared_modules/video_capture/__init__.py index c4e1767528..ca4bb197c3 100644 --- a/pupil_src/shared_modules/video_capture/__init__.py +++ b/pupil_src/shared_modules/video_capture/__init__.py @@ -37,7 +37,6 @@ InitialisationError, StreamError, ) -from .fake_backend import Fake_Manager, Fake_Source from .file_backend import File_Manager, File_Source, FileSeekError from .hmd_streaming import HMD_Streaming_Source from .uvc_backend import UVC_Manager, UVC_Source @@ -45,8 +44,8 @@ logger = logging.getLogger(__name__) -source_classes = [File_Source, UVC_Source, Fake_Source, HMD_Streaming_Source] -manager_classes = [File_Manager, UVC_Manager, Fake_Manager] +source_classes = [File_Source, UVC_Source, HMD_Streaming_Source] +manager_classes = [File_Manager, UVC_Manager] try: from .ndsi_backend import NDSI_Source, NDSI_Manager diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index c70e283b35..06ce000871 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -78,7 +78,7 @@ def recent_events(self, events): Adds events['frame']=Frame(args) Frame: Object containing image and time information of the current - source frame. See `fake_source.py` for a minimal implementation. + source frame. """ raise NotImplementedError() diff --git a/pupil_src/shared_modules/video_capture/fake_backend.py b/pupil_src/shared_modules/video_capture/fake_backend.py deleted file mode 100644 index cdf6a3149c..0000000000 --- a/pupil_src/shared_modules/video_capture/fake_backend.py +++ /dev/null @@ -1,384 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2020 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" -from .base_backend import Base_Source, Playback_Source, Base_Manager, EndofVideoError - -import os -import cv2 -import numpy as np -from time import time, sleep -from pyglui import ui -from camera_models import Dummy_Camera -from file_methods import load_object - -# logging -import logging - -logger = logging.getLogger(__name__) - - -class Frame(object): - """docstring of Frame""" - - def __init__(self, timestamp, img, index): - self.timestamp = timestamp - self._img = img - self.bgr = img - self.height, self.width, _ = img.shape - self._gray = None - self.index = index - # indicate that the frame does not have a native yuv or jpeg buffer - self.yuv_buffer = None - self.jpeg_buffer = None - - @property - def img(self): - return self._img - - @property - def gray(self): - if self._gray is None: - self._gray = cv2.cvtColor(self._img, cv2.COLOR_BGR2GRAY) - return self._gray - - def copy(self): - return Frame(self.timestamp, self._img.copy(), self.index) - - -class Fake_Source(Playback_Source, Base_Source): - """Simple source which shows random, static image. - - It is used as falback in case the original source fails. `preferred_source` - contains the necessary information to recover to the original source if - it becomes accessible again. - - Attributes: - current_frame_idx (int): Sequence counter - frame_rate (int) - frame_size (tuple) - """ - - def __init__( - self, - g_pool, - source_path=None, - frame_size=None, - frame_rate=None, - name="Fake Source", - *args, - **kwargs - ): - super().__init__(g_pool, *args, **kwargs) - if self.timing == "external": - self.recent_events = self.recent_events_external_timing - else: - self.recent_events = self.recent_events_own_timing - - if source_path: - meta = load_object(source_path) - frame_size = meta["frame_size"] - frame_rate = meta["frame_rate"] - self.timestamps = np.load( - os.path.splitext(source_path)[0] + "_timestamps.npy" - ) - else: - self.timestamps = None - - self.fps = frame_rate - self._name = name - self.make_img(tuple(frame_size)) - self.source_path = source_path - self.current_frame_idx = 0 - self.target_frame_idx = 0 - - def init_ui(self): - self.add_menu() - self.menu.label = "Static Image Source" - - text = ui.Info_Text("This plugin displays a static image.") - self.menu.append(text) - - self.menu.append( - ui.Text_Input( - "frame_size", - label="Frame size", - setter=lambda x: None, - getter=lambda: "{} x {}".format(*self.frame_size), - ) - ) - - self.menu.append( - ui.Text_Input( - "frame_rate", - label="Frame rate", - setter=lambda x: None, - getter=lambda: "{:.0f} FPS".format(self.frame_rate), - ) - ) - - if self.g_pool.app == "player": - # get_frame_count() is not constant in Capture and would trigger - # a redraw on each frame - self.menu.append( - ui.Text_Input( - "frame_num", - label="Number of frames", - setter=lambda x: None, - getter=lambda: self.get_frame_count(), - ) - ) - - def deinit_ui(self): - self.remove_menu() - - def make_img(self, size): - # Generate Pupil Labs colored gradient - self._img = np.zeros((size[1], size[0], 3), dtype=np.uint8) - self._img[:, :, 0] += np.linspace(91, 157, self.frame_size[0], dtype=np.uint8) - self._img[:, :, 1] += np.linspace(165, 161, self.frame_size[0], dtype=np.uint8) - self._img[:, :, 2] += np.linspace(35, 112, self.frame_size[0], dtype=np.uint8) - - self._intrinsics = Dummy_Camera(size, self.name) - - def recent_events_external_timing(self, events): - try: - last_index = self._recent_frame.index - except AttributeError: - # called once on start when self._recent_frame is None - last_index = -1 - - frame = None - pbt = self.g_pool.seek_control.current_playback_time - ts_idx = self.g_pool.seek_control.ts_idx_from_playback_time(pbt) - if ts_idx == last_index: - frame = self._recent_frame.copy() - if self.play and ts_idx == self.get_frame_count() - 1: - logger.info("Recording has ended.") - self.g_pool.seek_control.play = False - elif ts_idx < last_index or ts_idx > last_index + 1: - # time to seek - self.seek_to_frame(ts_idx) - - # Only call get_frame() if the next frame is actually needed - frame = frame or self.get_frame() - self._recent_frame = frame - events["frame"] = frame - self.g_pool.seek_control.end_of_seek() - - def recent_events_own_timing(self, events): - try: - frame = self.get_frame() - except IndexError: - logger.info("Recording has ended.") - self.play = False - else: - if self.timing: - self.wait(frame.timestamp) - self._recent_frame = frame - events["frame"] = frame - - def get_frame(self): - try: - timestamp = self.timestamps[self.target_frame_idx] - except IndexError: - raise EndofVideoError("Reached end of timestamps list.") - except TypeError: - timestamp = self._recent_wait_ts + 1 / self.fps - - frame = Frame(timestamp, self._img.copy(), self.target_frame_idx) - - frame_txt_font_name = cv2.FONT_HERSHEY_SIMPLEX - frame_txt_font_scale = 1.0 - frame_txt_thickness = 1 - - # first line: frame index - frame_txt = "Fake source frame {}".format(frame.index) - frame_txt_size = cv2.getTextSize( - frame_txt, frame_txt_font_name, frame_txt_font_scale, frame_txt_thickness - )[0] - - frame_txt_loc = ( - self.frame_size[0] // 2 - frame_txt_size[0] // 2, - self.frame_size[1] // 2 - frame_txt_size[1], - ) - - cv2.putText( - frame.img, - frame_txt, - frame_txt_loc, - frame_txt_font_name, - frame_txt_font_scale, - (255, 255, 255), - thickness=frame_txt_thickness, - lineType=cv2.LINE_8, - ) - - # second line: resolution @ fps - frame_txt = "{}x{} @ {} fps".format(*self.frame_size, self.frame_rate) - frame_txt_size = cv2.getTextSize( - frame_txt, frame_txt_font_name, frame_txt_font_scale, frame_txt_thickness - )[0] - - frame_txt_loc = ( - self.frame_size[0] // 2 - frame_txt_size[0] // 2, - self.frame_size[1] // 2 + frame_txt_size[1], - ) - - cv2.putText( - frame.img, - frame_txt, - frame_txt_loc, - frame_txt_font_name, - frame_txt_font_scale, - (255, 255, 255), - thickness=frame_txt_thickness, - lineType=cv2.LINE_8, - ) - - self.current_frame_idx = self.target_frame_idx - self.target_frame_idx += 1 - - return frame - - def get_frame_count(self): - try: - return len(self.timestamps) - except TypeError: - return self.current_frame_idx + 1 - - def seek_to_frame(self, frame_idx): - self.target_frame_idx = max(0, min(frame_idx, self.get_frame_count() - 1)) - self.finished_sleep = 0 - - def get_frame_index(self): - return self.current_frame_idx - - @property - def name(self): - return self._name - - @property - def settings(self): - return {"frame_size": self.frame_size, "frame_rate": self.frame_rate} - - @settings.setter - def settings(self, settings): - self.frame_size = settings.get("frame_size", self.frame_size) - self.frame_rate = settings.get("frame_rate", self.frame_rate) - - @property - def frame_size(self): - return self._img.shape[1], self._img.shape[0] - - @frame_size.setter - def frame_size(self, new_size): - # closest match for size - sizes = [abs(r[0] - new_size[0]) for r in self.frame_sizes] - best_size_idx = sizes.index(min(sizes)) - size = self.frame_sizes[best_size_idx] - if size != new_size: - logger.warning( - "%s resolution capture mode not available. Selected %s." - % (new_size, size) - ) - self.make_img(size) - - @property - def frame_rates(self): - return (30, 60, 90, 120) - - @property - def frame_sizes(self): - return ((640, 480), (1280, 720), (1920, 1080)) - - @property - def frame_rate(self): - return self.fps - - @frame_rate.setter - def frame_rate(self, new_rate): - rates = [abs(r - new_rate) for r in self.frame_rates] - best_rate_idx = rates.index(min(rates)) - rate = self.frame_rates[best_rate_idx] - if rate != new_rate: - logger.warning( - "%sfps capture mode not available at (%s) on 'Fake Source'. Selected %sfps. " - % (new_rate, self.frame_size, rate) - ) - self.fps = rate - - @property - def jpeg_support(self): - return False - - @property - def online(self): - return True - - def get_init_dict(self): - if self.g_pool.app == "capture": - d = super().get_init_dict() - d["frame_size"] = self.frame_size - d["frame_rate"] = self.frame_rate - d["name"] = self.name - return d - else: - raise NotImplementedError() - - -class Fake_Manager(Base_Manager): - """Simple manager to explicitly activate a fake source""" - - gui_name = "Test image" - - def __init__(self, g_pool): - super().__init__(g_pool) - - def init_ui(self): - self.add_menu() - from pyglui import ui - - self.add_auto_select_button() - text = ui.Info_Text("Convenience manager to select a fake source explicitly.") - - activation_button = ui.Button("Activate Fake Capture", self.activate) - self.menu.extend([text, activation_button]) - - def activate(self): - # a capture leaving is a must stop for recording. - self.notify_all({"subject": "recording.should_stop"}) - settings = {} - settings["timing"] = "own" - settings["frame_rate"] = self.g_pool.capture.frame_rate - settings["frame_size"] = self.g_pool.capture.frame_size - settings["name"] = "Fake Source" - # if the user set fake capture, we dont want it to auto jump back to the old capture. - if self.g_pool.process == "world": - self.notify_all( - {"subject": "start_plugin", "name": "Fake_Source", "args": settings} - ) - else: - self.notify_all( - { - "subject": "start_eye_plugin", - "target": self.g_pool.process, - "name": "Fake_Source", - "args": settings, - } - ) - - def auto_activate_source(self): - self.activate() - - def deinit_ui(self): - self.remove_menu() - - def recent_events(self, events): - pass From 9bb664bd638a224502ffb3aa5c09398197d83641 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 20 Jan 2020 16:03:50 +0100 Subject: [PATCH 02/65] Remove RealSense backends and managers --- .../shared_modules/video_capture/__init__.py | 18 - .../video_capture/realsense2_backend.py | 916 ----------------- .../video_capture/realsense_backend.py | 935 ------------------ 3 files changed, 1869 deletions(-) delete mode 100755 pupil_src/shared_modules/video_capture/realsense2_backend.py delete mode 100755 pupil_src/shared_modules/video_capture/realsense_backend.py diff --git a/pupil_src/shared_modules/video_capture/__init__.py b/pupil_src/shared_modules/video_capture/__init__.py index ca4bb197c3..4c075bddc8 100644 --- a/pupil_src/shared_modules/video_capture/__init__.py +++ b/pupil_src/shared_modules/video_capture/__init__.py @@ -54,21 +54,3 @@ else: source_classes.append(NDSI_Source) manager_classes.append(NDSI_Manager) - -try: - from .realsense_backend import Realsense_Source, Realsense_Manager -except ImportError: - logger.debug("Install pyrealsense to use the Intel RealSense backend") -else: - source_classes.append(Realsense_Source) - manager_classes.append(Realsense_Manager) - -try: - from .realsense2_backend import Realsense2_Source, Realsense2_Manager -except ImportError: - logger.debug( - "Install pyrealsense2 to use the Intel RealSense backend for D400 series cameras" - ) -else: - source_classes.append(Realsense2_Source) - manager_classes.append(Realsense2_Manager) diff --git a/pupil_src/shared_modules/video_capture/realsense2_backend.py b/pupil_src/shared_modules/video_capture/realsense2_backend.py deleted file mode 100755 index 903c28db95..0000000000 --- a/pupil_src/shared_modules/video_capture/realsense2_backend.py +++ /dev/null @@ -1,916 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2020 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" - -import logging -import time -import cv2 -import os - -import pyrealsense2 as rs - -from version_utils import VersionFormat -from .base_backend import Base_Source, Base_Manager -from av_writer import MPEG_Writer -from camera_models import load_intrinsics - -import glfw -import gl_utils -from OpenGL.GL import * -from OpenGL.GLU import * -from pyglui import cygl -import cython_methods -import numpy as np -from ctypes import * - -# check versions for our own depedencies as they are fast-changing -# assert VersionFormat(rs.__version__) >= VersionFormat("2.2") # FIXME - -# logging -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -TIMEOUT = 500 # ms FIXME -DEFAULT_COLOR_SIZE = (1280, 720) -DEFAULT_COLOR_FPS = 30 -DEFAULT_DEPTH_SIZE = (640, 480) -DEFAULT_DEPTH_FPS = 30 - -# very thin wrapper for rs.frame objects -class ColorFrame(object): - def __init__(self, data, timestamp, index): - self.timestamp = timestamp - self.index = index - - self.data = data[:, :, np.newaxis].view(dtype=np.uint8) - total_size = self.data.size - y_plane = total_size // 2 - u_plane = y_plane // 2 - self._yuv = np.empty(total_size, dtype=np.uint8) - self._yuv[:y_plane] = self.data[:, :, 0].ravel() - self._yuv[y_plane : y_plane + u_plane] = self.data[:, ::2, 1].ravel() - self._yuv[y_plane + u_plane :] = self.data[:, 1::2, 1].ravel() - self._shape = self.data.shape[:2] - - self._bgr = None - self._gray = None - - @property - def height(self): - return self._shape[0] - - @property - def width(self): - return self._shape[1] - - @property - def yuv_buffer(self): - return self._yuv - - @property - def yuv422(self): - Y = self._yuv[: self._yuv.size // 2] - U = self._yuv[self._yuv.size // 2 : 3 * self._yuv.size // 4] - V = self._yuv[3 * self._yuv.size // 4 :] - - Y.shape = self._shape - U.shape = self._shape[0], self._shape[1] // 2 - V.shape = self._shape[0], self._shape[1] // 2 - - return Y, U, V - - @property - def bgr(self): - if self._bgr is None: - self._bgr = cv2.cvtColor(self.data, cv2.COLOR_YUV2BGR_YUYV) - return self._bgr - - @property - def img(self): - return self.bgr - - @property - def gray(self): - if self._gray is None: - self._gray = self._yuv[: self._yuv.size // 2] - self._gray.shape = self._shape - return self._gray - - -class DepthFrame(object): - def __init__(self, data, timestamp, index): - self.timestamp = timestamp - self.index = index - - self._bgr = None - self._gray = None - self.depth = data - self.yuv_buffer = None - - @property - def height(self): - return self.depth.shape[0] - - @property - def width(self): - return self.depth.shape[1] - - @property - def bgr(self): - if self._bgr is None: - self._bgr = cython_methods.cumhist_color_map16(self.depth) - return self._bgr - - @property - def img(self): - return self.bgr - - @property - def gray(self): - if self._gray is None: - self._gray = cv2.cvtColor(self.bgr, cv2.cv2.COLOR_BGR2GRAY) - return self._gray - - -class Realsense2_Source(Base_Source): - def __init__( - self, - g_pool, - device_id=None, - frame_size=DEFAULT_COLOR_SIZE, - frame_rate=DEFAULT_COLOR_FPS, - depth_frame_size=DEFAULT_DEPTH_SIZE, - depth_frame_rate=DEFAULT_DEPTH_FPS, - preview_depth=False, - device_options=(), - record_depth=True, - ): - logger.debug("_init_ started") - super().__init__(g_pool) - self._intrinsics = None - self.color_frame_index = 0 - self.depth_frame_index = 0 - self.context = rs.context() - self.pipeline = rs.pipeline(self.context) - self.pipeline_profile = None - self.preview_depth = preview_depth - self.record_depth = record_depth - self.depth_video_writer = None - self._needs_restart = False - self.frame_size_backup = DEFAULT_COLOR_SIZE - self.depth_frame_size_backup = DEFAULT_DEPTH_SIZE - self.frame_rate_backup = DEFAULT_COLOR_FPS - self.depth_frame_rate_backup = DEFAULT_DEPTH_FPS - - self._initialize_device( - device_id, - frame_size, - frame_rate, - depth_frame_size, - depth_frame_rate, - device_options, - ) - logger.debug("_init_ completed") - - def _initialize_device( - self, - device_id, - color_frame_size, - color_fps, - depth_frame_size, - depth_fps, - device_options=(), - ): - self.stop_pipeline() - self.last_color_frame_ts = None - self.last_depth_frame_ts = None - self._recent_frame = None - self._recent_depth_frame = None - - if device_id is None: - device_id = self.device_id - - if device_id is None: # FIXME these two if blocks look ugly. - return - - # use default streams to filter modes by rs_stream and rs_format - self._available_modes = self._enumerate_formats(device_id) - logger.debug( - "device_id: {} self._available_modes: {}".format( - device_id, str(self._available_modes) - ) - ) - - if ( - color_frame_size is not None - and depth_frame_size is not None - and color_fps is not None - and depth_fps is not None - ): - color_frame_size = tuple(color_frame_size) - depth_frame_size = tuple(depth_frame_size) - - logger.debug( - "Initialize with Color {}@{}\tDepth {}@{}".format( - color_frame_size, color_fps, depth_frame_size, depth_fps - ) - ) - - # make sure the frame rates are compatible with the given frame sizes - color_fps = self._get_valid_frame_rate( - rs.stream.color, color_frame_size, color_fps - ) - depth_fps = self._get_valid_frame_rate( - rs.stream.depth, depth_frame_size, depth_fps - ) - - self.frame_size_backup = color_frame_size - self.depth_frame_size_backup = depth_frame_size - self.frame_rate_backup = color_fps - self.depth_frame_rate_backup = depth_fps - - config = self._prep_configuration( - color_frame_size, color_fps, depth_frame_size, depth_fps - ) - else: - config = self._get_default_config() - self.frame_size_backup = DEFAULT_COLOR_SIZE - self.depth_frame_size_backup = DEFAULT_DEPTH_SIZE - self.frame_rate_backup = DEFAULT_COLOR_FPS - self.depth_frame_rate_backup = DEFAULT_DEPTH_FPS - - try: - self.pipeline_profile = self.pipeline.start(config) - except RuntimeError as re: - logger.error("Cannot start pipeline! " + str(re)) - self.pipeline_profile = None - else: - self.stream_profiles = { - s.stream_type(): s.as_video_stream_profile() - for s in self.pipeline_profile.get_streams() - } - logger.debug("Pipeline started for device " + device_id) - logger.debug("Stream profiles: " + str(self.stream_profiles)) - - self._intrinsics = load_intrinsics( - self.g_pool.user_dir, self.name, self.frame_size - ) - self.update_menu() - self._needs_restart = False - - def _prep_configuration( - self, - color_frame_size=None, - color_fps=None, - depth_frame_size=None, - depth_fps=None, - ): - config = rs.config() - - # only use these two formats - color_format = rs.format.yuyv - depth_format = rs.format.z16 - - config.enable_stream( - rs.stream.depth, - depth_frame_size[0], - depth_frame_size[1], - depth_format, - depth_fps, - ) - - config.enable_stream( - rs.stream.color, - color_frame_size[0], - color_frame_size[1], - color_format, - color_fps, - ) - - return config - - def _get_default_config(self): - config = rs.config() # default config is RGB8, we want YUYV - config.enable_stream( - rs.stream.color, - DEFAULT_COLOR_SIZE[0], - DEFAULT_COLOR_SIZE[1], - rs.format.yuyv, - DEFAULT_COLOR_FPS, - ) - config.enable_stream( - rs.stream.depth, - DEFAULT_DEPTH_SIZE[0], - DEFAULT_DEPTH_SIZE[1], - rs.format.z16, - DEFAULT_DEPTH_FPS, - ) - return config - - def _get_valid_frame_rate(self, stream_type, frame_size, fps): - assert stream_type == rs.stream.color or stream_type == rs.stream.depth - - if not self._available_modes or stream_type not in self._available_modes: - logger.warning( - "_get_valid_frame_rate: self._available_modes not set yet. Returning default fps." - ) - if stream_type == rs.stream.color: - return DEFAULT_COLOR_FPS - elif stream_type == rs.stream.depth: - return DEFAULT_DEPTH_FPS - else: - raise ValueError("Unexpected `stream_type`: {}".format(stream_type)) - - if frame_size not in self._available_modes[stream_type]: - logger.error( - "Frame size not supported for {}: {}. Returning default fps".format( - stream_type, frame_size - ) - ) - if stream_type == rs.stream.color: - return DEFAULT_COLOR_FPS - elif stream_type == rs.stream.depth: - return DEFAULT_DEPTH_FPS - - if fps not in self._available_modes[stream_type][frame_size]: - old_fps = fps - rates = [ - abs(r - fps) for r in self._available_modes[stream_type][frame_size] - ] - best_rate_idx = rates.index(min(rates)) - fps = self._available_modes[stream_type][frame_size][best_rate_idx] - logger.warning( - "{} fps is not supported for ({}) for Color Stream. Fallback to {} fps".format( - old_fps, frame_size, fps - ) - ) - - return fps - - def _enumerate_formats(self, device_id): - """Enumerate formats into hierachical structure: - - streams: - resolutions: - framerates - """ - formats = {} - - if self.context is None: - return formats - - devices = self.context.query_devices() - current_device = None - - for d in devices: - try: - serial = d.get_info(rs.camera_info.serial_number) - except RuntimeError as re: - logger.error("Device no longer available " + str(re)) - else: - if device_id == serial: - current_device = d - - if current_device is None: - return formats - logger.debug("Found the current device: " + device_id) - - sensors = current_device.query_sensors() - for s in sensors: - stream_profiles = s.get_stream_profiles() - for sp in stream_profiles: - vp = sp.as_video_stream_profile() - stream_type = vp.stream_type() - - if stream_type not in (rs.stream.color, rs.stream.depth): - continue - elif vp.format() not in (rs.format.z16, rs.format.yuyv): - continue - - formats.setdefault(stream_type, {}) - stream_resolution = (vp.width(), vp.height()) - formats[stream_type].setdefault(stream_resolution, []).append(vp.fps()) - - return formats - - def stop_pipeline(self): - if self.online: - try: - self.pipeline_profile = None - self.stream_profiles = None - self.pipeline.stop() - logger.debug("Pipeline stopped.") - except RuntimeError as re: - logger.error("Cannot stop the pipeline: " + str(re)) - - def cleanup(self): - if self.depth_video_writer is not None: - self.stop_depth_recording() - self.stop_pipeline() - - def get_init_dict(self): - return { - "frame_size": self.frame_size, - "frame_rate": self.frame_rate, - "depth_frame_size": self.depth_frame_size, - "depth_frame_rate": self.depth_frame_rate, - "preview_depth": self.preview_depth, - "record_depth": self.record_depth, - } - - def get_frames(self): - if self.online: - try: - frames = self.pipeline.wait_for_frames(TIMEOUT) - except RuntimeError as e: - logger.error("get_frames: Timeout!") - raise RuntimeError(e) - else: - current_time = self.g_pool.get_timestamp() - - color = None - # if we're expecting color frames - if rs.stream.color in self.stream_profiles: - color_frame = frames.get_color_frame() - last_color_frame_ts = color_frame.get_timestamp() - if self.last_color_frame_ts != last_color_frame_ts: - self.last_color_frame_ts = last_color_frame_ts - color = ColorFrame( - np.asanyarray(color_frame.get_data()), - current_time, - self.color_frame_index, - ) - self.color_frame_index += 1 - - depth = None - # if we're expecting depth frames - if rs.stream.depth in self.stream_profiles: - depth_frame = frames.get_depth_frame() - last_depth_frame_ts = depth_frame.get_timestamp() - if self.last_depth_frame_ts != last_depth_frame_ts: - self.last_depth_frame_ts = last_depth_frame_ts - depth = DepthFrame( - np.asanyarray(depth_frame.get_data()), - current_time, - self.depth_frame_index, - ) - self.depth_frame_index += 1 - - return color, depth - return None, None - - def recent_events(self, events): - if self._needs_restart or not self.online: - logger.debug("recent_events -> restarting device") - self.restart_device() - time.sleep(0.01) - return - - try: - color_frame, depth_frame = self.get_frames() - except RuntimeError as re: - logger.warning("Realsense failed to provide frames." + str(re)) - self._recent_frame = None - self._recent_depth_frame = None - self._needs_restart = True - else: - if color_frame is not None: - self._recent_frame = color_frame - events["frame"] = color_frame - - if depth_frame is not None: - self._recent_depth_frame = depth_frame - events["depth_frame"] = depth_frame - - if self.depth_video_writer is not None: - self.depth_video_writer.write_video_frame(depth_frame) - - def deinit_ui(self): - self.remove_menu() - - def init_ui(self): - self.add_menu() - self.menu.label = "Local USB Video Source" - self.update_menu() - - def update_menu(self): - logger.debug("update_menu") - try: - del self.menu[:] - except AttributeError: - return - - from pyglui import ui - - if not self.online: - self.menu.append(ui.Info_Text("Capture initialization failed.")) - return - - self.menu.append(ui.Switch("record_depth", self, label="Record Depth Stream")) - self.menu.append(ui.Switch("preview_depth", self, label="Preview Depth")) - - if self._available_modes is not None: - - def frame_size_selection_getter(): - if self.device_id: - frame_size = sorted( - self._available_modes[rs.stream.color], reverse=True - ) - labels = ["({}, {})".format(t[0], t[1]) for t in frame_size] - return frame_size, labels - else: - return [self.frame_size_backup], [str(self.frame_size_backup)] - - selector = ui.Selector( - "frame_size", - self, - selection_getter=frame_size_selection_getter, - label="Color Resolution", - ) - self.menu.append(selector) - - def frame_rate_selection_getter(): - if self.device_id: - avail_fps = [ - fps - for fps in self._available_modes[rs.stream.color][ - self.frame_size - ] - ] - return avail_fps, [str(fps) for fps in avail_fps] - else: - return [self.frame_rate_backup], [str(self.frame_rate_backup)] - - selector = ui.Selector( - "frame_rate", - self, - selection_getter=frame_rate_selection_getter, - label="Color Frame Rate", - ) - self.menu.append(selector) - - def depth_frame_size_selection_getter(): - if self.device_id: - depth_sizes = sorted( - self._available_modes[rs.stream.depth], reverse=True - ) - labels = ["({}, {})".format(t[0], t[1]) for t in depth_sizes] - return depth_sizes, labels - else: - return ( - [self.depth_frame_size_backup], - [str(self.depth_frame_size_backup)], - ) - - selector = ui.Selector( - "depth_frame_size", - self, - selection_getter=depth_frame_size_selection_getter, - label="Depth Resolution", - ) - self.menu.append(selector) - - def depth_frame_rate_selection_getter(): - if self.device_id: - avail_fps = [ - fps - for fps in self._available_modes[rs.stream.depth][ - self.depth_frame_size - ] - ] - return avail_fps, [str(fps) for fps in avail_fps] - else: - return ( - [self.depth_frame_rate_backup], - [str(self.depth_frame_rate_backup)], - ) - - selector = ui.Selector( - "depth_frame_rate", - self, - selection_getter=depth_frame_rate_selection_getter, - label="Depth Frame Rate", - ) - self.menu.append(selector) - - def reset_options(): - logger.debug("reset_options") - self.reset_device(self.device_id) - - sensor_control = ui.Growing_Menu(label="Sensor Settings") - sensor_control.append( - ui.Button("Reset device options to default", reset_options) - ) - self.menu.append(sensor_control) - else: - logger.debug("update_menu: self._available_modes is None") - - def gl_display(self): - - if self.preview_depth and self._recent_depth_frame is not None: - self.g_pool.image_tex.update_from_ndarray(self._recent_depth_frame.bgr) - gl_utils.glFlush() - gl_utils.make_coord_system_norm_based() - self.g_pool.image_tex.draw() - elif self._recent_frame is not None: - self.g_pool.image_tex.update_from_yuv_buffer( - self._recent_frame.yuv_buffer, - self._recent_frame.width, - self._recent_frame.height, - ) - gl_utils.glFlush() - gl_utils.make_coord_system_norm_based() - self.g_pool.image_tex.draw() - - if not self.online: - super().gl_display() - - gl_utils.make_coord_system_pixel_based( - (self.frame_size[1], self.frame_size[0], 3) - ) - - def reset_device(self, device_id): - logger.debug("reset_device") - if device_id is None: - device_id = self.device_id - - self.notify_all( - { - "subject": "realsense2_source.restart", - "device_id": device_id, - "color_frame_size": None, - "color_fps": None, - "depth_frame_size": None, - "depth_fps": None, - "device_options": [], # FIXME - } - ) - - def restart_device( - self, - color_frame_size=None, - color_fps=None, - depth_frame_size=None, - depth_fps=None, - device_options=None, - ): - if color_frame_size is None: - color_frame_size = self.frame_size - if color_fps is None: - color_fps = self.frame_rate - if depth_frame_size is None: - depth_frame_size = self.depth_frame_size - if depth_fps is None: - depth_fps = self.depth_frame_rate - if device_options is None: - device_options = [] # FIXME - - self.notify_all( - { - "subject": "realsense2_source.restart", - "device_id": None, - "color_frame_size": color_frame_size, - "color_fps": color_fps, - "depth_frame_size": depth_frame_size, - "depth_fps": depth_fps, - "device_options": device_options, - } - ) - logger.debug("self.restart_device --> self.notify_all") - - def on_notify(self, notification): - logger.debug( - 'self.on_notify, notification["subject"]: ' + notification["subject"] - ) - if notification["subject"] == "realsense2_source.restart": - kwargs = notification.copy() - del kwargs["subject"] - del kwargs["topic"] - self._initialize_device(**kwargs) - elif notification["subject"] == "recording.started": - self.start_depth_recording(notification["rec_path"], notification["start_time_synced"]) - elif notification["subject"] == "recording.stopped": - self.stop_depth_recording() - - def start_depth_recording(self, rec_loc, start_time_synced): - if not self.record_depth: - return - - if self.depth_video_writer is not None: - logger.warning("Depth video recording has been started already") - return - - video_path = os.path.join(rec_loc, "depth.mp4") - self.depth_video_writer = MPEG_Writer(video_path, start_time_synced) - - def stop_depth_recording(self): - if self.depth_video_writer is None: - logger.warning("Depth video recording was not running") - return - - self.depth_video_writer.close() - self.depth_video_writer = None - - @property - def device_id(self): - if self.online: # already running - return self.pipeline_profile.get_device().get_info( - rs.camera_info.serial_number - ) - else: - # set the first available device - devices = self.context.query_devices() - if devices: - logger.info("device_id: first device by default.") - return devices[0].get_info(rs.camera_info.serial_number) - else: - logger.debug("device_id: No device connected.") - return None - - @property - def frame_size(self): - try: - stream_profile = self.stream_profiles[rs.stream.color] - # TODO check width & height is in self.available modes - return stream_profile.width(), stream_profile.height() - except AttributeError: - return self.frame_size_backup - except KeyError: - return self.frame_size_backup - except TypeError: - return self.frame_size_backup - - @frame_size.setter - def frame_size(self, new_size): - if new_size != self.frame_size: - self.restart_device(color_frame_size=new_size) - - @property - def frame_rate(self): - try: - stream_profile = self.stream_profiles[rs.stream.color] - # TODO check FPS is in self.available modes - return stream_profile.fps() - except AttributeError: - return self.frame_rate_backup - except KeyError: - return self.frame_rate_backup - except TypeError: - return self.frame_rate_backup - - @frame_rate.setter - def frame_rate(self, new_rate): - if new_rate != self.frame_rate: - self.restart_device(color_fps=new_rate) - - @property - def depth_frame_size(self): - try: - stream_profile = self.stream_profiles[rs.stream.depth] - # TODO check width & height is in self.available modes - return stream_profile.width(), stream_profile.height() - except AttributeError: - return self.depth_frame_size_backup - except KeyError: - return self.depth_frame_size_backup - except TypeError: - return self.depth_frame_size_backup - - @depth_frame_size.setter - def depth_frame_size(self, new_size): - if new_size != self.depth_frame_size: - self.restart_device(depth_frame_size=new_size) - - @property - def depth_frame_rate(self): - try: - stream_profile = self.stream_profiles[rs.stream.depth] - return stream_profile.fps() - except AttributeError: - return self.depth_frame_rate_backup - except KeyError: - return self.depth_frame_rate_backup - except TypeError: - return self.depth_frame_rate_backup - - @depth_frame_rate.setter - def depth_frame_rate(self, new_rate): - if new_rate != self.depth_frame_rate: - self.restart_device(depth_fps=new_rate) - - @property - def jpeg_support(self): - return False - - @property - def online(self): - return self.pipeline_profile is not None and self.pipeline is not None - - @property - def name(self): - if self.online: - return self.pipeline_profile.get_device().get_info(rs.camera_info.name) - else: - logger.debug( - "self.name: Realsense2 not online. Falling back to Ghost capture" - ) - return "Ghost capture" - - -class Realsense2_Manager(Base_Manager): - """Manages Intel RealSense D400 sources - - Attributes: - check_intervall (float): Intervall in which to look for new UVC devices - """ - - gui_name = "RealSense D400" - - def get_init_dict(self): - return {} - - def init_ui(self): - self.add_menu() - from pyglui import ui - - self.menu.append(ui.Info_Text("Intel RealSense D400 sources")) - - def is_streaming(device_id): - try: - c = rs.config() - c.enable_device(device_id) # device_id is in fact the serial_number - p = rs.pipeline() - p.start(c) - p.stop() - return False - except RuntimeError: - return True - - def get_device_info(d): - name = d.get_info(rs.camera_info.name) # FIXME is camera in use? - device_id = d.get_info(rs.camera_info.serial_number) - - fmt = "- " if is_streaming(device_id) else "" - fmt += name - - return device_id, fmt - - def dev_selection_list(): - default = (None, "Select to activate") - try: - ctx = rs.context() # FIXME cannot use "with rs.context() as ctx:" - # got "AttributeError: __enter__" - # see https://stackoverflow.com/questions/5093382/object-becomes-none-when-using-a-context-manager - dev_pairs = [default] + [get_device_info(d) for d in ctx.devices] - except Exception: # FIXME - dev_pairs = [default] - - return zip(*dev_pairs) - - def activate(source_uid): - if source_uid is None: - return - - settings = { - "frame_size": self.g_pool.capture.frame_size, - "frame_rate": self.g_pool.capture.frame_rate, - "device_id": source_uid, - } - if self.g_pool.process == "world": - self.notify_all( - { - "subject": "start_plugin", - "name": "Realsense2_Source", - "args": settings, - } - ) - else: - self.notify_all( - { - "subject": "start_eye_plugin", - "target": self.g_pool.process, - "name": "Realsense2_Source", - "args": settings, - } - ) - - self.menu.append( - ui.Selector( - "selected_source", - selection_getter=dev_selection_list, - getter=lambda: None, - setter=activate, - label="Activate source", - ) - ) - - def deinit_ui(self): - self.remove_menu() diff --git a/pupil_src/shared_modules/video_capture/realsense_backend.py b/pupil_src/shared_modules/video_capture/realsense_backend.py deleted file mode 100755 index 7b64befc72..0000000000 --- a/pupil_src/shared_modules/video_capture/realsense_backend.py +++ /dev/null @@ -1,935 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2020 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" - -import logging -import time -import cv2 -import os - -import pyrealsense as pyrs -from pyrealsense.stream import ColorStream, DepthStream, DACStream, PointStream -from pyrealsense.constants import rs_stream, rs_option, rs_preset -from pyrealsense.extlib import rsutilwrapper - -from version_utils import VersionFormat -from .base_backend import Base_Source, Base_Manager -from av_writer import MPEG_Writer -from camera_models import load_intrinsics - -import glfw -import gl_utils -from OpenGL.GL import * -from OpenGL.GLU import * -from pyglui import cygl -import cython_methods -import numpy as np -from ctypes import * - -# check versions for our own depedencies as they are fast-changing -assert VersionFormat(pyrs.__version__) >= VersionFormat("2.2") - -# logging -logging.getLogger("pyrealsense").setLevel(logging.ERROR + 1) -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -class ColorFrame(object): - def __init__(self, device): - # we need to keep this since there is no cv2 conversion for our planar format - self._yuv422 = device.color - self._shape = self._yuv422.shape[:2] - self._yuv = np.empty(self._yuv422.size, dtype=np.uint8) - y_plane = self._yuv422.size // 2 - u_plane = y_plane // 2 - self._yuv[:y_plane] = self._yuv422[:, :, 0].flatten() - self._yuv[y_plane : y_plane + u_plane] = self._yuv422[:, ::2, 1].flatten() - self._yuv[y_plane + u_plane :] = self._yuv422[:, 1::2, 1].flatten() - self._bgr = None - self._gray = None - - @property - def height(self): - return self._shape[0] - - @property - def width(self): - return self._shape[1] - - @property - def yuv_buffer(self): - return self._yuv - - @property - def yuv422(self): - Y = self._yuv[: self._yuv.size // 2] - U = self._yuv[self._yuv.size // 2 : 3 * self._yuv.size // 4] - V = self._yuv[3 * self._yuv.size // 4 :] - - Y.shape = self._shape - U.shape = self._shape[0], self._shape[1] // 2 - V.shape = self._shape[0], self._shape[1] // 2 - - return Y, U, V - - @property - def bgr(self): - if self._bgr is None: - self._bgr = cv2.cvtColor(self._yuv422, cv2.COLOR_YUV2BGR_YUYV) - return self._bgr - - @property - def img(self): - return self.bgr - - @property - def gray(self): - if self._gray is None: - self._gray = self._yuv[: self._yuv.size // 2] - self._gray.shape = self._shape - return self._gray - - -class DepthFrame(object): - def __init__(self, device): - self._bgr = None - self._gray = None - self.depth = device.depth - self.yuv_buffer = None - - @property - def height(self): - return self.depth.shape[0] - - @property - def width(self): - return self.depth.shape[1] - - @property - def bgr(self): - if self._bgr is None: - self._bgr = cython_methods.cumhist_color_map16(self.depth) - return self._bgr - - @property - def img(self): - return self.bgr - - @property - def gray(self): - if self._gray is None: - self._gray = cv2.cvtColor(self.bgr, cv2.cv2.COLOR_BGR2GRAY) - return self._gray - - -class Control(object): - def __init__(self, device, opt_range, value): - self._dev = device - self._value = value - self.range = opt_range - self.label = rs_option.name_for_value[opt_range.option] - self.label = self.label.replace("RS_OPTION_", "") - self.label = self.label.replace("R200_", "") - self.label = self.label.replace("_", " ") - self.label = self.label.title() - self.description = self._dev.get_device_option_description(opt_range.option) - - @property - def value(self): - return self._value - - @value.setter - def value(self, val): - try: - self._dev.set_device_option(self.range.option, val) - except pyrs.RealsenseError as err: - logger.error('Setting option "{}" failed'.format(self.label)) - logger.debug("Reason: {}".format(err)) - else: - self._value = val - - def refresh(self): - self._value = self._dev.get_device_option(self.range.option) - - -class Realsense_Controls(dict): - def __init__(self, device, presets=()): - if not device: - super().__init__() - return - if presets: - # presets: list of (option, value)-tuples - try: - device.set_device_options(*zip(*presets)) - except pyrs.RealsenseError as err: - logger.error("Setting device option presets failed") - logger.debug("Reason: {}".format(err)) - controls = {} - for opt_range, value in device.get_available_options(): - if opt_range.min < opt_range.max: - controls[opt_range.option] = Control(device, opt_range, value) - super().__init__(controls) - - def export_presets(self): - return [(opt, ctrl.value) for opt, ctrl in self.items()] - - def refresh(self): - for ctrl in self.values(): - ctrl.refresh() - - -class Realsense_Source(Base_Source): - """ - Camera Capture is a class that encapsualtes pyrs.Device: - """ - - def __init__( - self, - g_pool, - device_id=0, - frame_size=(1920, 1080), - frame_rate=30, - depth_frame_size=(640, 480), - depth_frame_rate=60, - align_streams=False, - preview_depth=False, - device_options=(), - record_depth=True, - stream_preset=None, - ): - super().__init__(g_pool) - self._intrinsics = None - self.color_frame_index = 0 - self.depth_frame_index = 0 - self.device = None - self.service = pyrs.Service() - self.align_streams = align_streams - self.preview_depth = preview_depth - self.record_depth = record_depth - self.depth_video_writer = None - self.controls = None - self.pitch = 0 - self.yaw = 0 - self.mouse_drag = False - self.last_pos = (0, 0) - self.depth_window = None - self._needs_restart = False - self.stream_preset = stream_preset - self._initialize_device( - device_id, - frame_size, - frame_rate, - depth_frame_size, - depth_frame_rate, - device_options, - ) - - def _initialize_device( - self, - device_id, - color_frame_size, - color_fps, - depth_frame_size, - depth_fps, - device_options=(), - ): - devices = tuple(self.service.get_devices()) - color_frame_size = tuple(color_frame_size) - depth_frame_size = tuple(depth_frame_size) - - self.streams = [ColorStream(), DepthStream(), PointStream()] - self.last_color_frame_ts = None - self.last_depth_frame_ts = None - self._recent_frame = None - self._recent_depth_frame = None - - if not devices: - if not self._needs_restart: - logger.error("Camera failed to initialize. No cameras connected.") - self.device = None - self.update_menu() - return - - if self.device is not None: - self.device.stop() # only call Device.stop() if its context - - if device_id >= len(devices): - logger.error( - "Camera with id {} not found. Initializing default camera.".format( - device_id - ) - ) - device_id = 0 - - # use default streams to filter modes by rs_stream and rs_format - self._available_modes = self._enumerate_formats(device_id) - - # make sure that given frame sizes and rates are available - color_modes = self._available_modes[rs_stream.RS_STREAM_COLOR] - if color_frame_size not in color_modes: - # automatically select highest resolution - color_frame_size = sorted(color_modes.keys(), reverse=True)[0] - - if color_fps not in color_modes[color_frame_size]: - # automatically select highest frame rate - color_fps = color_modes[color_frame_size][0] - - depth_modes = self._available_modes[rs_stream.RS_STREAM_DEPTH] - if self.align_streams: - depth_frame_size = color_frame_size - else: - if depth_frame_size not in depth_modes: - # automatically select highest resolution - depth_frame_size = sorted(depth_modes.keys(), reverse=True)[0] - - if depth_fps not in depth_modes[depth_frame_size]: - # automatically select highest frame rate - depth_fps = depth_modes[depth_frame_size][0] - - colorstream = ColorStream( - width=color_frame_size[0], - height=color_frame_size[1], - fps=color_fps, - color_format="yuv", - preset=self.stream_preset, - ) - depthstream = DepthStream( - width=depth_frame_size[0], - height=depth_frame_size[1], - fps=depth_fps, - preset=self.stream_preset, - ) - pointstream = PointStream( - width=depth_frame_size[0], height=depth_frame_size[1], fps=depth_fps - ) - - self.streams = [colorstream, depthstream, pointstream] - if self.align_streams: - dacstream = DACStream( - width=depth_frame_size[0], height=depth_frame_size[1], fps=depth_fps - ) - dacstream.name = "depth" # rename data accessor - self.streams.append(dacstream) - - # update with correctly initialized streams - # always initiliazes color + depth, adds rectified/aligned versions as necessary - - self.device = self.service.Device(device_id, streams=self.streams) - self.controls = Realsense_Controls(self.device, device_options) - self._intrinsics = load_intrinsics( - self.g_pool.user_dir, self.name, self.frame_size - ) - - self.update_menu() - self._needs_restart = False - - def _enumerate_formats(self, device_id): - """Enumerate formats into hierachical structure: - - streams: - resolutions: - framerates - """ - formats = {} - # only lists modes for native streams (RS_STREAM_COLOR/RS_STREAM_DEPTH) - for mode in self.service.get_device_modes(device_id): - if mode.stream in (rs_stream.RS_STREAM_COLOR, rs_stream.RS_STREAM_DEPTH): - # check if frame size dict is available - if mode.stream not in formats: - formats[mode.stream] = {} - stream_obj = next((s for s in self.streams if s.stream == mode.stream)) - if mode.format == stream_obj.format: - size = mode.width, mode.height - # check if framerate list is already available - if size not in formats[mode.stream]: - formats[mode.stream][size] = [] - formats[mode.stream][size].append(mode.fps) - - if self.align_streams: - depth_sizes = formats[rs_stream.RS_STREAM_DEPTH].keys() - color_sizes = formats[rs_stream.RS_STREAM_COLOR].keys() - # common_sizes = depth_sizes & color_sizes - discarded_sizes = depth_sizes ^ color_sizes - for size in discarded_sizes: - for sizes in formats.values(): - if size in sizes: - del sizes[size] - - return formats - - def cleanup(self): - if self.depth_video_writer is not None: - self.stop_depth_recording() - if self.device is not None: - self.device.stop() - self.service.stop() - - def get_init_dict(self): - return { - "device_id": self.device.device_id if self.device is not None else 0, - "frame_size": self.frame_size, - "frame_rate": self.frame_rate, - "depth_frame_size": self.depth_frame_size, - "depth_frame_rate": self.depth_frame_rate, - "preview_depth": self.preview_depth, - "record_depth": self.record_depth, - "align_streams": self.align_streams, - "device_options": self.controls.export_presets() - if self.controls is not None - else (), - "stream_preset": self.stream_preset, - } - - def get_frames(self): - if self.device: - self.device.wait_for_frames() - current_time = self.g_pool.get_timestamp() - - last_color_frame_ts = self.device.get_frame_timestamp( - self.streams[0].stream - ) - if self.last_color_frame_ts != last_color_frame_ts: - self.last_color_frame_ts = last_color_frame_ts - color = ColorFrame(self.device) - color.timestamp = current_time - color.index = self.color_frame_index - self.color_frame_index += 1 - else: - color = None - - last_depth_frame_ts = self.device.get_frame_timestamp( - self.streams[1].stream - ) - if self.last_depth_frame_ts != last_depth_frame_ts: - self.last_depth_frame_ts = last_depth_frame_ts - depth = DepthFrame(self.device) - depth.timestamp = current_time - depth.index = self.depth_frame_index - self.depth_frame_index += 1 - else: - depth = None - - return color, depth - return None, None - - def recent_events(self, events): - if self._needs_restart: - self.restart_device() - time.sleep(0.05) - elif not self.online: - time.sleep(0.05) - return - - try: - color_frame, depth_frame = self.get_frames() - except (pyrs.RealsenseError, TimeoutError) as err: - logger.warning("Realsense failed to provide frames. Attempting to reinit.") - self._recent_frame = None - self._recent_depth_frame = None - self._needs_restart = True - else: - if color_frame and depth_frame: - self._recent_frame = color_frame - events["frame"] = color_frame - - if depth_frame: - self._recent_depth_frame = depth_frame - events["depth_frame"] = depth_frame - - if self.depth_video_writer is not None: - self.depth_video_writer.write_video_frame(depth_frame) - - def deinit_ui(self): - self.remove_menu() - - def init_ui(self): - self.add_menu() - self.menu.label = "Local USB Video Source" - self.update_menu() - - def update_menu(self): - try: - del self.menu[:] - except AttributeError: - return - - from pyglui import ui - - if self.device is None: - self.menu.append(ui.Info_Text("Capture initialization failed.")) - return - - def align_and_restart(val): - self.align_streams = val - self.restart_device() - - self.menu.append(ui.Switch("record_depth", self, label="Record Depth Stream")) - self.menu.append(ui.Switch("preview_depth", self, label="Preview Depth")) - self.menu.append( - ui.Switch( - "align_streams", self, label="Align Streams", setter=align_and_restart - ) - ) - - def toggle_depth_display(): - def on_depth_mouse_button(window, button, action, mods): - if button == glfw.GLFW_MOUSE_BUTTON_LEFT and action == glfw.GLFW_PRESS: - self.mouse_drag = True - if ( - button == glfw.GLFW_MOUSE_BUTTON_LEFT - and action == glfw.GLFW_RELEASE - ): - self.mouse_drag = False - - if self.depth_window is None: - self.pitch = 0 - self.yaw = 0 - - win_size = glfw.glfwGetWindowSize(self.g_pool.main_window) - self.depth_window = glfw.glfwCreateWindow( - win_size[0], win_size[1], "3D Point Cloud" - ) - glfw.glfwSetMouseButtonCallback( - self.depth_window, on_depth_mouse_button - ) - active_window = glfw.glfwGetCurrentContext() - glfw.glfwMakeContextCurrent(self.depth_window) - gl_utils.basic_gl_setup() - gl_utils.make_coord_system_norm_based() - - # refresh speed settings - glfw.glfwSwapInterval(0) - - glfw.glfwMakeContextCurrent(active_window) - - native_presets = [ - ("None", None), - ("Best Quality", rs_preset.RS_PRESET_BEST_QUALITY), - ("Largest image", rs_preset.RS_PRESET_LARGEST_IMAGE), - ("Highest framerate", rs_preset.RS_PRESET_HIGHEST_FRAMERATE), - ] - - def set_stream_preset(val): - if self.stream_preset != val: - self.stream_preset = val - self.restart_device() - - self.menu.append( - ui.Selector( - "stream_preset", - self, - setter=set_stream_preset, - labels=[preset[0] for preset in native_presets], - selection=[preset[1] for preset in native_presets], - label="Stream preset", - ) - ) - color_sizes = sorted( - self._available_modes[rs_stream.RS_STREAM_COLOR], reverse=True - ) - selector = ui.Selector( - "frame_size", - self, - # setter=, - selection=color_sizes, - label="Resolution" if self.align_streams else "Color Resolution", - ) - selector.read_only = self.stream_preset is not None - self.menu.append(selector) - - def color_fps_getter(): - avail_fps = [ - fps - for fps in self._available_modes[rs_stream.RS_STREAM_COLOR][ - self.frame_size - ] - if self.depth_frame_rate % fps == 0 - ] - return avail_fps, [str(fps) for fps in avail_fps] - - selector = ui.Selector( - "frame_rate", - self, - # setter=, - selection_getter=color_fps_getter, - label="Color Frame Rate", - ) - selector.read_only = self.stream_preset is not None - self.menu.append(selector) - - if not self.align_streams: - depth_sizes = sorted( - self._available_modes[rs_stream.RS_STREAM_DEPTH], reverse=True - ) - selector = ui.Selector( - "depth_frame_size", - self, - # setter=, - selection=depth_sizes, - label="Depth Resolution", - ) - selector.read_only = self.stream_preset is not None - self.menu.append(selector) - - def depth_fps_getter(): - avail_fps = [ - fps - for fps in self._available_modes[rs_stream.RS_STREAM_DEPTH][ - self.depth_frame_size - ] - if fps % self.frame_rate == 0 - ] - return avail_fps, [str(fps) for fps in avail_fps] - - selector = ui.Selector( - "depth_frame_rate", - self, - selection_getter=depth_fps_getter, - label="Depth Frame Rate", - ) - selector.read_only = self.stream_preset is not None - self.menu.append(selector) - - def reset_options(): - if self.device: - try: - self.device.reset_device_options_to_default(self.controls.keys()) - except pyrs.RealsenseError as err: - logger.info("Resetting some device options failed") - logger.debug("Reason: {}".format(err)) - finally: - self.controls.refresh() - - self.menu.append(ui.Button("Point Cloud Window", toggle_depth_display)) - sensor_control = ui.Growing_Menu(label="Sensor Settings") - sensor_control.append( - ui.Button("Reset device options to default", reset_options) - ) - for ctrl in sorted(self.controls.values(), key=lambda x: x.range.option): - # sensor_control.append(ui.Info_Text(ctrl.description)) - if ( - ctrl.range.min == 0.0 - and ctrl.range.max == 1.0 - and ctrl.range.step == 1.0 - ): - sensor_control.append( - ui.Switch("value", ctrl, label=ctrl.label, off_val=0.0, on_val=1.0) - ) - else: - sensor_control.append( - ui.Slider( - "value", - ctrl, - label=ctrl.label, - min=ctrl.range.min, - max=ctrl.range.max, - step=ctrl.range.step, - ) - ) - self.menu.append(sensor_control) - - def gl_display(self): - from math import floor - - if self.depth_window is not None and glfw.glfwWindowShouldClose( - self.depth_window - ): - glfw.glfwDestroyWindow(self.depth_window) - self.depth_window = None - - if self.depth_window is not None and self._recent_depth_frame is not None: - active_window = glfw.glfwGetCurrentContext() - glfw.glfwMakeContextCurrent(self.depth_window) - - win_size = glfw.glfwGetFramebufferSize(self.depth_window) - gl_utils.adjust_gl_view(win_size[0], win_size[1]) - pos = glfw.glfwGetCursorPos(self.depth_window) - if self.mouse_drag: - self.pitch = np.clip(self.pitch + (pos[1] - self.last_pos[1]), -80, 80) - self.yaw = np.clip(self.yaw - (pos[0] - self.last_pos[0]), -120, 120) - self.last_pos = pos - - glClearColor(0, 0, 0, 0) - glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) - glMatrixMode(GL_PROJECTION) - glLoadIdentity() - gluPerspective(60, win_size[0] / win_size[1], 0.01, 20.0) - glMatrixMode(GL_MODELVIEW) - glLoadIdentity() - gluLookAt(0, 0, 0, 0, 0, 1, 0, -1, 0) - glTranslatef(0, 0, 0.5) - glRotated(self.pitch, 1, 0, 0) - glRotated(self.yaw, 0, 1, 0) - glTranslatef(0, 0, -0.5) - - # glPointSize(2) - glEnable(GL_DEPTH_TEST) - extrinsics = self.device.get_device_extrinsics( - rs_stream.RS_STREAM_DEPTH, rs_stream.RS_STREAM_COLOR - ) - depth_frame = self._recent_depth_frame - color_frame = self._recent_frame - depth_scale = self.device.depth_scale - - glEnableClientState(GL_VERTEX_ARRAY) - - pointcloud = self.device.pointcloud - glVertexPointer(3, GL_FLOAT, 0, pointcloud) - glEnableClientState(GL_COLOR_ARRAY) - depth_to_color = np.zeros( - depth_frame.height * depth_frame.width * 3, np.uint8 - ) - rsutilwrapper.project_pointcloud_to_pixel( - depth_to_color, - self.device.depth_intrinsics, - self.device.color_intrinsics, - extrinsics, - pointcloud, - self._recent_frame.bgr, - ) - glColorPointer(3, GL_UNSIGNED_BYTE, 0, depth_to_color) - glDrawArrays(GL_POINTS, 0, depth_frame.width * depth_frame.height) - gl_utils.glFlush() - glDisable(GL_DEPTH_TEST) - # gl_utils.make_coord_system_norm_based() - glfw.glfwSwapBuffers(self.depth_window) - glfw.glfwMakeContextCurrent(active_window) - - if self.preview_depth and self._recent_depth_frame is not None: - self.g_pool.image_tex.update_from_ndarray(self._recent_depth_frame.bgr) - gl_utils.glFlush() - gl_utils.make_coord_system_norm_based() - self.g_pool.image_tex.draw() - elif self._recent_frame is not None: - self.g_pool.image_tex.update_from_yuv_buffer( - self._recent_frame.yuv_buffer, - self._recent_frame.width, - self._recent_frame.height, - ) - gl_utils.glFlush() - gl_utils.make_coord_system_norm_based() - self.g_pool.image_tex.draw() - - if not self.online: - super().gl_display() - - gl_utils.make_coord_system_pixel_based( - (self.frame_size[1], self.frame_size[0], 3) - ) - - def restart_device( - self, - device_id=None, - color_frame_size=None, - color_fps=None, - depth_frame_size=None, - depth_fps=None, - device_options=None, - ): - if device_id is None: - if self.device is not None: - device_id = self.device.device_id - else: - device_id = 0 - if color_frame_size is None: - color_frame_size = self.frame_size - if color_fps is None: - color_fps = self.frame_rate - if depth_frame_size is None: - depth_frame_size = self.depth_frame_size - if depth_fps is None: - depth_fps = self.depth_frame_rate - if device_options is None: - device_options = self.controls.export_presets() - if self.device is not None: - self.device.stop() - self.device = None - self.service.stop() - self.service.start() - self.notify_all( - { - "subject": "realsense_source.restart", - "device_id": device_id, - "color_frame_size": color_frame_size, - "color_fps": color_fps, - "depth_frame_size": depth_frame_size, - "depth_fps": depth_fps, - "device_options": device_options, - } - ) - - def on_click(self, pos, button, action): - if button == glfw.GLFW_MOUSE_BUTTON_LEFT and action == glfw.GLFW_PRESS: - self.mouse_drag = True - if button == glfw.GLFW_MOUSE_BUTTON_LEFT and action == glfw.GLFW_RELEASE: - self.mouse_drag = False - - def on_notify(self, notification): - if notification["subject"] == "realsense_source.restart": - kwargs = notification.copy() - del kwargs["subject"] - del kwargs["topic"] - self._initialize_device(**kwargs) - elif notification["subject"] == "recording.started": - self.start_depth_recording(notification["rec_path"], notification["start_time_synced"]) - elif notification["subject"] == "recording.stopped": - self.stop_depth_recording() - - def start_depth_recording(self, rec_loc, start_time_synced): - if not self.record_depth: - return - - if self.depth_video_writer is not None: - logger.warning("Depth video recording has been started already") - return - - video_path = os.path.join(rec_loc, "depth.mp4") - self.depth_video_writer = MPEG_Writer(video_path, start_time_synced) - - def stop_depth_recording(self): - if self.depth_video_writer is None: - logger.warning("Depth video recording was not running") - return - - self.depth_video_writer.close() - self.depth_video_writer = None - - @property - def frame_size(self): - stream = self.streams[0] - return stream.width, stream.height - - @frame_size.setter - def frame_size(self, new_size): - if self.device is not None and new_size != self.frame_size: - self.restart_device(color_frame_size=new_size) - - @property - def frame_rate(self): - return self.streams[0].fps - - @frame_rate.setter - def frame_rate(self, new_rate): - if self.device is not None and new_rate != self.frame_rate: - self.restart_device(color_fps=new_rate) - - @property - def depth_frame_size(self): - stream = self.streams[1] - return stream.width, stream.height - - @depth_frame_size.setter - def depth_frame_size(self, new_size): - if self.device is not None and new_size != self.depth_frame_size: - self.restart_device(depth_frame_size=new_size) - - @property - def depth_frame_rate(self): - return self.streams[1].fps - - @depth_frame_rate.setter - def depth_frame_rate(self, new_rate): - if self.device is not None and new_rate != self.depth_frame_rate: - self.restart_device(depth_fps=new_rate) - - @property - def jpeg_support(self): - return False - - @property - def online(self): - return self.device and self.device.is_streaming() - - @property - def name(self): - # not the same as `if self.device:`! - if self.device is not None: - return self.device.name - else: - return "Ghost capture" - - -class Realsense_Manager(Base_Manager): - """Manages Intel RealSense R200 sources - - Attributes: - check_intervall (float): Intervall in which to look for new UVC devices - """ - - gui_name = "RealSense R200" - - def get_init_dict(self): - return {} - - def init_ui(self): - self.add_menu() - from pyglui import ui - - self.menu.append(ui.Info_Text("Intel RealSense R200 sources")) - - def pair(d): - fmt = "- " if d["is_streaming"] else "" - fmt += d["name"] - return d["id"], fmt - - def dev_selection_list(): - default = (None, "Select to activate") - try: - with pyrs.Service() as service: - dev_pairs = [default] + [pair(d) for d in service.get_devices()] - except pyrs.RealsenseError: - dev_pairs = [default] - - return zip(*dev_pairs) - - def activate(source_uid): - if source_uid is None: - return - - # with pyrs.Service() as service: - # if not service.is_device_streaming(source_uid): - # logger.error("The selected camera is already in use or blocked.") - # return - settings = { - "frame_size": self.g_pool.capture.frame_size, - "frame_rate": self.g_pool.capture.frame_rate, - "device_id": source_uid, - } - if self.g_pool.process == "world": - self.notify_all( - { - "subject": "start_plugin", - "name": "Realsense_Source", - "args": settings, - } - ) - else: - self.notify_all( - { - "subject": "start_eye_plugin", - "target": self.g_pool.process, - "name": "Realsense_Source", - "args": settings, - } - ) - - self.menu.append( - ui.Selector( - "selected_source", - selection_getter=dev_selection_list, - getter=lambda: None, - setter=activate, - label="Activate source", - ) - ) - - def deinit_ui(self): - self.remove_menu() From a8e679ad77c2722200beb1e30c02f20e5fadf07d Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 20 Jan 2020 17:12:45 +0100 Subject: [PATCH 03/65] Add uniform source plugin name "Video Source" to Base_Source --- pupil_src/shared_modules/video_capture/base_backend.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 06ce000871..47a5871f11 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -63,6 +63,10 @@ class Base_Source(Plugin): icon_chr = chr(0xE412) icon_font = "pupil_icons" + @property + def pretty_class_name(self): + return "Video Source" + def __init__(self, g_pool): super().__init__(g_pool) self.g_pool.capture = self From 5f6cee9942ba607da6b212c35d8142242975f518 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 20 Jan 2020 17:56:26 +0100 Subject: [PATCH 04/65] Move pyglui.ui import to top of the files and sort backend imports --- .../video_capture/file_backend.py | 17 +++++++++-------- .../video_capture/hmd_streaming.py | 2 +- .../video_capture/ndsi_backend.py | 10 +++++----- .../shared_modules/video_capture/uvc_backend.py | 3 +-- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index a5a1437f7c..271264c536 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -13,20 +13,22 @@ import logging import os import os.path -import av -import numpy as np import typing as T - -from multiprocessing import cpu_count from abc import ABC, abstractmethod +from multiprocessing import cpu_count from time import sleep -from camera_models import load_intrinsics -from .utils import VideoSet + +import av +import numpy as np +from pyglui import ui import player_methods as pm -from .base_backend import Base_Manager, Base_Source, EndofVideoError, Playback_Source +from camera_models import load_intrinsics from pupil_recording import PupilRecording +from .base_backend import Base_Manager, Base_Source, EndofVideoError, Playback_Source +from .utils import VideoSet + logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) av.logging.set_level(av.logging.ERROR) @@ -512,7 +514,6 @@ def on_notify(self, notification): def init_ui(self): self.add_menu() self.menu.label = "File Source: {}".format(os.path.split(self.source_path)[-1]) - from pyglui import ui self.menu.append( ui.Info_Text( diff --git a/pupil_src/shared_modules/video_capture/hmd_streaming.py b/pupil_src/shared_modules/video_capture/hmd_streaming.py index 6648f0d60b..4135c5c154 100644 --- a/pupil_src/shared_modules/video_capture/hmd_streaming.py +++ b/pupil_src/shared_modules/video_capture/hmd_streaming.py @@ -15,7 +15,7 @@ from pyglui import ui import zmq_tools -from camera_models import Radial_Dist_Camera, Dummy_Camera +from camera_models import Dummy_Camera, Radial_Dist_Camera from video_capture.base_backend import Base_Manager, Base_Source logger = logging.getLogger(__name__) diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index 886ec48e33..b1a752ef97 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -9,15 +9,17 @@ ---------------------------------------------------------------------------~(*) """ -import time import logging -from packaging.version import Version +import time import ndsi +from packaging.version import Version +from pyglui import ui -from .base_backend import Base_Source, Base_Manager from camera_models import load_intrinsics +from .base_backend import Base_Manager, Base_Source + try: from ndsi import __version__ @@ -251,7 +253,6 @@ def init_ui(self): self._sensor_name, self._host_name ) - from pyglui import ui self.has_ui = True self.uvc_menu = ui.Growing_Menu("UVC Controls") @@ -334,7 +335,6 @@ def initiate_value_change(val): def update_control_menu(self): if not self.has_ui: return - from pyglui import ui del self.menu[:] del self.uvc_menu[:] diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index d623eee9d5..834c03a551 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -16,7 +16,7 @@ import time import numpy as np -from pyglui import cygl +from pyglui import cygl, ui import gl_utils import uvc @@ -573,7 +573,6 @@ def init_ui(self): def update_menu(self): del self.menu[:] - from pyglui import ui ui_elements = [] From 326cd3bb723a5727618ae419c3d5bf82874f13e2 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 20 Jan 2020 17:57:44 +0100 Subject: [PATCH 05/65] Add skeleton for a shared menu across sources in Base_Source --- .../video_capture/base_backend.py | 11 ++++++++ .../video_capture/file_backend.py | 9 ++++--- .../video_capture/hmd_streaming.py | 12 ++------- .../video_capture/ndsi_backend.py | 27 ++++++++++--------- .../video_capture/uvc_backend.py | 10 ++----- 5 files changed, 35 insertions(+), 34 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 47a5871f11..83254a6369 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -77,6 +77,17 @@ def add_menu(self): super().add_menu() self.menu_icon.order = 0.2 + def init_ui(self): + self.add_menu() + self.menu.label = "Video Source" + self.update_menu() + + def deinit_ui(self): + self.remove_menu() + + def update_menu(self): + pass + def recent_events(self, events): """Returns None diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index 271264c536..b493186fc3 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -511,9 +511,12 @@ def on_notify(self, notification): ): self.play = False - def init_ui(self): - self.add_menu() - self.menu.label = "File Source: {}".format(os.path.split(self.source_path)[-1]) + def update_menu(self): + super().update_menu() + del self.menu[:] + self.menu.append( + ui.Info_Text(f"File Source: {os.path.split(self.source_path)[-1]}") + ) self.menu.append( ui.Info_Text( diff --git a/pupil_src/shared_modules/video_capture/hmd_streaming.py b/pupil_src/shared_modules/video_capture/hmd_streaming.py index 4135c5c154..5c3fbb2d9f 100644 --- a/pupil_src/shared_modules/video_capture/hmd_streaming.py +++ b/pupil_src/shared_modules/video_capture/hmd_streaming.py @@ -52,16 +52,8 @@ def __init__(self, g_pool, *args, **kwargs): topics=("hmd_streaming.world",), ) - # def get_init_dict(self): - - def init_ui(self): # was gui - self.add_menu() - self.menu.label = "HMD Streaming" - text = ui.Info_Text("HMD Streaming Info") - self.menu.append(text) - - def deinit_ui(self): - self.remove_menu() + def update_menu(self): + self.menu.append(ui.Info_Text(f"HMD Streaming")) def cleanup(self): self.frame_sub = None diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index b1a752ef97..f4b7ffc149 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -248,19 +248,16 @@ def get_init_dict(self): return settings def init_ui(self): - self.add_menu() - self.menu.label = "NDSI Source: {} @ {}".format( - self._sensor_name, self._host_name - ) - - self.has_ui = True - self.uvc_menu = ui.Growing_Menu("UVC Controls") + super().init_ui() + + def update_menu(self): + # TODO: Refactor this to be more uniform across sources self.update_control_menu() def deinit_ui(self): - self.uvc_menu = None - self.remove_menu() + super().deinit_ui() + # TODO: Refactor this to be more uniform across sources self.has_ui = False def add_controls_to_menu(self, menu, controls): @@ -337,7 +334,13 @@ def update_control_menu(self): return del self.menu[:] - del self.uvc_menu[:] + + self.menu.append( + ui.Info_Text(f"NDSI Source: {self._sensor_name} @ {self._host_name}") + ) + + self.uvc_menu = ui.Growing_Menu("UVC Controls") + self.control_id_ui_mapping = {} if not self.sensor: self.menu.append( @@ -383,8 +386,7 @@ class NDSI_Manager(Base_Manager): def __init__(self, g_pool): super().__init__(g_pool) self.network = ndsi.Network( - formats={ndsi.DataFormat.V3, ndsi.DataFormat.V4}, - callbacks=(self.on_event,) + formats={ndsi.DataFormat.V3, ndsi.DataFormat.V4}, callbacks=(self.on_event,) ) self.network.start() self.selected_host = None @@ -613,4 +615,3 @@ def select_host(self, selected_host): else: self.should_select_host = selected_host - diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index 834c03a551..d9967978b1 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -563,16 +563,10 @@ def jpeg_support(self): def online(self): return bool(self.uvc_capture) - def deinit_ui(self): - self.remove_menu() - - def init_ui(self): - self.add_menu() - self.menu.label = "Local USB Source: {}".format(self.name) - self.update_menu() - def update_menu(self): + super().update_menu() del self.menu[:] + self.menu.append(ui.Info_Text(f"Local USB Source: {self.name}")) ui_elements = [] From 9bd994fc1e4781fcf7b92eea96060635a18b3e9c Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 20 Jan 2020 18:34:02 +0100 Subject: [PATCH 06/65] Streamline ndsi_backend menu updating --- .../shared_modules/video_capture/base_backend.py | 2 +- .../shared_modules/video_capture/file_backend.py | 1 - .../shared_modules/video_capture/ndsi_backend.py | 11 ++++++----- pupil_src/shared_modules/video_capture/uvc_backend.py | 1 - 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 83254a6369..fcb52f3242 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -86,7 +86,7 @@ def deinit_ui(self): self.remove_menu() def update_menu(self): - pass + del self.menu[:] def recent_events(self, events): """Returns None diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index b493186fc3..166cd87292 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -513,7 +513,6 @@ def on_notify(self, notification): def update_menu(self): super().update_menu() - del self.menu[:] self.menu.append( ui.Info_Text(f"File Source: {os.path.split(self.source_path)[-1]}") ) diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index f4b7ffc149..40168bbc40 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -251,10 +251,6 @@ def init_ui(self): self.has_ui = True super().init_ui() - def update_menu(self): - # TODO: Refactor this to be more uniform across sources - self.update_control_menu() - def deinit_ui(self): super().deinit_ui() # TODO: Refactor this to be more uniform across sources @@ -330,10 +326,15 @@ def initiate_value_change(val): return menu def update_control_menu(self): + # TODO: Refactor this to be more uniform across sources if not self.has_ui: return - del self.menu[:] + self.update_menu() + + def update_menu(self): + # TODO: Refactor this to be more uniform across sources + super().update_menu() self.menu.append( ui.Info_Text(f"NDSI Source: {self._sensor_name} @ {self._host_name}") diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index d9967978b1..a3db64fba4 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -565,7 +565,6 @@ def online(self): def update_menu(self): super().update_menu() - del self.menu[:] self.menu.append(ui.Info_Text(f"Local USB Source: {self.name}")) ui_elements = [] From 48f05dd298d7a536aa600be0a21fd864851cdebb Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 20 Jan 2020 18:34:32 +0100 Subject: [PATCH 07/65] Add dummy source menu to Base_Source --- .../video_capture/base_backend.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index fcb52f3242..1a7f161b7f 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -13,7 +13,7 @@ from time import monotonic, sleep import numpy as np -from pyglui import cygl +from pyglui import cygl, ui import gl_utils from plugin import Plugin @@ -85,8 +85,31 @@ def init_ui(self): def deinit_ui(self): self.remove_menu() + def device_list(self): + return zip( + *[ + (None, "... select to activate ..."), + ("test key 1", "test value 1"), + ("test key 2", "test value 2"), + ] + ) + + def on_activate(self, arg): + print(arg) + def update_menu(self): del self.menu[:] + self.menu.append(ui.Info_Text("Select your video input source.")) + + self.menu.append( + ui.Selector( + "selected_source", + selection_getter=self.device_list, + getter=lambda: None, + setter=self.on_activate, + label="Activate Source:", + ) + ) def recent_events(self, events): """Returns None From bcc9ad0650775e8a7f3b91a9f665621e0a09a1d4 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 11:58:30 +0100 Subject: [PATCH 08/65] Add note about double base class with Base_Source --- pupil_src/shared_modules/video_capture/file_backend.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index 166cd87292..e5b2568e15 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -192,6 +192,8 @@ def get_frame_iterator(self): yield frame +# NOTE:Base_Source is included as base class for uniqueness:by_base_class to work +# correctly with other Source plugins. class File_Source(Playback_Source, Base_Source): """Simple file capture. From a97a470048da9d23d78e9619ace0cffa143cc129 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 11:58:53 +0100 Subject: [PATCH 09/65] Gitignore potential .venv --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e16d4b09be..b29aefeb60 100644 --- a/.gitignore +++ b/.gitignore @@ -55,4 +55,5 @@ deployment/pupil_v* *.pyd *.dll win_drv -.vs \ No newline at end of file +.vs +.venv From e6d6066de3df9bdf1be97777e24d0e9977a1fa98 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 14:00:13 +0100 Subject: [PATCH 10/65] TODO: pyglui Selector not working as expected --- pupil_src/shared_modules/video_capture/base_backend.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 1a7f161b7f..06f175dbf3 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -102,6 +102,10 @@ def update_menu(self): self.menu.append(ui.Info_Text("Select your video input source.")) self.menu.append( + # TODO: why can't we keep the selection? Even if the device/camera + # disconnects, we also don't change the source, so maybe just keep it as it + # is? + # TODO: selector does not jump back to first field! pyglui bug? ui.Selector( "selected_source", selection_getter=self.device_list, From db9652931eb9d17110e1a910ef91ed809d478ef2 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 14:21:25 +0100 Subject: [PATCH 11/65] Add simple switch for auto/manual mode --- .../video_capture/base_backend.py | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 06f175dbf3..b96ef4e15d 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -10,6 +10,7 @@ """ import logging +from enum import Enum, auto from time import monotonic, sleep import numpy as np @@ -37,6 +38,11 @@ class NoMoreVideoError(Exception): pass +class SourceMode(Enum): + AUTO = auto() + MANUAL = auto() + + class Base_Source(Plugin): """Abstract source class @@ -70,6 +76,8 @@ def pretty_class_name(self): def __init__(self, g_pool): super().__init__(g_pool) self.g_pool.capture = self + # TODO: serialize source mode + self.g_pool.source_mode = SourceMode.AUTO self._recent_frame = None self._intrinsics = None @@ -97,6 +105,18 @@ def device_list(self): def on_activate(self, arg): print(arg) + @property + def auto_mode(self) -> bool: + return self.g_pool.source_mode == SourceMode.AUTO + + @auto_mode.setter + def auto_mode(self, enable) -> None: + new_mode = SourceMode.AUTO if enable else SourceMode.MANUAL + if new_mode != self.g_pool.source_mode: + logger.debug(f"Setting source mode: {new_mode}") + self.g_pool.source_mode = new_mode + # TODO: broadcast + def update_menu(self): del self.menu[:] self.menu.append(ui.Info_Text("Select your video input source.")) @@ -115,6 +135,10 @@ def update_menu(self): ) ) + self.menu.append( + ui.Switch("auto_mode", self, label="Automatic Camera Selection") + ) + def recent_events(self, events): """Returns None From 55c06f22ec9c9fcbc402b2f1080ab5a654a50bc1 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 14:56:49 +0100 Subject: [PATCH 12/65] Broadcast source mode changes and react to those --- .../video_capture/base_backend.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index b96ef4e15d..980039cef5 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -10,7 +10,7 @@ """ import logging -from enum import Enum, auto +from enum import IntEnum, auto from time import monotonic, sleep import numpy as np @@ -38,7 +38,8 @@ class NoMoreVideoError(Exception): pass -class SourceMode(Enum): +class SourceMode(IntEnum): + # NOTE: IntEnum is serializable with msgpack AUTO = auto() MANUAL = auto() @@ -113,9 +114,18 @@ def auto_mode(self) -> bool: def auto_mode(self, enable) -> None: new_mode = SourceMode.AUTO if enable else SourceMode.MANUAL if new_mode != self.g_pool.source_mode: - logger.debug(f"Setting source mode: {new_mode}") + logger.debug(f"Setting source mode: {new_mode.name}") self.g_pool.source_mode = new_mode - # TODO: broadcast + self.notify_all({"subject": "backend.change_mode", "mode": new_mode}) + + def on_notify(self, notification): + subject = notification["subject"] + + if subject == "backend.change_mode": + mode = SourceMode(notification["mode"]) + if mode != self.g_pool.source_mode: + logger.debug(f"Setting source mode from network: {mode.name}") + self.g_pool.source_mode = mode def update_menu(self): del self.menu[:] From 7d514f5803b15e386e3438d2c8f46b134dc53ecd Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 14:57:42 +0100 Subject: [PATCH 13/65] Broadcast source mode from world when eyes start This way we ensure that freshly started eye processes always get the latest mode from world. --- pupil_src/shared_modules/video_capture/base_backend.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 980039cef5..f259d70fbd 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -127,6 +127,14 @@ def on_notify(self, notification): logger.debug(f"Setting source mode from network: {mode.name}") self.g_pool.source_mode = mode + elif subject == "eye_process.started": + # Make sure to broadcast current source mode once to newly started eyes so + # they are always in sync! + if self.g_pool.process == "world": + self.notify_all( + {"subject": "backend.change_mode", "mode": self.g_pool.source_mode} + ) + def update_menu(self): del self.menu[:] self.menu.append(ui.Info_Text("Select your video input source.")) From 1b40a3f46a4769e70ee5e76933c6c4b9d746143a Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 15:32:50 +0100 Subject: [PATCH 14/65] Fix incorrect polymorphism for init_dict and source classes --- pupil_src/shared_modules/video_capture/base_backend.py | 4 ++-- pupil_src/shared_modules/video_capture/hmd_streaming.py | 2 +- pupil_src/shared_modules/video_capture/ndsi_backend.py | 4 +++- pupil_src/shared_modules/video_capture/uvc_backend.py | 4 +++- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index f259d70fbd..0c7c59f6a9 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -74,7 +74,7 @@ class Base_Source(Plugin): def pretty_class_name(self): return "Video Source" - def __init__(self, g_pool): + def __init__(self, g_pool, *args, **kwargs): super().__init__(g_pool) self.g_pool.capture = self # TODO: serialize source mode @@ -370,7 +370,7 @@ def __init__(self, g_pool, timing="own", *args, **kwargs): most appropriate frame; does not wait on its own None: Simply returns next frame as fast as possible; used for detectors """ - super().__init__(g_pool) + super().__init__(g_pool, *args, **kwargs) assert timing in ( "external", "own", diff --git a/pupil_src/shared_modules/video_capture/hmd_streaming.py b/pupil_src/shared_modules/video_capture/hmd_streaming.py index 5c3fbb2d9f..96a806535e 100644 --- a/pupil_src/shared_modules/video_capture/hmd_streaming.py +++ b/pupil_src/shared_modules/video_capture/hmd_streaming.py @@ -169,4 +169,4 @@ def recent_events(self, events): pass def get_init_dict(self): - return {} + return super().get_init_dict() diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index 40168bbc40..e64fc13a20 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -47,8 +47,10 @@ def __init__( source_id=None, host_name=None, sensor_name=None, + *args, + **kwargs, ): - super().__init__(g_pool) + super().__init__(g_pool, *args, **kwargs) self.sensor = None self._source_id = source_id self._sensor_name = sensor_name diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index a3db64fba4..84e991eb7a 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -61,9 +61,11 @@ def __init__( uvc_controls={}, check_stripes=True, exposure_mode="manual", + *args, + **kwargs, ): - super().__init__(g_pool) + super().__init__(g_pool, *args, **kwargs) self.uvc_capture = None self._restart_in = 3 assert name or preferred_names or uid From 4a2d780275e112f5902096dd8275d559ad5fa964 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 15:33:29 +0100 Subject: [PATCH 15/65] Serialize source mode to settings --- .../shared_modules/video_capture/base_backend.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 0c7c59f6a9..0e15bdf9a8 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -10,6 +10,7 @@ """ import logging +import typing as T from enum import IntEnum, auto from time import monotonic, sleep @@ -74,14 +75,19 @@ class Base_Source(Plugin): def pretty_class_name(self): return "Video Source" - def __init__(self, g_pool, *args, **kwargs): + def __init__(self, g_pool, *, source_mode: T.Optional[SourceMode] = None, **kwargs): super().__init__(g_pool) self.g_pool.capture = self - # TODO: serialize source mode - self.g_pool.source_mode = SourceMode.AUTO self._recent_frame = None self._intrinsics = None + # Three relevant cases for initializing source_mode: + # - Plugin started at runtime: use existing source mode in g_pool + # - Fresh start without settings: initialize to auto + # - Start with settings: will be passed as parameter, use those + if not hasattr(self.g_pool, "source_mode"): + self.g_pool.source_mode = source_mode or SourceMode.AUTO + def add_menu(self): super().add_menu() self.menu_icon.order = 0.2 @@ -194,7 +200,7 @@ def name(self): raise NotImplementedError() def get_init_dict(self): - return {} + return {"source_mode": self.g_pool.source_mode} @property def frame_size(self): From d20334cee43ac39384353647f1bd42fb0b561956 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 16:14:26 +0100 Subject: [PATCH 16/65] Activate all source managers as system plugins in parallel --- pupil_src/launchables/eye.py | 3 +++ pupil_src/launchables/world.py | 3 +++ pupil_src/shared_modules/video_capture/__init__.py | 4 ++-- pupil_src/shared_modules/video_capture/base_backend.py | 5 ----- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index 3e639e631c..d607431281 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -226,6 +226,9 @@ def get_timestamp(): # TODO: extend with plugins default_capture_settings, ("UVC_Manager", {}), + ("NDSI_Manager", {}), + ("HMD_Streaming_Manager", {}), + ("File_Manager", {}), # Detector needs to be loaded first to set `g_pool.pupil_detector` (default_detector_cls.__name__, {}), ("PupilDetectorManager", {}), diff --git a/pupil_src/launchables/world.py b/pupil_src/launchables/world.py index 66aa0ae167..44c1495414 100644 --- a/pupil_src/launchables/world.py +++ b/pupil_src/launchables/world.py @@ -308,6 +308,9 @@ def get_timestamp(): ("UVC_Source", default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), + ("NDSI_Manager", {}), + ("HMD_Streaming_Manager", {}), + ("File_Manager", {}), ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), diff --git a/pupil_src/shared_modules/video_capture/__init__.py b/pupil_src/shared_modules/video_capture/__init__.py index 4c075bddc8..70bc1e1254 100644 --- a/pupil_src/shared_modules/video_capture/__init__.py +++ b/pupil_src/shared_modules/video_capture/__init__.py @@ -38,14 +38,14 @@ StreamError, ) from .file_backend import File_Manager, File_Source, FileSeekError -from .hmd_streaming import HMD_Streaming_Source +from .hmd_streaming import HMD_Streaming_Source, HMD_Streaming_Manager from .uvc_backend import UVC_Manager, UVC_Source logger = logging.getLogger(__name__) source_classes = [File_Source, UVC_Source, HMD_Streaming_Source] -manager_classes = [File_Manager, UVC_Manager] +manager_classes = [File_Manager, UVC_Manager, HMD_Streaming_Manager] try: from .ndsi_backend import NDSI_Source, NDSI_Manager diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 0e15bdf9a8..2a49a4c040 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -261,11 +261,6 @@ class Base_Manager(Plugin): gui_name (str): String used for manager selector labels """ - uniqueness = "by_base_class" - gui_name = "Base Manager" - icon_chr = chr(0xEC01) - icon_font = "pupil_icons" - def __init__(self, g_pool): super().__init__(g_pool) From 73da9318346a87b9541faa24581dfe2c5918f323 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 17:31:43 +0100 Subject: [PATCH 17/65] Register source managers in g_pool --- pupil_src/shared_modules/video_capture/base_backend.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 2a49a4c040..da15d665c9 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -264,6 +264,13 @@ class Base_Manager(Plugin): def __init__(self, g_pool): super().__init__(g_pool) + if not hasattr(g_pool, "source_managers"): + g_pool.source_managers = [] + + if self not in g_pool.source_managers: + g_pool.source_managers.append(self) + + # TODO: cleanup this from . import manager_classes self.manager_classes = {m.__name__: m for m in manager_classes} From a83e543ca7802400fb57cc6c9ef061c46d9592b9 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 18:04:42 +0100 Subject: [PATCH 18/65] Suppress pyre debug log floods --- pupil_src/shared_modules/video_capture/ndsi_backend.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index e64fc13a20..0210200880 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -27,6 +27,13 @@ from ndsi import __protocol_version__ except (ImportError, AssertionError): raise Exception("pyndsi version is to old. Please upgrade") from None + +# TODO: This is a quick hack to limit pyre log floods when running with --debug. Think +# about whether we need this? +for namespace, logger in logging.root.manager.loggerDict.items(): + if isinstance(logger, logging.Logger) and namespace.startswith("pyre"): + logger.setLevel(logging.WARNING) + logger = logging.getLogger(__name__) From 938e6f4144e5b7a6a45af3ab2e9ff1cfd00af864 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 18:06:32 +0100 Subject: [PATCH 19/65] Add SourceInfo skeleton for referencing sources across managers --- .../video_capture/base_backend.py | 45 ++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index da15d665c9..bbe49f594e 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -101,16 +101,23 @@ def deinit_ui(self): self.remove_menu() def device_list(self): - return zip( - *[ - (None, "... select to activate ..."), - ("test key 1", "test value 1"), - ("test key 2", "test value 2"), - ] - ) + entries = [(None, "... select to activate ...")] + + try: + for manager in self.g_pool.source_managers: + for info in manager.get_cameras(): + entries.append((info, info.name)) + except AttributeError: + # TODO: If no manager has been instantiated yet, g_pool.source_managers does + # not exist. Find a better way for this, probably ensure that the list + # exists? + pass - def on_activate(self, arg): - print(arg) + return zip(*entries) + + def activate_source(self, source_info): + print(source_info) + source_info.activate() @property def auto_mode(self) -> bool: @@ -154,7 +161,7 @@ def update_menu(self): "selected_source", selection_getter=self.device_list, getter=lambda: None, - setter=self.on_activate, + setter=self.activate_source, label="Activate Source:", ) ) @@ -367,6 +374,24 @@ def add_menu(self): # here is where you add all your menu entries. self.menu.label = "Backend Manager" + def get_devices(self) -> T.Sequence["Base_Manager.SourceInfo"]: + return [] + + def get_cameras(self) -> T.Sequence["Base_Manager.SourceInfo"]: + return [] + + class SourceInfo: + def __init__(self, name, manager, key): + self.name = name + self.manager = manager + self.key = key + + def activate(self): + self.manager.activate(self.key) + + def __str__(self): + return f"{self.name} - {self.manager.class_name}({self.key})" + class Playback_Source(Base_Source): def __init__(self, g_pool, timing="own", *args, **kwargs): From 7a92a1a8404672991b13c4745cbc4bbe6de55203 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 21 Jan 2020 18:07:24 +0100 Subject: [PATCH 20/65] Add SourceInfo proof-of-concept with UVC backend --- pupil_src/shared_modules/video_capture/uvc_backend.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index 84e991eb7a..62255dbebb 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -872,6 +872,13 @@ def dev_selection_list(): ) self.menu.extend(ui_elements) + def get_cameras(self): + self.devices.update() + return [ + self.SourceInfo(device["name"], self, device["uid"]) + for device in self.devices + ] + def activate(self, source_uid): if not source_uid: return From 57af88bfdde9ef02f7020fd186d34e8f4c1367d9 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 22 Jan 2020 10:57:06 +0100 Subject: [PATCH 21/65] Suppress pyre debug logs except pyre.zbeacon --- pupil_src/shared_modules/remote_recorder.py | 5 +++++ pupil_src/shared_modules/video_capture/ndsi_backend.py | 10 +++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pupil_src/shared_modules/remote_recorder.py b/pupil_src/shared_modules/remote_recorder.py index 80f3c10d11..20b4049cca 100644 --- a/pupil_src/shared_modules/remote_recorder.py +++ b/pupil_src/shared_modules/remote_recorder.py @@ -18,6 +18,11 @@ logger = logging.getLogger(__name__) +# Suppress pyre debug logs (except beacon) +logger.debug("Suppressing pyre debug logs (except zbeacon)") +logging.getLogger("pyre").setLevel(logging.WARNING) +logging.getLogger("pyre.zbeacon").setLevel(logging.WARNING) + class Remote_Recording_State: __slots__ = ["sensor"] diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index 0210200880..786c02851d 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -28,14 +28,14 @@ except (ImportError, AssertionError): raise Exception("pyndsi version is to old. Please upgrade") from None -# TODO: This is a quick hack to limit pyre log floods when running with --debug. Think -# about whether we need this? -for namespace, logger in logging.root.manager.loggerDict.items(): - if isinstance(logger, logging.Logger) and namespace.startswith("pyre"): - logger.setLevel(logging.WARNING) logger = logging.getLogger(__name__) +# Suppress pyre debug logs (except beacon) +logger.debug("Suppressing pyre debug logs (except zbeacon)") +logging.getLogger("pyre").setLevel(logging.WARNING) +logging.getLogger("pyre.zbeacon").setLevel(logging.WARNING) + class NDSI_Source(Base_Source): """Pupil Mobile video source From efa724b3598c7339cce64f74ff47becb90b98701 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 22 Jan 2020 15:26:35 +0100 Subject: [PATCH 22/65] Implement dummy auto mode for UVC backend --- .../shared_modules/video_capture/uvc_backend.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index 62255dbebb..3190a01e7e 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -31,7 +31,7 @@ # logging logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) +logger.setLevel(logging.DEBUG) class TJSAMP(enum.IntEnum): @@ -872,14 +872,23 @@ def dev_selection_list(): ) self.menu.extend(ui_elements) + def get_devices(self): + return [self.SourceInfo("Local USB", self, "usb")] + def get_cameras(self): self.devices.update() return [ - self.SourceInfo(device["name"], self, device["uid"]) + self.SourceInfo(device["name"], self, f"cam.{device['uid']}") for device in self.devices ] def activate(self, source_uid): + if source_uid == "usb": + logger.debug("AUTO ACTIVATE USB") + return + + source_uid = source_uid[4:] + if not source_uid: return From f014bedc73253b6de1b67d2bd69d94a5a7f7c142 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 22 Jan 2020 15:27:12 +0100 Subject: [PATCH 23/65] Show only devices or cameras depending on source mode --- pupil_src/shared_modules/video_capture/base_backend.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index bbe49f594e..6b75648ec9 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -105,7 +105,12 @@ def device_list(self): try: for manager in self.g_pool.source_managers: - for info in manager.get_cameras(): + if self.auto_mode: + sources = manager.get_devices() + else: + sources = manager.get_cameras() + + for info in sources: entries.append((info, info.name)) except AttributeError: # TODO: If no manager has been instantiated yet, g_pool.source_managers does From 743920523253caf5120c18170ea7c68ad8fcbf60 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 22 Jan 2020 15:27:35 +0100 Subject: [PATCH 24/65] Catch case of not selecting a source to activate --- pupil_src/shared_modules/video_capture/base_backend.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 6b75648ec9..0cd9977135 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -122,7 +122,8 @@ def device_list(self): def activate_source(self, source_info): print(source_info) - source_info.activate() + if source_info is not None: + source_info.activate() @property def auto_mode(self) -> bool: From 134a66f67ee6f7b37e9bfb0c898cb5511f6abe1f Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 22 Jan 2020 15:42:00 +0100 Subject: [PATCH 25/65] Remove pyglui todo which gets resolved with pyglui 1.27 --- pupil_src/shared_modules/video_capture/base_backend.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 0cd9977135..2df92d7eae 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -159,10 +159,6 @@ def update_menu(self): self.menu.append(ui.Info_Text("Select your video input source.")) self.menu.append( - # TODO: why can't we keep the selection? Even if the device/camera - # disconnects, we also don't change the source, so maybe just keep it as it - # is? - # TODO: selector does not jump back to first field! pyglui bug? ui.Selector( "selected_source", selection_getter=self.device_list, From 9b7c0b32c762be0acdb3c9c83f5aa118a9de2fca Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 22 Jan 2020 15:48:30 +0100 Subject: [PATCH 26/65] Always change source mode via notification only --- pupil_src/shared_modules/video_capture/base_backend.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 2df92d7eae..79967c7c4e 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -134,7 +134,6 @@ def auto_mode(self, enable) -> None: new_mode = SourceMode.AUTO if enable else SourceMode.MANUAL if new_mode != self.g_pool.source_mode: logger.debug(f"Setting source mode: {new_mode.name}") - self.g_pool.source_mode = new_mode self.notify_all({"subject": "backend.change_mode", "mode": new_mode}) def on_notify(self, notification): @@ -143,7 +142,6 @@ def on_notify(self, notification): if subject == "backend.change_mode": mode = SourceMode(notification["mode"]) if mode != self.g_pool.source_mode: - logger.debug(f"Setting source mode from network: {mode.name}") self.g_pool.source_mode = mode elif subject == "eye_process.started": From b2d1432bc80aae346b3dbe987005df692fb00bff Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 22 Jan 2020 15:48:48 +0100 Subject: [PATCH 27/65] Redraw source list on source mode change --- pupil_src/shared_modules/video_capture/base_backend.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 79967c7c4e..08d809f97b 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -143,6 +143,8 @@ def on_notify(self, notification): mode = SourceMode(notification["mode"]) if mode != self.g_pool.source_mode: self.g_pool.source_mode = mode + # redraw menu to close potentially open (and now incorrect) dropdown + self.update_menu() elif subject == "eye_process.started": # Make sure to broadcast current source mode once to newly started eyes so From 7fea7ca720bcb7e1cc7f59b08e112d053aabd6c8 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 22 Jan 2020 16:02:48 +0100 Subject: [PATCH 28/65] Rename SourceInfo.name to .label --- pupil_src/shared_modules/video_capture/base_backend.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 08d809f97b..f4b2b10a1c 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -111,7 +111,7 @@ def device_list(self): sources = manager.get_cameras() for info in sources: - entries.append((info, info.name)) + entries.append((info, info.label)) except AttributeError: # TODO: If no manager has been instantiated yet, g_pool.source_managers does # not exist. Find a better way for this, probably ensure that the list @@ -383,8 +383,8 @@ def get_cameras(self) -> T.Sequence["Base_Manager.SourceInfo"]: return [] class SourceInfo: - def __init__(self, name, manager, key): - self.name = name + def __init__(self, label, manager, key): + self.label = label self.manager = manager self.key = key @@ -392,7 +392,7 @@ def activate(self): self.manager.activate(self.key) def __str__(self): - return f"{self.name} - {self.manager.class_name}({self.key})" + return f"{self.label} - {self.manager.class_name}({self.key})" class Playback_Source(Base_Source): From d50528c4ce41e53dc6c20b0bca21ae1f70f235ad Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 22 Jan 2020 16:03:10 +0100 Subject: [PATCH 29/65] Include @ Local USB in UVC cam labels --- pupil_src/shared_modules/video_capture/uvc_backend.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index 3190a01e7e..917396e8f2 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -873,12 +873,16 @@ def dev_selection_list(): self.menu.extend(ui_elements) def get_devices(self): - return [self.SourceInfo("Local USB", self, "usb")] + return [self.SourceInfo(label="Local USB", manager=self, key="usb")] def get_cameras(self): self.devices.update() return [ - self.SourceInfo(device["name"], self, f"cam.{device['uid']}") + self.SourceInfo( + label=f"{device['name']} @ Local USB", + manager=self, + key=f"cam.{device['uid']}", + ) for device in self.devices ] From cafba1035aac09162e22bf9faf04ef19e733d522 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 22 Jan 2020 17:49:03 +0100 Subject: [PATCH 30/65] Implement get devices/cameras skeleton for ndsi backend --- .../video_capture/ndsi_backend.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index 786c02851d..6bc09689c0 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -425,6 +425,29 @@ def view_host(self, host_uuid): self.selected_host = host_uuid self.re_build_ndsi_menu() + def get_devices(self): + # store hosts in dict to remove duplicates from multiple sensors + active_hosts = { + s["host_uuid"]: s["host_name"] + for s in self.network.sensors.values() + if s["sensor_type"] == "video" + } + return [ + self.SourceInfo(label=host_name, manager=self, key=f"host.{host_uuid}") + for host_uuid, host_name in active_hosts.items() + ] + + def get_cameras(self): + return [ + self.SourceInfo( + label=f"{s['sensor_name']} @ PM {s['host_name']}", + manager=self, + key=f"sensor.{s['sensor_uuid']}", + ) + for s in self.network.sensors.values() + if s["sensor_type"] == "video" + ] + def host_selection_list(self): devices = { s["host_uuid"]: s["host_name"] # removes duplicates @@ -492,6 +515,12 @@ def re_build_ndsi_menu(self): self.menu.extend(ui_elements) def activate(self, source_uid): + source_type, uid = source_uid.split(".", maxsplit=1) + if source_type == "host": + logger.debug("AUTO ACTIVATE HOST") + return + source_uid = uid + if not source_uid: return settings = { From bdbed50431a0f4b510200de710f5b6823567b59a Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 22 Jan 2020 17:49:25 +0100 Subject: [PATCH 31/65] Fix ndsi backend not correctly forwarding notifications --- pupil_src/shared_modules/video_capture/ndsi_backend.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index 6bc09689c0..c41f2f50c9 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -199,6 +199,7 @@ def on_notification(self, sensor, event): # local notifications def on_notify(self, notification): + super().on_notify(notification) subject = notification["subject"] if subject.startswith("remote_recording.") and self.online: if "should_start" in subject and self.online: From a33b45ca6d069e231564f4e6099d6aba5abf7c6f Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Thu, 23 Jan 2020 16:22:19 +0100 Subject: [PATCH 32/65] Make SourceInfo not an inner class of Base_Manager --- .../video_capture/base_backend.py | 26 +++++++++++-------- .../video_capture/ndsi_backend.py | 6 ++--- .../video_capture/uvc_backend.py | 8 +++--- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index f4b2b10a1c..0ef5f80e8a 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -376,23 +376,27 @@ def add_menu(self): # here is where you add all your menu entries. self.menu.label = "Backend Manager" - def get_devices(self) -> T.Sequence["Base_Manager.SourceInfo"]: + def get_devices(self) -> T.Sequence["SourceInfo"]: return [] - def get_cameras(self) -> T.Sequence["Base_Manager.SourceInfo"]: + def get_cameras(self) -> T.Sequence["SourceInfo"]: return [] - class SourceInfo: - def __init__(self, label, manager, key): - self.label = label - self.manager = manager - self.key = key + def activate(self, key: T.Any) -> None: + pass + + +class SourceInfo: + def __init__(self, label: str, manager: Base_Manager, key: T.Any): + self.label = label + self.manager = manager + self.key = key - def activate(self): - self.manager.activate(self.key) + def activate(self) -> None: + self.manager.activate(self.key) - def __str__(self): - return f"{self.label} - {self.manager.class_name}({self.key})" + def __str__(self) -> str: + return f"{self.label} - {self.manager.class_name}({self.key})" class Playback_Source(Base_Source): diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index c41f2f50c9..fa84d1493a 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -18,7 +18,7 @@ from camera_models import load_intrinsics -from .base_backend import Base_Manager, Base_Source +from .base_backend import Base_Manager, Base_Source, SourceInfo try: from ndsi import __version__ @@ -434,13 +434,13 @@ def get_devices(self): if s["sensor_type"] == "video" } return [ - self.SourceInfo(label=host_name, manager=self, key=f"host.{host_uuid}") + SourceInfo(label=host_name, manager=self, key=f"host.{host_uuid}") for host_uuid, host_name in active_hosts.items() ] def get_cameras(self): return [ - self.SourceInfo( + SourceInfo( label=f"{s['sensor_name']} @ PM {s['host_name']}", manager=self, key=f"sensor.{s['sensor_uuid']}", diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index 917396e8f2..aa8c38e00b 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -16,14 +16,14 @@ import time import numpy as np +import uvc from pyglui import cygl, ui import gl_utils -import uvc from camera_models import load_intrinsics from version_utils import VersionFormat -from .base_backend import Base_Manager, Base_Source, InitialisationError +from .base_backend import Base_Manager, Base_Source, InitialisationError, SourceInfo from .utils import Check_Frame_Stripes, Exposure_Time # check versions for our own depedencies as they are fast-changing @@ -873,12 +873,12 @@ def dev_selection_list(): self.menu.extend(ui_elements) def get_devices(self): - return [self.SourceInfo(label="Local USB", manager=self, key="usb")] + return [SourceInfo(label="Local USB", manager=self, key="usb")] def get_cameras(self): self.devices.update() return [ - self.SourceInfo( + SourceInfo( label=f"{device['name']} @ Local USB", manager=self, key=f"cam.{device['uid']}", From 450ef66c8d01520becae634abef1ebc521cc27eb Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Thu, 23 Jan 2020 16:25:11 +0100 Subject: [PATCH 33/65] Remove base source frame size and rate setters Since they will not be overwritten in most child sources, this means changing a read/write property of the base interface to a read/only property in the inherited classes. Mypy legitimately complains here. --- pupil_src/shared_modules/video_capture/base_backend.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 0ef5f80e8a..579fcbb15d 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -219,10 +219,6 @@ def frame_size(self): """ raise NotImplementedError() - @frame_size.setter - def frame_size(self, new_size): - raise NotImplementedError() - @property def frame_rate(self): """ @@ -231,10 +227,6 @@ def frame_rate(self): """ raise NotImplementedError() - @frame_rate.setter - def frame_rate(self, new_rate): - pass - @property def jpeg_support(self): """ From 5fb670b3d3b011fb3ac417420e14f040e2013c40 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Thu, 23 Jan 2020 17:53:20 +0100 Subject: [PATCH 34/65] Only show "Local USB" when uvc detects any device --- pupil_src/shared_modules/video_capture/uvc_backend.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index aa8c38e00b..20df27cb4a 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -873,7 +873,11 @@ def dev_selection_list(): self.menu.extend(ui_elements) def get_devices(self): - return [SourceInfo(label="Local USB", manager=self, key="usb")] + self.devices.update() + if len(self.devices) == 0: + return [] + else: + return [SourceInfo(label="Local USB", manager=self, key="usb")] def get_cameras(self): self.devices.update() From 1fac4f573ada94ec29ddd8810b855977d889bdaf Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Thu, 23 Jan 2020 18:22:47 +0100 Subject: [PATCH 35/65] First draft of uvc auto activation --- .../video_capture/uvc_backend.py | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index 20df27cb4a..31d5a23895 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -890,13 +890,15 @@ def get_cameras(self): for device in self.devices ] - def activate(self, source_uid): - if source_uid == "usb": - logger.debug("AUTO ACTIVATE USB") + def activate(self, key): + if key == "usb": + self.notify_all({"subject": "backend.uvc.auto_activate_source"}) return - source_uid = source_uid[4:] + source_uid = key[4:] + self.activate_source(source_uid) + def activate_source(self, source_uid): if not source_uid: return @@ -927,22 +929,30 @@ def activate(self, source_uid): } ) + def on_notify(self, notification): + super().on_notify(notification) + + if notification["subject"] == "backend.uvc.auto_activate_source": + self.auto_activate_source() + def auto_activate_source(self): + logger.debug("Auto activating USB source.") + self.devices.update() if not self.devices or len(self.devices) == 0: logger.warning("No default device is available.") return - cam_ids = self.cam_selection_lut[self.g_pool.process] + name_patterns = self.cam_selection_lut[self.g_pool.process] + matching_cams_ids = [ + device["uid"] + for device in self.devices + if any(pattern in device["name"] for pattern in name_patterns) + ] - for cam_id in cam_ids: - try: - source_id = next(d["uid"] for d in self.devices if cam_id in d["name"]) - self.activate(source_id) - break - except StopIteration: - source_id = None + if matching_cams_ids: + self.activate_source(matching_cams_ids[0]) else: - logger.warning("The default device is not found.") + logger.warning("Could not find default device.") def deinit_ui(self): self.remove_menu() From b13f733d3250440ae7772b78a4fe5fbeaa93745b Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 09:51:46 +0100 Subject: [PATCH 36/65] Auto-activate cams from same USB-bus first in UVC backend --- .../shared_modules/video_capture/uvc_backend.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index 31d5a23895..f53d9370a9 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -943,16 +943,23 @@ def auto_activate_source(self): return name_patterns = self.cam_selection_lut[self.g_pool.process] - matching_cams_ids = [ - device["uid"] + matching_cams = [ + device for device in self.devices if any(pattern in device["name"] for pattern in name_patterns) ] - if matching_cams_ids: - self.activate_source(matching_cams_ids[0]) - else: + if not matching_cams: logger.warning("Could not find default device.") + return + + # Sorting cams by bus_number increases chances of selecting only cams from the + # same headset when having multiple headsets connected. Note that two headsets + # might have the same bus_number when they share an internal USB bus. + cam = min( + matching_cams, key=lambda device: device.get("bus_number", float("inf")) + ) + self.activate_source(cam["uid"]) def deinit_ui(self): self.remove_menu() From 5a05da660d07eff5a4740109c8992aacfeb65713 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 11:17:30 +0100 Subject: [PATCH 37/65] Implement auto-activation in NDSI backend - Network does not need to be shared anymore since NDSI_Manager runs always. - Activation now works the same way it does in UVC backend. - Cam matching logic has been made clearer (for-try-StopIteration part) --- .../video_capture/ndsi_backend.py | 77 ++++++++++++------- 1 file changed, 49 insertions(+), 28 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index fa84d1493a..e420f6589a 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -50,7 +50,6 @@ def __init__( g_pool, frame_size, frame_rate, - network=None, source_id=None, host_name=None, sensor_name=None, @@ -71,6 +70,12 @@ def __init__( self._initial_refresh = True self.last_update = self.g_pool.get_timestamp() + manager = next((p for p in g_pool.plugins if isinstance(p, NDSI_Manager)), None) + if not manager: + logger.error("Error connecting to Pupil Mobile: NDSI Manager not found!") + return + + network = manager.network if not network: logger.debug( "No network reference provided. Capture is started " @@ -515,13 +520,16 @@ def re_build_ndsi_menu(self): self.menu.extend(ui_elements) - def activate(self, source_uid): - source_type, uid = source_uid.split(".", maxsplit=1) + def activate(self, key): + source_type, uid = key.split(".", maxsplit=1) if source_type == "host": - logger.debug("AUTO ACTIVATE HOST") - return - source_uid = uid + self.notify_all( + {"subject": "backend.ndsi.auto_activate_source", "host_uid": uid} + ) + elif source_type == "sensor": + self.activate_source(source_uid=uid) + def activate_source(self, source_uid): if not source_uid: return settings = { @@ -529,7 +537,19 @@ def activate(self, source_uid): "frame_rate": self.g_pool.capture.frame_rate, "source_id": source_uid, } - self.activate_source(settings) + if self.g_pool.process == "world": + self.notify_all( + {"subject": "start_plugin", "name": "NDSI_Source", "args": settings} + ) + else: + self.notify_all( + { + "subject": "start_eye_plugin", + "target": self.g_pool.process, + "name": "NDSI_Source", + "args": settings, + } + ) def auto_select_manager(self): super().auto_select_manager() @@ -541,28 +561,30 @@ def auto_select_manager(self): } ) - def auto_activate_source(self): - if not self.selected_host: - return + def auto_activate_source(self, host_uid): + host_sensors = [ + sensor + for sensor in self.network.sensors.values() + if (sensor["sensor_type"] == "video" and sensor["host_uuid"] == host_uid) + ] - src_sel, src_sel_labels = self.source_selection_list() - if len(src_sel) < 2: # "Select to Activate" is always presenet as first element - logger.warning("No device is available on the remote host.") + if not host_sensors: + logger.warning("No devices available on the remote host.") return - cam_ids = self.cam_selection_lut[self.g_pool.process] + name_patterns = self.cam_selection_lut[self.g_pool.process] + matching_cams = [ + sensor + for sensor in host_sensors + if any(pattern in sensor["sensor_name"] for pattern in name_patterns) + ] - for cam_id in cam_ids: - try: - source_id = next( - src_sel[i] for i, lab in enumerate(src_sel_labels) if cam_id in lab - ) - self.activate(source_id) - break - except StopIteration: - source_id = None - else: + if not matching_cams: logger.warning("The default device was not found on the remote host.") + return + + cam = matching_cams[0] + self.activate_source(cam["sensor_uuid"]) def poll_events(self): while self.network.has_events: @@ -611,10 +633,6 @@ def on_event(self, caller, event): self.re_build_ndsi_menu() - def activate_source(self, settings={}): - settings["network"] = self.network - self.g_pool.plugins.add(NDSI_Source, args=settings) - def recover(self): self.g_pool.capture.recover(self.network) @@ -642,6 +660,9 @@ def on_notify(self, n): if n["subject"].startswith("backend.ndsi_do_select_host"): self.select_host(n["target_host"]) + if n["subject"] == "backend.ndsi.auto_activate_source": + self.auto_activate_source(n["host_uid"]) + def select_host(self, selected_host): host_sel, _ = self.host_selection_list() if selected_host in host_sel: From 5f7f140846d194f010855a4fabc972691eae93af Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 11:32:14 +0100 Subject: [PATCH 38/65] Start source managers before sources --- pupil_src/shared_modules/video_capture/base_backend.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 579fcbb15d..369ed53b9a 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -262,6 +262,8 @@ class Base_Manager(Plugin): gui_name (str): String used for manager selector labels """ + order = -1 + def __init__(self, g_pool): super().__init__(g_pool) From 57739d0c27b6059cd4fabb2f89b150a7b2e2c026 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 11:49:10 +0100 Subject: [PATCH 39/65] Better dropdown UI --- pupil_src/shared_modules/video_capture/base_backend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 369ed53b9a..45a018d1d0 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -101,7 +101,7 @@ def deinit_ui(self): self.remove_menu() def device_list(self): - entries = [(None, "... select to activate ...")] + entries = [(None, "Activate Source")] try: for manager in self.g_pool.source_managers: @@ -164,7 +164,7 @@ def update_menu(self): selection_getter=self.device_list, getter=lambda: None, setter=self.activate_source, - label="Activate Source:", + label=" ", # TODO: Hide label completely ) ) From a47eebd755733244fea0daa751db09f7dca87981 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 12:02:50 +0100 Subject: [PATCH 40/65] Re-enable debug logs in File backend --- pupil_src/shared_modules/video_capture/file_backend.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index e5b2568e15..e6cec19d35 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -30,7 +30,6 @@ from .utils import VideoSet logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) av.logging.set_level(av.logging.ERROR) logging.getLogger("libav").setLevel(logging.ERROR) From f5fdb6f86fafda4f5ee523de44333f9b381872f4 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 14:35:18 +0100 Subject: [PATCH 41/65] Unify/simplify source UI - Base UI is different depending on mode - Child classes return list of UI elements that will be grouped under single growing menu --- .../video_capture/base_backend.py | 52 +++++++++++++++---- .../video_capture/file_backend.py | 24 ++++----- .../video_capture/hmd_streaming.py | 3 -- .../video_capture/ndsi_backend.py | 34 ++++++------ .../video_capture/uvc_backend.py | 14 +++-- 5 files changed, 75 insertions(+), 52 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 45a018d1d0..3cce83680b 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -101,14 +101,15 @@ def deinit_ui(self): self.remove_menu() def device_list(self): - entries = [(None, "Activate Source")] + label = "Activate Camera" if self.manual_mode else "Activate Device" + entries = [(None, label)] try: for manager in self.g_pool.source_managers: - if self.auto_mode: - sources = manager.get_devices() - else: + if self.manual_mode: sources = manager.get_cameras() + else: + sources = manager.get_devices() for info in sources: entries.append((info, info.label)) @@ -126,12 +127,12 @@ def activate_source(self, source_info): source_info.activate() @property - def auto_mode(self) -> bool: - return self.g_pool.source_mode == SourceMode.AUTO + def manual_mode(self) -> bool: + return self.g_pool.source_mode == SourceMode.MANUAL - @auto_mode.setter - def auto_mode(self, enable) -> None: - new_mode = SourceMode.AUTO if enable else SourceMode.MANUAL + @manual_mode.setter + def manual_mode(self, enable) -> None: + new_mode = SourceMode.MANUAL if enable else SourceMode.AUTO if new_mode != self.g_pool.source_mode: logger.debug(f"Setting source mode: {new_mode.name}") self.notify_all({"subject": "backend.change_mode", "mode": new_mode}) @@ -156,7 +157,18 @@ def on_notify(self, notification): def update_menu(self): del self.menu[:] - self.menu.append(ui.Info_Text("Select your video input source.")) + + if self.manual_mode: + self.menu.append( + ui.Info_Text("Select a camera to use as input for this window.") + ) + else: + self.menu.append( + ui.Info_Text( + "Select a device to use as video input." + " The best matching cameras will be automatically selected." + ) + ) self.menu.append( ui.Selector( @@ -168,10 +180,28 @@ def update_menu(self): ) ) + if not self.manual_mode: + self.menu.append( + ui.Info_Text( + "Enable manual camera selection to choose a specific camera" + " as input for every window." + ) + ) + self.menu.append( - ui.Switch("auto_mode", self, label="Automatic Camera Selection") + ui.Switch("manual_mode", self, label="Enable Manual Camera Selection") ) + source_settings = self.settings_ui_elements() + if source_settings: + settings_menu = ui.Growing_Menu(f"Settings") + settings_menu.collapsed = True + settings_menu.extend(source_settings) + self.menu.append(settings_menu) + + def settings_ui_elements(self) -> T.List[ui.UI_element]: + return [] + def recent_events(self, events): """Returns None diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index e6cec19d35..8acc2f44cd 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -512,19 +512,12 @@ def on_notify(self, notification): ): self.play = False - def update_menu(self): - super().update_menu() - self.menu.append( + def settings_ui_elements(self): + ui_elements = [] + ui_elements.append( ui.Info_Text(f"File Source: {os.path.split(self.source_path)[-1]}") ) - self.menu.append( - ui.Info_Text( - "The file source plugin loads and " - + "displays video from a given file." - ) - ) - if self.g_pool.app == "capture": def toggle_looping(val): @@ -532,13 +525,13 @@ def toggle_looping(val): if val: self.play = True - self.menu.append(ui.Switch("loop", self, setter=toggle_looping)) + ui_elements.append(ui.Switch("loop", self, setter=toggle_looping)) - self.menu.append( + ui_elements.append( ui.Text_Input("source_path", self, label="Full path", setter=lambda x: None) ) - self.menu.append( + ui_elements.append( ui.Text_Input( "frame_size", label="Frame size", @@ -547,7 +540,7 @@ def toggle_looping(val): ) ) - self.menu.append( + ui_elements.append( ui.Text_Input( "frame_rate", label="Frame rate", @@ -556,7 +549,7 @@ def toggle_looping(val): ) ) - self.menu.append( + ui_elements.append( ui.Text_Input( "frame_num", label="Number of frames", @@ -564,6 +557,7 @@ def toggle_looping(val): getter=lambda: self.get_frame_count(), ) ) + return ui_elements def deinit_ui(self): self.remove_menu() diff --git a/pupil_src/shared_modules/video_capture/hmd_streaming.py b/pupil_src/shared_modules/video_capture/hmd_streaming.py index 96a806535e..fac675f692 100644 --- a/pupil_src/shared_modules/video_capture/hmd_streaming.py +++ b/pupil_src/shared_modules/video_capture/hmd_streaming.py @@ -52,9 +52,6 @@ def __init__(self, g_pool, *args, **kwargs): topics=("hmd_streaming.world",), ) - def update_menu(self): - self.menu.append(ui.Info_Text(f"HMD Streaming")) - def cleanup(self): self.frame_sub = None diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index e420f6589a..54eb969345 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -336,36 +336,32 @@ def initiate_value_change(val): import traceback as tb tb.print_exc() - if len(menu) == 0: - menu.append(ui.Info_Text("No {} settings found".format(menu.label))) return menu def update_control_menu(self): # TODO: Refactor this to be more uniform across sources if not self.has_ui: return - self.update_menu() - def update_menu(self): - # TODO: Refactor this to be more uniform across sources - super().update_menu() + def settings_ui_elements(self): - self.menu.append( - ui.Info_Text(f"NDSI Source: {self._sensor_name} @ {self._host_name}") + ui_elements = [] + ui_elements.append( + ui.Info_Text(f"Camera: {self._sensor_name} @ {self._host_name}") ) - self.uvc_menu = ui.Growing_Menu("UVC Controls") + uvc_menu = ui.Growing_Menu("UVC Controls") self.control_id_ui_mapping = {} if not self.sensor: - self.menu.append( + ui_elements.append( ui.Info_Text( ("Sensor %s @ %s not available. " + "Running in ghost mode.") % (self._sensor_name, self._host_name) ) ) - return + return ui_elements uvc_controls = [] other_controls = [] @@ -375,14 +371,22 @@ def update_menu(self): else: other_controls.append(entry) - self.add_controls_to_menu(self.menu, other_controls) - self.add_controls_to_menu(self.uvc_menu, uvc_controls) - self.menu.append(self.uvc_menu) + if other_controls: + self.add_controls_to_menu(ui_elements, other_controls) - self.menu.append( + if uvc_controls: + self.add_controls_to_menu(uvc_menu, uvc_controls) + else: + uvc_menu.append(ui.Info_Text("No UVC settings found.")) + + ui_elements.append(uvc_menu) + + ui_elements.append( ui.Button("Reset to default values", self.sensor.reset_all_control_values) ) + return ui_elements + def cleanup(self): if self.online: self.sensor.unlink() diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index f53d9370a9..8608c50dc0 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -565,12 +565,11 @@ def jpeg_support(self): def online(self): return bool(self.uvc_capture) - def update_menu(self): - super().update_menu() - self.menu.append(ui.Info_Text(f"Local USB Source: {self.name}")) - + def settings_ui_elements(self): ui_elements = [] + ui_elements.append(ui.Info_Text(f"Camera: {self.name} @ Local USB")) + # lets define some helper functions: def gui_load_defaults(): for c in self.uvc_capture.controls: @@ -592,10 +591,8 @@ def set_frame_rate(new_rate): if self.uvc_capture is None: ui_elements.append(ui.Info_Text("Capture initialization failed.")) - self.menu.extend(ui_elements) - return + return ui_elements - ui_elements.append(ui.Info_Text("{} Controls".format(self.name))) sensor_control = ui.Growing_Menu(label="Sensor Settings") sensor_control.append( ui.Info_Text("Do not change these during calibration or recording!") @@ -783,7 +780,8 @@ def set_check_stripes(enable_stripe_checks): label="Check Stripes", ) ) - self.menu.extend(ui_elements) + + return ui_elements def cleanup(self): self.devices.cleanup() From 0842cfb7bd97d0dfdce3fd14ebd5dfcc2a926ce2 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 15:02:38 +0100 Subject: [PATCH 42/65] Fix initialization error by sorting plugins before adding --- pupil_src/shared_modules/plugin.py | 3 +++ .../video_capture/base_backend.py | 22 +++++++------------ 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/pupil_src/shared_modules/plugin.py b/pupil_src/shared_modules/plugin.py index 928b79d718..1d6193b795 100644 --- a/pupil_src/shared_modules/plugin.py +++ b/pupil_src/shared_modules/plugin.py @@ -322,6 +322,9 @@ def __init__(self, g_pool, plugin_initializers): expanded_initializers.append((plugin_by_name[name], name, args)) except KeyError: logger.debug(f"Plugin {name} failed to load, not available for import.") + + expanded_initializers.sort(key=lambda data: data[0].order) + # only add plugins that won't be replaced by newer plugins for i, (plugin, name, args) in enumerate(expanded_initializers): for new_plugin, new_name, _ in expanded_initializers[i + 1 :]: diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 3cce83680b..909aedf977 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -104,20 +104,14 @@ def device_list(self): label = "Activate Camera" if self.manual_mode else "Activate Device" entries = [(None, label)] - try: - for manager in self.g_pool.source_managers: - if self.manual_mode: - sources = manager.get_cameras() - else: - sources = manager.get_devices() - - for info in sources: - entries.append((info, info.label)) - except AttributeError: - # TODO: If no manager has been instantiated yet, g_pool.source_managers does - # not exist. Find a better way for this, probably ensure that the list - # exists? - pass + for manager in self.g_pool.source_managers: + if self.manual_mode: + sources = manager.get_cameras() + else: + sources = manager.get_devices() + + for info in sources: + entries.append((info, info.label)) return zip(*entries) From 5b50663ccb4722966d86bdd52ad1e5f30aa17d91 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 16:04:23 +0100 Subject: [PATCH 43/65] Cleanup source managers - Remove all UI - Remove old auto source and host selection - Cleanup unused stub methods - Remove HMD manager completely --- .../shared_modules/video_capture/__init__.py | 4 +- .../video_capture/base_backend.py | 101 +------------ .../video_capture/file_backend.py | 83 +---------- .../video_capture/hmd_streaming.py | 42 +----- .../video_capture/ndsi_backend.py | 135 +----------------- .../video_capture/uvc_backend.py | 49 +------ 6 files changed, 14 insertions(+), 400 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/__init__.py b/pupil_src/shared_modules/video_capture/__init__.py index 70bc1e1254..4c075bddc8 100644 --- a/pupil_src/shared_modules/video_capture/__init__.py +++ b/pupil_src/shared_modules/video_capture/__init__.py @@ -38,14 +38,14 @@ StreamError, ) from .file_backend import File_Manager, File_Source, FileSeekError -from .hmd_streaming import HMD_Streaming_Source, HMD_Streaming_Manager +from .hmd_streaming import HMD_Streaming_Source from .uvc_backend import UVC_Manager, UVC_Source logger = logging.getLogger(__name__) source_classes = [File_Source, UVC_Source, HMD_Streaming_Source] -manager_classes = [File_Manager, UVC_Manager, HMD_Streaming_Manager] +manager_classes = [File_Manager, UVC_Manager] try: from .ndsi_backend import NDSI_Source, NDSI_Manager diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 909aedf977..1a9aba7a13 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -281,9 +281,6 @@ class Base_Manager(Plugin): Managers are plugins that enumerate and load accessible sources from different backends, e.g. locally USB-connected cameras. - - Attributes: - gui_name (str): String used for manager selector labels """ order = -1 @@ -291,109 +288,13 @@ class Base_Manager(Plugin): def __init__(self, g_pool): super().__init__(g_pool) + # register all instances in g_pool.source_managers list if not hasattr(g_pool, "source_managers"): g_pool.source_managers = [] if self not in g_pool.source_managers: g_pool.source_managers.append(self) - # TODO: cleanup this - from . import manager_classes - - self.manager_classes = {m.__name__: m for m in manager_classes} - - def on_notify(self, notification): - """ - Reacts to notification: - ``backend.auto_select_manager``: Changes the current Manager to one that's emitted - ``backend.auto_activate_source``: Activates the current source via self.auto_activate_source() - - Emmits notifications (indirectly): - ``start_plugin``: For world thread - ``start_eye_plugin``: For eye thread - ``backend.auto_activate_source`` - """ - - if notification["subject"].startswith("backend.auto_select_manager"): - target_manager_class = self.manager_classes[notification["name"]] - self.replace_backend_manager(target_manager_class, auto_activate=True) - if ( - notification["subject"].startswith("backend.auto_activate_source") - and notification["proc_name"] == self.g_pool.process - ): - self.auto_activate_source() - - def replace_backend_manager(self, manager_class, auto_activate=False): - if not isinstance(self, manager_class): - if self.g_pool.process.startswith("eye"): - self.notify_all( - { - "subject": "start_eye_plugin", - "target": self.g_pool.process, - "name": manager_class.__name__, - } - ) - else: - self.notify_all( - {"subject": "start_plugin", "name": manager_class.__name__} - ) - if auto_activate: - self.notify_all( - { - "subject": "backend.auto_activate_source.{}".format( - self.g_pool.process - ), - "proc_name": self.g_pool.process, - "delay": 0.5, - } - ) - - def auto_activate_source(self): - """This function should be implemented in *_Manager classes - to activate the corresponding source with following preferences: - eye0: Pupil Cam1/2/3 ID0 - eye1: Pupil Cam1/2/3 ID1 - world: Pupil Cam1 ID2 - - See issue #1278 for more details. - """ - pass - - def auto_select_manager(self): - self.notify_all( - {"subject": "backend.auto_select_manager", "name": self.class_name} - ) - - def add_auto_select_button(self): - from pyglui import ui - - self.menu.append( - ui.Button("Start with default devices", self.auto_select_manager) - ) - - def add_menu(self): - super().add_menu() - from . import manager_classes - from pyglui import ui - - self.menu_icon.order = 0.1 - - # We add the capture selection menu - manager_classes.sort(key=lambda x: x.gui_name) - self.menu.append( - ui.Selector( - "capture_manager", - setter=self.replace_backend_manager, - getter=lambda: self.__class__, - selection=manager_classes, - labels=[b.gui_name for b in manager_classes], - label="Manager", - ) - ) - - # here is where you add all your menu entries. - self.menu.label = "Backend Manager" - def get_devices(self) -> T.Sequence["SourceInfo"]: return [] diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index 8acc2f44cd..4390a7cb49 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -591,68 +591,10 @@ class File_Manager(Base_Manager): Attributes: file_exts (list): File extensions to filter displayed files - root_folder (str): Folder path, which includes file sources """ - gui_name = "Video File Source" file_exts = [".mp4", ".mkv", ".mov", ".mjpeg"] - def __init__(self, g_pool, root_folder=None): - super().__init__(g_pool) - base_dir = self.g_pool.user_dir.rsplit(os.path.sep, 1)[0] - default_rec_dir = os.path.join(base_dir, "recordings") - self.root_folder = root_folder or default_rec_dir - - def init_ui(self): - self.add_menu() - from pyglui import ui - - self.add_auto_select_button() - self.menu.append( - ui.Info_Text( - "Enter a folder to enumerate all eligible video files. " - + "Be aware that entering folders with a lot of files can " - + "slow down Pupil Capture." - ) - ) - - def set_root(folder): - if not os.path.isdir(folder): - logger.error("`%s` is not a valid folder path." % folder) - else: - self.root_folder = folder - - self.menu.append( - ui.Text_Input("root_folder", self, label="Source Folder", setter=set_root) - ) - - def split_enumeration(): - eligible_files = self.enumerate_folder(self.root_folder) - eligible_files.insert(0, (None, "Select to activate")) - return zip(*eligible_files) - - self.menu.append( - ui.Selector( - "selected_file", - selection_getter=split_enumeration, - getter=lambda: None, - setter=self.activate, - label="Video File", - ) - ) - - def deinit_ui(self): - self.remove_menu() - - def activate(self, full_path): - if not full_path: - return - settings = {"source_path": full_path, "timing": "own"} - self.activate_source(settings) - - def auto_activate_source(self): - self.activate(None) - def on_drop(self, paths): for p in paths: if os.path.splitext(p)[-1] in self.file_exts: @@ -660,25 +602,11 @@ def on_drop(self, paths): return True return False - def enumerate_folder(self, path): - eligible_files = [] - is_eligible = lambda f: os.path.splitext(f)[-1] in self.file_exts - path = os.path.abspath(os.path.expanduser(path)) - for root, dirs, files in os.walk(path): - - def root_split(file): - full_p = os.path.join(root, file) - disp_p = full_p.replace(path, "") - return (full_p, disp_p) - - eligible_files.extend(map(root_split, filter(is_eligible, files))) - eligible_files.sort(key=lambda x: x[1]) - return eligible_files - - def get_init_dict(self): - return {"root_folder": self.root_folder} + def activate(self, full_path): + if not full_path: + return - def activate_source(self, settings={}): + settings = {"source_path": full_path, "timing": "own"} if self.g_pool.process == "world": self.notify_all( {"subject": "start_plugin", "name": "File_Source", "args": settings} @@ -692,6 +620,3 @@ def activate_source(self, settings={}): "args": settings, } ) - - def recent_events(self, events): - pass diff --git a/pupil_src/shared_modules/video_capture/hmd_streaming.py b/pupil_src/shared_modules/video_capture/hmd_streaming.py index fac675f692..f410985093 100644 --- a/pupil_src/shared_modules/video_capture/hmd_streaming.py +++ b/pupil_src/shared_modules/video_capture/hmd_streaming.py @@ -16,7 +16,7 @@ import zmq_tools from camera_models import Dummy_Camera, Radial_Dist_Camera -from video_capture.base_backend import Base_Manager, Base_Source +from video_capture.base_backend import Base_Source logger = logging.getLogger(__name__) @@ -127,43 +127,3 @@ def intrinsics(self, model): logger.error( "HMD Streaming backend does not support setting intrinsics manually" ) - - -class HMD_Streaming_Manager(Base_Manager): - """Simple manager to explicitly activate a fake source""" - - gui_name = "HMD Streaming" - - def __init__(self, g_pool): - super().__init__(g_pool) - - # Initiates the UI for starting the webcam. - def init_ui(self): - self.add_menu() - from pyglui import ui - - self.menu.append(ui.Info_Text("Backend for HMD Streaming")) - self.menu.append(ui.Button("Activate HMD Streaming", self.activate_source)) - - def activate_source(self): - settings = {} - # if the user set fake capture, we dont want it to auto jump back to the old capture. - if self.g_pool.process == "world": - self.notify_all( - { - "subject": "start_plugin", - "name": "HMD_Streaming_Source", - "args": settings, - } - ) - else: - logger.warning("HMD Streaming backend is not supported in the eye process.") - - def deinit_ui(self): - self.remove_menu() - - def recent_events(self, events): - pass - - def get_init_dict(self): - return super().get_init_dict() diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index 54eb969345..d2fac4c53a 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -394,14 +394,7 @@ def cleanup(self): class NDSI_Manager(Base_Manager): - """Enumerates and activates Pupil Mobile video sources - - Attributes: - network (ndsi.Network): NDSI Network backend - selected_host (unicode): Selected host uuid - """ - - gui_name = "Pupil Mobile" + """Enumerates and activates Pupil Mobile video sources""" def __init__(self, g_pool): super().__init__(g_pool) @@ -409,32 +402,20 @@ def __init__(self, g_pool): formats={ndsi.DataFormat.V3, ndsi.DataFormat.V4}, callbacks=(self.on_event,) ) self.network.start() - self.selected_host = None self._recover_in = 3 self._rejoin_in = 400 - self.should_select_host = None self.cam_selection_lut = { "eye0": ["ID0", "PI right"], "eye1": ["ID1", "PI left"], "world": ["ID2", "Logitech", "PI world"], } + + # TODO: this is now useless here! logger.warning("Make sure the `time_sync` plugin is loaded!") def cleanup(self): self.network.stop() - def init_ui(self): - self.add_menu() - self.re_build_ndsi_menu() - - def deinit_ui(self): - self.remove_menu() - - def view_host(self, host_uuid): - if self.selected_host != host_uuid: - self.selected_host = host_uuid - self.re_build_ndsi_menu() - def get_devices(self): # store hosts in dict to remove duplicates from multiple sensors active_hosts = { @@ -458,72 +439,6 @@ def get_cameras(self): if s["sensor_type"] == "video" ] - def host_selection_list(self): - devices = { - s["host_uuid"]: s["host_name"] # removes duplicates - for s in self.network.sensors.values() - } - - if devices: - return list(devices.keys()), list(devices.values()) - else: - return [None], ["No hosts found"] - - def source_selection_list(self): - default = (None, "Select to activate") - sources = [default] + [ - (s["sensor_uuid"], s["sensor_name"]) - for s in self.network.sensors.values() - if (s["sensor_type"] == "video" and s["host_uuid"] == self.selected_host) - ] - return zip(*sources) - - def re_build_ndsi_menu(self): - del self.menu[1:] - from pyglui import ui - - ui_elements = [] - ui_elements.append(ui.Info_Text("Remote Pupil Mobile sources")) - ui_elements.append( - ui.Info_Text("Pupil Mobile Commspec v{}".format(__protocol_version__)) - ) - - host_sel, host_sel_labels = self.host_selection_list() - ui_elements.append( - ui.Selector( - "selected_host", - self, - selection=host_sel, - labels=host_sel_labels, - setter=self.view_host, - label="Remote host", - ) - ) - - self.menu.extend(ui_elements) - self.add_auto_select_button() - - if not self.selected_host: - return - ui_elements = [] - - host_menu = ui.Growing_Menu("Remote Host Information") - ui_elements.append(host_menu) - - src_sel, src_sel_labels = self.source_selection_list() - host_menu.append( - ui.Selector( - "selected_source", - selection=src_sel, - labels=src_sel_labels, - getter=lambda: None, - setter=self.activate, - label="Source", - ) - ) - - self.menu.extend(ui_elements) - def activate(self, key): source_type, uid = key.split(".", maxsplit=1) if source_type == "host": @@ -555,16 +470,6 @@ def activate_source(self, source_uid): } ) - def auto_select_manager(self): - super().auto_select_manager() - self.notify_all( - { - "subject": "backend.ndsi_do_select_host", - "target_host": self.selected_host, - "delay": 0.4, - } - ) - def auto_activate_source(self, host_uid): host_sensors = [ sensor @@ -597,6 +502,7 @@ def poll_events(self): def recent_events(self, events): self.poll_events() + # TODO: Move to source if ( isinstance(self.g_pool.capture, NDSI_Source) and not self.g_pool.capture.sensor @@ -618,25 +524,12 @@ def on_event(self, caller, event): if event["subject"] == "detach": logger.debug("detached: %s" % event) sensors = [s for s in self.network.sensors.values()] - if self.selected_host == event["host_uuid"]: - if sensors: - self.selected_host = sensors[0]["host_uuid"] - else: - self.selected_host = None - self.re_build_ndsi_menu() elif event["subject"] == "attach": if event["sensor_type"] == "video": logger.debug("attached: {}".format(event)) self.notify_all({"subject": "backend.ndsi_source_found"}) - if not self.selected_host and not self.should_select_host: - self.selected_host = event["host_uuid"] - elif self.should_select_host and event["sensor_type"] == "video": - self.select_host(self.should_select_host) - - self.re_build_ndsi_menu() - def recover(self): self.g_pool.capture.recover(self.network) @@ -645,15 +538,14 @@ def on_notify(self, n): Reacts to notification: ``backend.ndsi_source_found``: Check if recovery is possible - ``backend.ndsi_do_select_host``: Switches to selected host from other process Emmits notifications: ``backend.ndsi_source_found`` - ``backend.ndsi_do_select_host` """ super().on_notify(n) + # TODO: move to source if ( n["subject"].startswith("backend.ndsi_source_found") and isinstance(self.g_pool.capture, NDSI_Source) @@ -661,22 +553,5 @@ def on_notify(self, n): ): self.recover() - if n["subject"].startswith("backend.ndsi_do_select_host"): - self.select_host(n["target_host"]) - if n["subject"] == "backend.ndsi.auto_activate_source": self.auto_activate_source(n["host_uid"]) - - def select_host(self, selected_host): - host_sel, _ = self.host_selection_list() - if selected_host in host_sel: - self.view_host(selected_host) - self.should_select_host = None - self.re_build_ndsi_menu() - src_sel, _ = self.source_selection_list() - # "Select to Activate" is always presenet as first element - if len(src_sel) >= 2: - self.auto_activate_source() - - else: - self.should_select_host = selected_host diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index 8608c50dc0..99ffbc4a73 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -820,13 +820,7 @@ def gl_display(self): class UVC_Manager(Base_Manager): - """Manages local USB sources - - Attributes: - check_intervall (float): Intervall in which to look for new UVC devices - """ - - gui_name = "Local USB" + """Manages local USB sources""" def __init__(self, g_pool): super().__init__(g_pool) @@ -837,39 +831,6 @@ def __init__(self, g_pool): "world": ["ID2", "Logitech"], } - def get_init_dict(self): - return {} - - def init_ui(self): - self.add_menu() - - from pyglui import ui - - self.add_auto_select_button() - ui_elements = [] - ui_elements.append(ui.Info_Text("Local UVC sources")) - - def dev_selection_list(): - default = (None, "Select to activate") - self.devices.update() - dev_pairs = [default] + [ - (d["uid"], d["name"]) - for d in self.devices - if "RealSense" not in d["name"] - ] - return zip(*dev_pairs) - - ui_elements.append( - ui.Selector( - "selected_source", - selection_getter=dev_selection_list, - getter=lambda: None, - setter=self.activate, - label="Activate source", - ) - ) - self.menu.extend(ui_elements) - def get_devices(self): self.devices.update() if len(self.devices) == 0: @@ -928,8 +889,6 @@ def activate_source(self, source_uid): ) def on_notify(self, notification): - super().on_notify(notification) - if notification["subject"] == "backend.uvc.auto_activate_source": self.auto_activate_source() @@ -959,12 +918,6 @@ def auto_activate_source(self): ) self.activate_source(cam["uid"]) - def deinit_ui(self): - self.remove_menu() - def cleanup(self): self.devices.cleanup() self.devices = None - - def recent_events(self, events): - pass From d2cb1edc3431e95c3a87970f9cdd47791a3a5bd2 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 17:28:21 +0100 Subject: [PATCH 44/65] Change user-facing ghost-mode to 'disconnected'. --- .../video_capture/base_backend.py | 1 - .../video_capture/file_backend.py | 2 +- .../video_capture/ndsi_backend.py | 21 +++++-------------- .../video_capture/uvc_backend.py | 14 ++++++------- 4 files changed, 12 insertions(+), 26 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 1a9aba7a13..31e4b7a1a0 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -189,7 +189,6 @@ def update_menu(self): source_settings = self.settings_ui_elements() if source_settings: settings_menu = ui.Growing_Menu(f"Settings") - settings_menu.collapsed = True settings_menu.extend(source_settings) self.menu.append(settings_menu) diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index 4390a7cb49..4b20aff3cc 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -349,7 +349,7 @@ def name(self): if self.source_path: return os.path.splitext(self.source_path)[0] else: - return "File source in ghost mode" + return "File source (no file loaded)" def get_frame_index(self): return int(self.current_frame_idx) diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index d2fac4c53a..d93b5e6a65 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -86,10 +86,7 @@ def __init__( self.recover(network) if not self.sensor or not self.sensor.supports_data_subscription: - logger.error( - "Init failed. Capture is started in ghost mode. " - + "No images will be supplied." - ) + logger.error("Could not connect to device! No images will be supplied.") self.cleanup() logger.debug("NDSI Source Sensor: %s" % self.sensor) @@ -172,7 +169,7 @@ def recent_events(self, events): elif ( self.g_pool.get_timestamp() - self.last_update > self.ghost_mode_timeout ): - logger.info("Entering ghost mode") + logger.info("Device disconnected.") if self.online: self.sensor.unlink() self.sensor = None @@ -351,16 +348,8 @@ def settings_ui_elements(self): ui.Info_Text(f"Camera: {self._sensor_name} @ {self._host_name}") ) - uvc_menu = ui.Growing_Menu("UVC Controls") - - self.control_id_ui_mapping = {} if not self.sensor: - ui_elements.append( - ui.Info_Text( - ("Sensor %s @ %s not available. " + "Running in ghost mode.") - % (self._sensor_name, self._host_name) - ) - ) + ui_elements.append(ui.Info_Text("Camera disconnected!")) return ui_elements uvc_controls = [] @@ -371,14 +360,14 @@ def settings_ui_elements(self): else: other_controls.append(entry) + uvc_menu = ui.Growing_Menu("UVC Controls") + self.control_id_ui_mapping = {} if other_controls: self.add_controls_to_menu(ui_elements, other_controls) - if uvc_controls: self.add_controls_to_menu(uvc_menu, uvc_controls) else: uvc_menu.append(ui.Info_Text("No UVC settings found.")) - ui_elements.append(uvc_menu) ui_elements.append( diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index 99ffbc4a73..b49d23b256 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -126,9 +126,7 @@ def __init__( # check if we were sucessfull if not self.uvc_capture: - logger.error( - "Init failed. Capture is started in ghost mode. No images will be supplied." - ) + logger.error("Could not connect to device! No images will be supplied.") self.name_backup = preferred_names self.frame_size_backup = frame_size self.frame_rate_backup = frame_rate @@ -453,7 +451,7 @@ def name(self): if self.uvc_capture: return self.uvc_capture.name else: - return "Ghost capture" + return "(disconnected)" @property def frame_size(self): @@ -568,6 +566,10 @@ def online(self): def settings_ui_elements(self): ui_elements = [] + if self.uvc_capture is None: + ui_elements.append(ui.Info_Text("Local USB: camera disconnected!")) + return ui_elements + ui_elements.append(ui.Info_Text(f"Camera: {self.name} @ Local USB")) # lets define some helper functions: @@ -589,10 +591,6 @@ def set_frame_rate(new_rate): self.frame_rate = new_rate self.update_menu() - if self.uvc_capture is None: - ui_elements.append(ui.Info_Text("Capture initialization failed.")) - return ui_elements - sensor_control = ui.Growing_Menu(label="Sensor Settings") sensor_control.append( ui.Info_Text("Do not change these during calibration or recording!") From f8299c28c4d88d2814859d092e8c290e6271e386 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 17:34:02 +0100 Subject: [PATCH 45/65] Cleanup NDSI log messages --- .../shared_modules/video_capture/ndsi_backend.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index d93b5e6a65..3c7a80f1a1 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -77,10 +77,7 @@ def __init__( network = manager.network if not network: - logger.debug( - "No network reference provided. Capture is started " - + "in ghost mode. No images will be supplied." - ) + logger.error("Error connecting to Pupil Mobile: No NDSI network!") return self.recover(network) @@ -89,7 +86,9 @@ def __init__( logger.error("Could not connect to device! No images will be supplied.") self.cleanup() - logger.debug("NDSI Source Sensor: %s" % self.sensor) + logger.warning( + "Make sure to enable the Time_Sync plugin for recording with Pupil Mobile!" + ) def recover(self, network): logger.debug( @@ -399,9 +398,6 @@ def __init__(self, g_pool): "world": ["ID2", "Logitech", "PI world"], } - # TODO: this is now useless here! - logger.warning("Make sure the `time_sync` plugin is loaded!") - def cleanup(self): self.network.stop() From 9031c4d6d4f772a8292ce94ed5f32ba82ec04e66 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 17:39:49 +0100 Subject: [PATCH 46/65] Add "No X found" entry in activate source --- .../shared_modules/video_capture/base_backend.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 31e4b7a1a0..7106c0aae4 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -100,9 +100,9 @@ def init_ui(self): def deinit_ui(self): self.remove_menu() - def device_list(self): - label = "Activate Camera" if self.manual_mode else "Activate Device" - entries = [(None, label)] + def source_list(self): + source_type = "Camera" if self.manual_mode else "Device" + entries = [(None, f"Activate {source_type}")] for manager in self.g_pool.source_managers: if self.manual_mode: @@ -113,6 +113,9 @@ def device_list(self): for info in sources: entries.append((info, info.label)) + if len(entries) == 1: + entries.append((None, f"No {source_type}s Found!")) + return zip(*entries) def activate_source(self, source_info): @@ -167,7 +170,7 @@ def update_menu(self): self.menu.append( ui.Selector( "selected_source", - selection_getter=self.device_list, + selection_getter=self.source_list, getter=lambda: None, setter=self.activate_source, label=" ", # TODO: Hide label completely From f9292445ff342a21abf6c4835a4feba7548effec Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 18:07:19 +0100 Subject: [PATCH 47/65] Disable continuous UI refreshing when UVC source is disconnected --- pupil_src/shared_modules/video_capture/uvc_backend.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index b49d23b256..1f160577ad 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -382,12 +382,13 @@ def _restart_logic(self): ) except (InitialisationError, uvc.InitError): time.sleep(0.02) - self.update_menu() self._restart_in = int(5 / 0.02) else: self._restart_in -= 1 def recent_events(self, events): + was_online = self.online + try: frame = self.uvc_capture.get_frame(0.05) @@ -426,6 +427,9 @@ def recent_events(self, events): events["frame"] = frame self._restart_in = 3 + if was_online != self.online: + self.update_menu() + def _get_uvc_controls(self): d = {} if self.uvc_capture: From 0820d642e805ed55ac1a23b63b330959b6f27984 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 27 Jan 2020 18:19:57 +0100 Subject: [PATCH 48/65] Fix recover logic Decided not to touch the timers. --- pupil_src/shared_modules/video_capture/ndsi_backend.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index 3c7a80f1a1..47d05a61f1 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -487,7 +487,6 @@ def poll_events(self): def recent_events(self, events): self.poll_events() - # TODO: Move to source if ( isinstance(self.g_pool.capture, NDSI_Source) and not self.g_pool.capture.sensor @@ -501,6 +500,7 @@ def recent_events(self, events): if self._rejoin_in <= 0: logger.debug("Rejoining network...") self.network.rejoin() + # frame-timeout independent timer self._rejoin_in = int(10 * 1e3 / self.g_pool.capture.get_frame_timeout) else: self._rejoin_in -= 1 @@ -516,7 +516,8 @@ def on_event(self, caller, event): self.notify_all({"subject": "backend.ndsi_source_found"}) def recover(self): - self.g_pool.capture.recover(self.network) + if isinstance(self.g_pool.capture, NDSI_Source): + self.g_pool.capture.recover(self.network) def on_notify(self, n): """Provides UI for the capture selection @@ -530,7 +531,6 @@ def on_notify(self, n): super().on_notify(n) - # TODO: move to source if ( n["subject"].startswith("backend.ndsi_source_found") and isinstance(self.g_pool.capture, NDSI_Source) From c3c52a0f6368496d6df92640d91df93c3eb2d8cc Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 28 Jan 2020 11:07:55 +0100 Subject: [PATCH 49/65] Rename settings_ui_elements to just ui_elements --- pupil_src/shared_modules/video_capture/base_backend.py | 4 ++-- pupil_src/shared_modules/video_capture/file_backend.py | 2 +- pupil_src/shared_modules/video_capture/ndsi_backend.py | 2 +- pupil_src/shared_modules/video_capture/uvc_backend.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 7106c0aae4..d0d1198c89 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -189,13 +189,13 @@ def update_menu(self): ui.Switch("manual_mode", self, label="Enable Manual Camera Selection") ) - source_settings = self.settings_ui_elements() + source_settings = self.ui_elements() if source_settings: settings_menu = ui.Growing_Menu(f"Settings") settings_menu.extend(source_settings) self.menu.append(settings_menu) - def settings_ui_elements(self) -> T.List[ui.UI_element]: + def ui_elements(self) -> T.List[ui.UI_element]: return [] def recent_events(self, events): diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index 4b20aff3cc..99ec510b5e 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -512,7 +512,7 @@ def on_notify(self, notification): ): self.play = False - def settings_ui_elements(self): + def ui_elements(self): ui_elements = [] ui_elements.append( ui.Info_Text(f"File Source: {os.path.split(self.source_path)[-1]}") diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index 47d05a61f1..35373afc70 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -340,7 +340,7 @@ def update_control_menu(self): return self.update_menu() - def settings_ui_elements(self): + def ui_elements(self): ui_elements = [] ui_elements.append( diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index 1f160577ad..3d2f3352e3 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -567,7 +567,7 @@ def jpeg_support(self): def online(self): return bool(self.uvc_capture) - def settings_ui_elements(self): + def ui_elements(self): ui_elements = [] if self.uvc_capture is None: From d977ddecb58aca3710d1655294bf47625dca6ec1 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 28 Jan 2020 11:18:10 +0100 Subject: [PATCH 50/65] Remove realsense documentation --- README.md | 8 ------ docs/dependencies-realsense-r200.md | 41 ----------------------------- 2 files changed, 49 deletions(-) delete mode 100644 docs/dependencies-realsense-r200.md diff --git a/README.md b/README.md index 1ef052742e..28cae45738 100644 --- a/README.md +++ b/README.md @@ -50,14 +50,6 @@ All setup and dependency installation instructions are contained in this repo. A - [macOS](./docs/dependencies-macos.md "Pupil dependency installation for macOS") - [Windows 10](./docs/dependencies-windows.md "Pupil dependency installation for Windows 10") -#### Intel RealSense 3D Support - -If you want to use an Intel RealSense 3D scene camera, please follow the additional setup instructions for the camera model you have. - -* **Intel RealSense R200**: Please follow our detailed [Setup Guide](./docs/dependencies-realsense-r200.md "RealSense R200 setup guide") -* **Intel RealSense D400**: You need to install the [Python wrapper for librealsense](https://github.com/IntelRealSense/librealsense/tree/master/wrappers/python#python-wrapper "Install instructions for librealsense Python wrapper") - - ### Clone the repo After you have installed all dependencies, clone this repo and start Pupil software. diff --git a/docs/dependencies-realsense-r200.md b/docs/dependencies-realsense-r200.md deleted file mode 100644 index 48d08a7afa..0000000000 --- a/docs/dependencies-realsense-r200.md +++ /dev/null @@ -1,41 +0,0 @@ -# Intel RealSense R200 Support - -**Note:** Support for the Intel RealSense R200 is currently not available for **Linux**. This is due to ["librealsense" requiring kernel patches for the "Video4Linux" backend](https://github.com/IntelRealSense/librealsense/blob/66e42069837ed6e0eb46351cc4aa2acca49a4728/doc/installation.md#video4linux-backend-preparation). - -## Dependencies - -### librealsense - -All Intel RealSense cameras require [`librealsense`](https://github.com/pupil-labs/librealsense/) to be installed. Please follow the [install instructions](https://github.com/pupil-labs/librealsense/#table-of-contents) for your operating system. - -### pyrealsense - -[`pyrealsense`](https://github.com/pupil-labs/pyrealsense) provides Python bindings for [`librealsense`](#librealsense). Run the following command in your terminal to install it. - -```sh -pip install git+https://github.com/pupil-labs/pyrealsense -``` - -## Usage - -Select `RealSense 3D` in the Capture Selection menu and activate your RealSense camera. Afterwards you should see the colored video stream of the selected camera. - -Pupil Capture accesses both streams, color and depth, at all times but only previews one at a time. Enable the `Preview Depth` option to see the normalized depth video stream. - -The `Record Depth Stream` option (enabled by default) will save the depth stream during a recording session to the file `depth.mp4` within your recording folder. - -By default, you can choose different resolutions for the color and depth streams. This is advantageous if you want to run both streams at full resolution. The Intel RealSense R200 has a maximum color resolution of `1920 x 1080` pixels and maximum depth resolution of `640 x 480` pixels. `librealsense` also provides the possibility to pixel-align color and depth streams. `Align Streams` enables this functionality. This is required if you want to infer from depth pixels to color pixels and vice versa. - -The `Sensor Settings` menu lists all available device options. These may differ depending on your OS, installed `librealsense` version, and device firmware. - - -**Note:** Not all resolutions support all frame rates. Try different resolutions if your desired frame rate is not listed. - - -### Color Frames - -Pupil Capture accesses the `YUVY` color stream of the RealSense camera. All color frames are accessible through the `events` object using the `frame` key within your plugin's `recent_events` method. - -### Depth Frames - -Depth frame objects are accessible through the `events` object using the `depth_frame` key within your plugin's `recent_events` method. The orginal 16-bit grayscale image of the camera can be accessed using the `depth` attribute of the frame object. The `bgr` attribute provides a colored image that is calculated using [histogram equalization](https://en.wikipedia.org/wiki/Histogram_equalization). These colored images are previewed in Pupil Capture, stored during recordings, and referred to as "normalized depth stream" in the above section. The [`librealsense` examples](https://github.com/IntelRealSense/librealsense/tree/master/examples) use the same coloring method to visualize depth images. From 2627900405a7ee82c44d366c36ee09bdc2166d15 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 28 Jan 2020 11:18:52 +0100 Subject: [PATCH 51/65] Remove realsense from bundle specs --- deployment/deploy_capture/bundle.spec | 8 +------- deployment/deploy_player/bundle.spec | 6 +++--- deployment/deploy_service/bundle.spec | 6 +++--- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/deployment/deploy_capture/bundle.spec b/deployment/deploy_capture/bundle.spec index c9705dd5f5..e3d6868d92 100644 --- a/deployment/deploy_capture/bundle.spec +++ b/deployment/deploy_capture/bundle.spec @@ -56,10 +56,6 @@ if platform.system() == "Darwin": sys.path.append(".") from version import pupil_version - import pyrealsense - - pyrealsense_path = pathlib.Path(pyrealsense.__file__).parent / "lrs_parsed_classes" - del sys.path[-1] a = Analysis( ["../../pupil_src/main.py"], @@ -98,11 +94,9 @@ if platform.system() == "Darwin": a.datas, [("libuvc.0.dylib", "/usr/local/lib/libuvc.0.dylib", "BINARY")], [("libglfw.dylib", "/usr/local/lib/libglfw.dylib", "BINARY")], - [("librealsense.dylib", "/usr/local/lib/librealsense.dylib", "BINARY")], [("pyglui/OpenSans-Regular.ttf", ui.get_opensans_font_path(), "DATA")], [("pyglui/Roboto-Regular.ttf", ui.get_roboto_font_path(), "DATA")], [("pyglui/pupil_icons.ttf", ui.get_pupil_icons_font_path(), "DATA")], - [("pyrealsense/lrs_parsed_classes", pyrealsense_path, "DATA")], apriltag_libs, strip=None, upx=True, @@ -127,7 +121,7 @@ elif platform.system() == "Linux": + apriltag_hidden_imports, hookspath=None, runtime_hooks=None, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) diff --git a/deployment/deploy_player/bundle.spec b/deployment/deploy_player/bundle.spec index 3db861f62a..e8ee8f64af 100644 --- a/deployment/deploy_player/bundle.spec +++ b/deployment/deploy_player/bundle.spec @@ -69,7 +69,7 @@ if platform.system() == "Darwin": ), hookspath=None, runtime_hooks=None, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) @@ -126,7 +126,7 @@ elif platform.system() == "Linux": + apriltag_hidden_imports, hookspath=None, runtime_hooks=None, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) @@ -214,7 +214,7 @@ elif platform.system() == "Windows": + apriltag_hidden_imports, hookspath=None, runtime_hooks=None, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) diff --git a/deployment/deploy_service/bundle.spec b/deployment/deploy_service/bundle.spec index 5f01431bb7..ba415e6d74 100644 --- a/deployment/deploy_service/bundle.spec +++ b/deployment/deploy_service/bundle.spec @@ -53,7 +53,7 @@ if platform.system() == "Darwin": hiddenimports=[] + av_hidden_imports + pyglui_hidden_imports, hookspath=None, runtime_hooks=None, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) exe = EXE( @@ -99,7 +99,7 @@ elif platform.system() == "Linux": hiddenimports=[] + av_hidden_imports + pyglui_hidden_imports, hookspath=None, runtime_hooks=None, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) @@ -181,7 +181,7 @@ elif platform.system() == "Windows": runtime_hooks=None, win_no_prefer_redirects=False, win_private_assemblies=False, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) From 211dfbb1a88c278879384cbf7eeed5d7a24beddb Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 28 Jan 2020 12:02:43 +0100 Subject: [PATCH 52/65] Add minimal UI for HMD source --- pupil_src/shared_modules/video_capture/hmd_streaming.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/hmd_streaming.py b/pupil_src/shared_modules/video_capture/hmd_streaming.py index f410985093..809850bc02 100644 --- a/pupil_src/shared_modules/video_capture/hmd_streaming.py +++ b/pupil_src/shared_modules/video_capture/hmd_streaming.py @@ -127,3 +127,8 @@ def intrinsics(self, model): logger.error( "HMD Streaming backend does not support setting intrinsics manually" ) + + def ui_elements(self): + ui_elements = [] + ui_elements.append(ui.Info_Text(f"HMD Streaming")) + return ui_elements From fc539f430623bdc5d513dde98213b80bcc856752 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 28 Jan 2020 12:03:24 +0100 Subject: [PATCH 53/65] Cleanup TODOs, comments, docstrings, imports --- .../video_capture/base_backend.py | 27 ++++++++++++++++--- .../video_capture/file_backend.py | 1 - .../video_capture/ndsi_backend.py | 24 +++++++---------- .../video_capture/uvc_backend.py | 13 +++++++-- 4 files changed, 44 insertions(+), 21 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index d0d1198c89..5fdbc50482 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -152,7 +152,12 @@ def on_notify(self, notification): {"subject": "backend.change_mode", "mode": self.g_pool.source_mode} ) - def update_menu(self): + def update_menu(self) -> None: + """Update the UI for the source. + + Do not overwrite this in inherited classes. Use ui_elements() instead. + """ + del self.menu[:] if self.manual_mode: @@ -173,7 +178,7 @@ def update_menu(self): selection_getter=self.source_list, getter=lambda: None, setter=self.activate_source, - label=" ", # TODO: Hide label completely + label=" ", # TODO: pyglui does not allow using no label at all ) ) @@ -196,6 +201,7 @@ def update_menu(self): self.menu.append(settings_menu) def ui_elements(self) -> T.List[ui.UI_element]: + """Returns a list of ui elements with info and settings for the source.""" return [] def recent_events(self, events): @@ -281,10 +287,14 @@ def intrinsics(self, model): class Base_Manager(Plugin): """Abstract base class for source managers. - Managers are plugins that enumerate and load accessible sources from - different backends, e.g. locally USB-connected cameras. + Managers are plugins that enumerate and load accessible sources from different + backends, e.g. locally USB-connected cameras. + + Supported sources can be either single cameras or whole devices. Identification and + activation of sources works via SourceInfo (see below). """ + # backend managers are always loaded and need to be loaded before the sources order = -1 def __init__(self, g_pool): @@ -298,16 +308,25 @@ def __init__(self, g_pool): g_pool.source_managers.append(self) def get_devices(self) -> T.Sequence["SourceInfo"]: + """Return source infos for all devices that the backend supports.""" return [] def get_cameras(self) -> T.Sequence["SourceInfo"]: + """Return source infos for all cameras that the backend supports.""" return [] def activate(self, key: T.Any) -> None: + """Activate a source (device or camera) by key from source info.""" pass class SourceInfo: + """SourceInfo is a proxy for a source (camera or device) from a manager. + + Managers hand out source infos that can be activated from other places in the code. + A manager needs to identify a source uniquely by a key. + """ + def __init__(self, label: str, manager: Base_Manager, key: T.Any): self.label = label self.manager = manager diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index 99ec510b5e..e2215ea0d7 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -22,7 +22,6 @@ import numpy as np from pyglui import ui -import player_methods as pm from camera_models import load_intrinsics from pupil_recording import PupilRecording diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index 35373afc70..f2b340f44f 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -174,7 +174,7 @@ def recent_events(self, events): self.sensor = None self._source_id = None self._initial_refresh = True - self.update_control_menu() + self.update_menu() self.last_update = self.g_pool.get_timestamp() else: time.sleep(self.get_frame_timeout / 1e3) @@ -196,7 +196,7 @@ def on_notification(self, sensor, event): or event["changes"].get("dtype") == "strmapping" or event["changes"].get("dtype") == "intmapping" ): - self.update_control_menu() + self.update_menu() # local notifications def on_notify(self, notification): @@ -259,13 +259,12 @@ def get_init_dict(self): return settings def init_ui(self): - self.has_ui = True super().init_ui() + self.has_ui = True def deinit_ui(self): - super().deinit_ui() - # TODO: Refactor this to be more uniform across sources self.has_ui = False + super().deinit_ui() def add_controls_to_menu(self, menu, controls): from pyglui import ui @@ -334,12 +333,6 @@ def initiate_value_change(val): tb.print_exc() return menu - def update_control_menu(self): - # TODO: Refactor this to be more uniform across sources - if not self.has_ui: - return - self.update_menu() - def ui_elements(self): ui_elements = [] @@ -382,7 +375,7 @@ def cleanup(self): class NDSI_Manager(Base_Manager): - """Enumerates and activates Pupil Mobile video sources""" + """Enumerates and activates NDSI video sources""" def __init__(self, g_pool): super().__init__(g_pool) @@ -520,13 +513,16 @@ def recover(self): self.g_pool.capture.recover(self.network) def on_notify(self, n): - """Provides UI for the capture selection + """Starts appropriate NDSI sources. Reacts to notification: ``backend.ndsi_source_found``: Check if recovery is possible + ``backend.ndsi.auto_activate_source``: Auto activate best source for process Emmits notifications: - ``backend.ndsi_source_found`` + ``backend.ndsi_source_found``: New NDSI source available + ``backend.ndsi.auto_activate_source``: All NDSI managers should auto activate a source + ``start_(eye_)plugin``: Starts NDSI sources """ super().on_notify(n) diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index 3d2f3352e3..59b034bb3f 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -16,10 +16,10 @@ import time import numpy as np -import uvc from pyglui import cygl, ui import gl_utils +import uvc from camera_models import load_intrinsics from version_utils import VersionFormat @@ -822,7 +822,7 @@ def gl_display(self): class UVC_Manager(Base_Manager): - """Manages local USB sources""" + """Manages local USB sources.""" def __init__(self, g_pool): super().__init__(g_pool) @@ -891,6 +891,15 @@ def activate_source(self, source_uid): ) def on_notify(self, notification): + """Starts appropriate UVC sources. + + Emits notifications: + ``backend.uvc.auto_activate_source``: All UVC managers should auto activate a source + ``start_(eye_)plugin``: Starts UVC sources + + Reacts to notifications: + ``backend.uvc.auto_activate_source``: Auto activate best source for process + """ if notification["subject"] == "backend.uvc.auto_activate_source": self.auto_activate_source() From f763257e11fa9fb997f40fcbcb9b2018745879d0 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 28 Jan 2020 12:24:15 +0100 Subject: [PATCH 54/65] Remove debug print --- pupil_src/shared_modules/video_capture/base_backend.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 5fdbc50482..0a78546027 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -119,7 +119,6 @@ def source_list(self): return zip(*entries) def activate_source(self, source_info): - print(source_info) if source_info is not None: source_info.activate() From 8d95fb118ae27f2b1ffe05385fba7bad32f2e6b0 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 28 Jan 2020 13:00:18 +0100 Subject: [PATCH 55/65] Fix incorrect pyre log level --- pupil_src/shared_modules/remote_recorder.py | 2 +- pupil_src/shared_modules/video_capture/ndsi_backend.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/remote_recorder.py b/pupil_src/shared_modules/remote_recorder.py index 20b4049cca..d6fa0f2fd3 100644 --- a/pupil_src/shared_modules/remote_recorder.py +++ b/pupil_src/shared_modules/remote_recorder.py @@ -21,7 +21,7 @@ # Suppress pyre debug logs (except beacon) logger.debug("Suppressing pyre debug logs (except zbeacon)") logging.getLogger("pyre").setLevel(logging.WARNING) -logging.getLogger("pyre.zbeacon").setLevel(logging.WARNING) +logging.getLogger("pyre.zbeacon").setLevel(logging.DEBUG) class Remote_Recording_State: diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index f2b340f44f..f2bf86cf96 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -34,7 +34,7 @@ # Suppress pyre debug logs (except beacon) logger.debug("Suppressing pyre debug logs (except zbeacon)") logging.getLogger("pyre").setLevel(logging.WARNING) -logging.getLogger("pyre.zbeacon").setLevel(logging.WARNING) +logging.getLogger("pyre.zbeacon").setLevel(logging.DEBUG) class NDSI_Source(Base_Source): From 1f664beb3c1e9fd479adb562b0f0f25bed2ab898 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 28 Jan 2020 14:23:46 +0100 Subject: [PATCH 56/65] Rename NDSI source has_ui to ui_initialized --- pupil_src/shared_modules/video_capture/ndsi_backend.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index f2bf86cf96..e64c765bf8 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -63,7 +63,7 @@ def __init__( self._host_name = host_name self._frame_size = frame_size self._frame_rate = frame_rate - self.has_ui = False + self.ui_initialized = False self.control_id_ui_mapping = {} self.get_frame_timeout = 100 # ms self.ghost_mode_timeout = 10 # sec @@ -191,7 +191,7 @@ def on_notification(self, sensor, event): logger.warning("Error {}".format(event["error_str"])) if "control_id" in event and event["control_id"] in self.sensor.controls: logger.debug(str(self.sensor.controls[event["control_id"]])) - elif self.has_ui and ( + elif self.ui_initialized and ( event["control_id"] not in self.control_id_ui_mapping or event["changes"].get("dtype") == "strmapping" or event["changes"].get("dtype") == "intmapping" @@ -260,10 +260,10 @@ def get_init_dict(self): def init_ui(self): super().init_ui() - self.has_ui = True + self.ui_initialized = True def deinit_ui(self): - self.has_ui = False + self.ui_initialized = False super().deinit_ui() def add_controls_to_menu(self, menu, controls): From f01623723fc7e23e545578b220bf643d151b565b Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 28 Jan 2020 14:25:03 +0100 Subject: [PATCH 57/65] Ensure that sources can always access source_managers even if empty --- pupil_src/shared_modules/video_capture/base_backend.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 0a78546027..16f9a5ad70 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -88,6 +88,10 @@ def __init__(self, g_pool, *, source_mode: T.Optional[SourceMode] = None, **kwar if not hasattr(self.g_pool, "source_mode"): self.g_pool.source_mode = source_mode or SourceMode.AUTO + if not hasattr(self.g_pool, "source_managers"): + # If for some reason no manager is loaded, we initialize this ourselves. + self.g_pool.source_managers = [] + def add_menu(self): super().add_menu() self.menu_icon.order = 0.2 From a091dd4642ae8789183693c62a705b942f0694cd Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Tue, 28 Jan 2020 14:43:55 +0100 Subject: [PATCH 58/65] Remove FileSource selector UI from player --- .../video_capture/base_backend.py | 16 +++++++++++++++- .../video_capture/file_backend.py | 17 ++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index 16f9a5ad70..a55b1e0b83 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -64,6 +64,7 @@ class Base_Source(Plugin): Attributes: g_pool (object): Global container, see `Plugin.g_pool` + allow_source_selection (bool): if False, no Selector will be drawn, only source info """ uniqueness = "by_base_class" @@ -75,11 +76,19 @@ class Base_Source(Plugin): def pretty_class_name(self): return "Video Source" - def __init__(self, g_pool, *, source_mode: T.Optional[SourceMode] = None, **kwargs): + def __init__( + self, + g_pool, + *, + allow_source_selection: bool = True, + source_mode: T.Optional[SourceMode] = None, + **kwargs, + ): super().__init__(g_pool) self.g_pool.capture = self self._recent_frame = None self._intrinsics = None + self.allow_source_selection = allow_source_selection # Three relevant cases for initializing source_mode: # - Plugin started at runtime: use existing source mode in g_pool @@ -163,6 +172,11 @@ def update_menu(self) -> None: del self.menu[:] + if not self.allow_source_selection: + # only render source info/settings + self.menu.extend(self.ui_elements()) + return + if self.manual_mode: self.menu.append( ui.Info_Text("Select a camera to use as input for this window.") diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index e2215ea0d7..fecbf5d0b9 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -212,10 +212,16 @@ def __init__( loop=False, buffered_decoding=False, fill_gaps=False, + allow_source_selection=False, *args, **kwargs, ): - super().__init__(g_pool, *args, **kwargs) + # NOTE: File_Source is normally not intended to be used as capture source, so we + # just want to render info in the menu. When used as capture source, need to + # specify allow_source_selection=True. + super().__init__( + g_pool, *args, allow_source_selection=allow_source_selection, **kwargs + ) if self.timing == "external": self.recent_events = self.recent_events_external_timing else: @@ -558,9 +564,6 @@ def toggle_looping(val): ) return ui_elements - def deinit_ui(self): - self.remove_menu() - def cleanup(self): try: self.video_stream.cleanup() @@ -605,7 +608,11 @@ def activate(self, full_path): if not full_path: return - settings = {"source_path": full_path, "timing": "own"} + settings = { + "source_path": full_path, + "timing": "own", + "allow_source_selection": True, + } if self.g_pool.process == "world": self.notify_all( {"subject": "start_plugin", "name": "File_Source", "args": settings} From 86564527707b83b4c67491e220ddca3fd8b934d3 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 29 Jan 2020 11:00:04 +0100 Subject: [PATCH 59/65] Catch NDSI race condition with better user feedback --- pupil_src/shared_modules/video_capture/ndsi_backend.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index e64c765bf8..3d8f839c05 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -154,6 +154,13 @@ def recent_events(self, events): ) except ndsi.StreamError: frame = None + except ndsi.sensor.NotDataSubSupportedError: + # NOTE: This (most likely) is a race-condition in NDSI initialization + # that is waiting to be fixed for Pupil Mobile. It happens rarely and + # can be solved by simply reconnecting the headset to the mobile phone. + # Preventing traceback logfloods here and displaying more helpful + # message to the user. + logger.warning("Connection problem! Please reconnect headset to phone!") except Exception: frame = None import traceback From a6593848372b57843cd3214ea3a6183ccb9782f2 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 29 Jan 2020 11:58:50 +0100 Subject: [PATCH 60/65] Hide File_Source UI by default and do not store in session settings - Show UI only when instanted by manager - Do not store in session settings - Make sure there's a default capture even if restoring session --- pupil_src/launchables/eye.py | 24 ++++++++----- pupil_src/launchables/world.py | 10 +++++- .../video_capture/base_backend.py | 14 +------- .../video_capture/file_backend.py | 35 +++++++++++-------- 4 files changed, 45 insertions(+), 38 deletions(-) diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index d607431281..bac84150fb 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -213,18 +213,17 @@ def get_timestamp(): ] if eye_id == 0: preferred_names += ["HD-6000"] - default_capture_settings = ( - "UVC_Source", - { - "preferred_names": preferred_names, - "frame_size": (320, 240), - "frame_rate": 120, - }, - ) + + default_capture = "UVC_Source" + default_capture_settings = { + "preferred_names": preferred_names, + "frame_size": (320, 240), + "frame_rate": 120, + } default_plugins = [ # TODO: extend with plugins - default_capture_settings, + (default_capture, default_capture_settings), ("UVC_Manager", {}), ("NDSI_Manager", {}), ("HMD_Streaming_Manager", {}), @@ -440,6 +439,13 @@ def uroi_on_mouse_button(button, action, mods): g_pool.plugins = Plugin_List(g_pool, plugins_to_load) + if not g_pool.capture: + # Make sure we always have a capture running. Important if there was no + # capture stored in session settings. + g_pool.plugins.add( + g_pool.plugin_by_name[default_capture], default_capture_settings + ) + g_pool.writer = None g_pool.u_r = UIRoi((g_pool.capture.frame_size[1], g_pool.capture.frame_size[0])) diff --git a/pupil_src/launchables/world.py b/pupil_src/launchables/world.py index 44c1495414..7c2973ecf6 100644 --- a/pupil_src/launchables/world.py +++ b/pupil_src/launchables/world.py @@ -288,6 +288,7 @@ def get_timestamp(): ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} + default_capture = "UVC_Source" default_capture_settings = { "preferred_names": [ "Pupil Cam1 ID2", @@ -305,7 +306,7 @@ def get_timestamp(): } default_plugins = [ - ("UVC_Source", default_capture_settings), + (default_capture, default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), ("NDSI_Manager", {}), @@ -576,6 +577,13 @@ def set_window_size(): g_pool, session_settings.get("loaded_plugins", default_plugins) ) + if not g_pool.capture: + # Make sure we always have a capture running. Important if there was no + # capture stored in session settings. + g_pool.plugins.add( + g_pool.plugin_by_name[default_capture], default_capture_settings + ) + # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index a55b1e0b83..a9d6f41f04 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -64,7 +64,6 @@ class Base_Source(Plugin): Attributes: g_pool (object): Global container, see `Plugin.g_pool` - allow_source_selection (bool): if False, no Selector will be drawn, only source info """ uniqueness = "by_base_class" @@ -77,18 +76,12 @@ def pretty_class_name(self): return "Video Source" def __init__( - self, - g_pool, - *, - allow_source_selection: bool = True, - source_mode: T.Optional[SourceMode] = None, - **kwargs, + self, g_pool, *, source_mode: T.Optional[SourceMode] = None, **kwargs, ): super().__init__(g_pool) self.g_pool.capture = self self._recent_frame = None self._intrinsics = None - self.allow_source_selection = allow_source_selection # Three relevant cases for initializing source_mode: # - Plugin started at runtime: use existing source mode in g_pool @@ -172,11 +165,6 @@ def update_menu(self) -> None: del self.menu[:] - if not self.allow_source_selection: - # only render source info/settings - self.menu.extend(self.ui_elements()) - return - if self.manual_mode: self.menu.append( ui.Info_Text("Select a camera to use as input for this window.") diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index fecbf5d0b9..9c9a54e2b6 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -195,6 +195,9 @@ def get_frame_iterator(self): class File_Source(Playback_Source, Base_Source): """Simple file capture. + Note that File_Source is special since it is usually not intended to be used as + capture plugin. Therefore it hides it's UI by default. + Playback_Source arguments: timing (str): "external", "own" (default), None @@ -203,6 +206,7 @@ class File_Source(Playback_Source, Base_Source): loop (bool): loop video set if timing!="external" buffered_decoding (bool): use buffered decode fill_gaps (bool): fill gaps with static frames + show_plugin_menu (bool): enable to show regular capture UI with source selection """ def __init__( @@ -212,16 +216,11 @@ def __init__( loop=False, buffered_decoding=False, fill_gaps=False, - allow_source_selection=False, + show_plugin_menu=False, *args, **kwargs, ): - # NOTE: File_Source is normally not intended to be used as capture source, so we - # just want to render info in the menu. When used as capture source, need to - # specify allow_source_selection=True. - super().__init__( - g_pool, *args, allow_source_selection=allow_source_selection, **kwargs - ) + super().__init__(g_pool, *args, **kwargs) if self.timing == "external": self.recent_events = self.recent_events_external_timing else: @@ -247,6 +246,8 @@ def __init__( self.reset_video() self._intrinsics = load_intrinsics(rec, set_name, self.frame_size) + self.show_plugin_menu = show_plugin_menu + def get_rec_set_name(self, source_path): """ Return dir and set name by source_path @@ -340,14 +341,18 @@ def frame_size(self): def frame_rate(self): return self._frame_rate + def init_ui(self): + if self.show_plugin_menu: + super().init_ui() + + def deinit_ui(self): + if self.show_plugin_menu: + super().deinit_ui() + def get_init_dict(self): - if self.g_pool.app == "capture": - settings = super().get_init_dict() - settings["source_path"] = self.source_path - settings["loop"] = self.loop - return settings - else: - raise NotImplementedError() + # We do not want to store file capture as selected plugin since we would have to + # do a lot of validation on opening. + raise NotImplementedError() @property def name(self): @@ -611,7 +616,7 @@ def activate(self, full_path): settings = { "source_path": full_path, "timing": "own", - "allow_source_selection": True, + "show_plugin_menu": True, } if self.g_pool.process == "world": self.notify_all( From 14ac7e054702aa467a4fc199c39701a2c404b812 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Wed, 29 Jan 2020 12:03:49 +0100 Subject: [PATCH 61/65] Fix source mode toggle not working for File_Source --- pupil_src/shared_modules/video_capture/file_backend.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index 9c9a54e2b6..91f052d94e 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -506,6 +506,7 @@ def seek_to_frame(self, seek_pos): self.target_frame_idx = seek_pos def on_notify(self, notification): + super().on_notify(notification) if ( notification["subject"] == "file_source.seek" and notification.get("source_path") == self.source_path From f316d675c08e57ce1ab9ac7607fd21a2218e9a52 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Thu, 30 Jan 2020 13:16:35 +0100 Subject: [PATCH 62/65] Make File_Source persistent in capture Display appropriate message if file does not exist --- .../video_capture/base_backend.py | 3 ++ .../video_capture/file_backend.py | 34 ++++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index a9d6f41f04..e594563c3e 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -389,3 +389,6 @@ def wait(self, timestamp): sleep(target_wait_time) self._recent_wait_ts = timestamp self.finished_sleep = monotonic() + + def get_init_dict(self): + return dict(**super().get_init_dict(), timing=self.timing) diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index 91f052d94e..d20cab7dbc 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -333,6 +333,10 @@ def _get_streams(self, container, should_buffer): def initialised(self): return not self.videoset.is_empty() + @property + def online(self): + return not self.videoset.is_empty() + @property def frame_size(self): return self.video_stream.frame_size @@ -350,9 +354,14 @@ def deinit_ui(self): super().deinit_ui() def get_init_dict(self): - # We do not want to store file capture as selected plugin since we would have to - # do a lot of validation on opening. - raise NotImplementedError() + return dict( + **super().get_init_dict(), + source_path=self.source_path, + loop=self.loop, + buffered_decoding=self.buffering, + fill_gaps=self.fill_gaps, + show_plugin_menu=self.show_plugin_menu, + ) @property def name(self): @@ -529,14 +538,21 @@ def ui_elements(self): ui.Info_Text(f"File Source: {os.path.split(self.source_path)[-1]}") ) - if self.g_pool.app == "capture": + if not self.online: + ui_elements.append( + ui.Info_Text( + "Could not playback file! Check if file exists and if" + " corresponding timestamps file is present." + ) + ) + return ui_elements - def toggle_looping(val): - self.loop = val - if val: - self.play = True + def toggle_looping(val): + self.loop = val + if val: + self.play = True - ui_elements.append(ui.Switch("loop", self, setter=toggle_looping)) + ui_elements.append(ui.Switch("loop", self, setter=toggle_looping)) ui_elements.append( ui.Text_Input("source_path", self, label="Full path", setter=lambda x: None) From f33cf04aaba8e56159c15b87af870f61908453df Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Thu, 30 Jan 2020 13:22:08 +0100 Subject: [PATCH 63/65] Rename default_capture to default_capture_name --- pupil_src/launchables/eye.py | 6 +++--- pupil_src/launchables/world.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index 536bd81a5d..d824bdeb38 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -214,7 +214,7 @@ def get_timestamp(): if eye_id == 0: preferred_names += ["HD-6000"] - default_capture = "UVC_Source" + default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": preferred_names, "frame_size": (320, 240), @@ -223,7 +223,7 @@ def get_timestamp(): default_plugins = [ # TODO: extend with plugins - (default_capture, default_capture_settings), + (default_capture_name, default_capture_settings), ("UVC_Manager", {}), ("NDSI_Manager", {}), ("HMD_Streaming_Manager", {}), @@ -419,7 +419,7 @@ def set_window_size(): # Make sure we always have a capture running. Important if there was no # capture stored in session settings. g_pool.plugins.add( - g_pool.plugin_by_name[default_capture], default_capture_settings + g_pool.plugin_by_name[default_capture_name], default_capture_settings ) g_pool.writer = None diff --git a/pupil_src/launchables/world.py b/pupil_src/launchables/world.py index 29f4f4c04a..b56407cee1 100644 --- a/pupil_src/launchables/world.py +++ b/pupil_src/launchables/world.py @@ -288,7 +288,7 @@ def get_timestamp(): ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} - default_capture = "UVC_Source" + default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": [ "Pupil Cam1 ID2", @@ -306,7 +306,7 @@ def get_timestamp(): } default_plugins = [ - (default_capture, default_capture_settings), + (default_capture_name, default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), ("NDSI_Manager", {}), @@ -581,7 +581,7 @@ def set_window_size(): # Make sure we always have a capture running. Important if there was no # capture stored in session settings. g_pool.plugins.add( - g_pool.plugin_by_name[default_capture], default_capture_settings + g_pool.plugin_by_name[default_capture_name], default_capture_settings ) # Register callbacks main_window From 0f9d4c7059aa058a6061351b89818745f4a77f72 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 3 Feb 2020 11:08:14 +0100 Subject: [PATCH 64/65] Adjust copy text after revision --- pupil_src/shared_modules/video_capture/base_backend.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index e594563c3e..c4232a954f 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -172,8 +172,8 @@ def update_menu(self) -> None: else: self.menu.append( ui.Info_Text( - "Select a device to use as video input." - " The best matching cameras will be automatically selected." + "Select a Pupil Core headset from the list." + " Cameras will be automatically selected for world and eye windows." ) ) From e1a8305e918bb1046d5d6adfbb8c01040439bfd1 Mon Sep 17 00:00:00 2001 From: Patrick Faion Date: Mon, 3 Feb 2020 13:15:17 +0100 Subject: [PATCH 65/65] Fix crash in Player --- pupil_src/shared_modules/video_capture/base_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index c4232a954f..8a805fa739 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -152,7 +152,7 @@ def on_notify(self, notification): elif subject == "eye_process.started": # Make sure to broadcast current source mode once to newly started eyes so # they are always in sync! - if self.g_pool.process == "world": + if self.g_pool.app == "capture" and self.g_pool.process == "world": self.notify_all( {"subject": "backend.change_mode", "mode": self.g_pool.source_mode} )