diff --git a/README.md b/README.md index 1ef052742e..28cae45738 100644 --- a/README.md +++ b/README.md @@ -50,14 +50,6 @@ All setup and dependency installation instructions are contained in this repo. A - [macOS](./docs/dependencies-macos.md "Pupil dependency installation for macOS") - [Windows 10](./docs/dependencies-windows.md "Pupil dependency installation for Windows 10") -#### Intel RealSense 3D Support - -If you want to use an Intel RealSense 3D scene camera, please follow the additional setup instructions for the camera model you have. - -* **Intel RealSense R200**: Please follow our detailed [Setup Guide](./docs/dependencies-realsense-r200.md "RealSense R200 setup guide") -* **Intel RealSense D400**: You need to install the [Python wrapper for librealsense](https://github.com/IntelRealSense/librealsense/tree/master/wrappers/python#python-wrapper "Install instructions for librealsense Python wrapper") - - ### Clone the repo After you have installed all dependencies, clone this repo and start Pupil software. diff --git a/deployment/deploy_capture/bundle.spec b/deployment/deploy_capture/bundle.spec index c9705dd5f5..e3d6868d92 100644 --- a/deployment/deploy_capture/bundle.spec +++ b/deployment/deploy_capture/bundle.spec @@ -56,10 +56,6 @@ if platform.system() == "Darwin": sys.path.append(".") from version import pupil_version - import pyrealsense - - pyrealsense_path = pathlib.Path(pyrealsense.__file__).parent / "lrs_parsed_classes" - del sys.path[-1] a = Analysis( ["../../pupil_src/main.py"], @@ -98,11 +94,9 @@ if platform.system() == "Darwin": a.datas, [("libuvc.0.dylib", "/usr/local/lib/libuvc.0.dylib", "BINARY")], [("libglfw.dylib", "/usr/local/lib/libglfw.dylib", "BINARY")], - [("librealsense.dylib", "/usr/local/lib/librealsense.dylib", "BINARY")], [("pyglui/OpenSans-Regular.ttf", ui.get_opensans_font_path(), "DATA")], [("pyglui/Roboto-Regular.ttf", ui.get_roboto_font_path(), "DATA")], [("pyglui/pupil_icons.ttf", ui.get_pupil_icons_font_path(), "DATA")], - [("pyrealsense/lrs_parsed_classes", pyrealsense_path, "DATA")], apriltag_libs, strip=None, upx=True, @@ -127,7 +121,7 @@ elif platform.system() == "Linux": + apriltag_hidden_imports, hookspath=None, runtime_hooks=None, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) diff --git a/deployment/deploy_player/bundle.spec b/deployment/deploy_player/bundle.spec index 3db861f62a..e8ee8f64af 100644 --- a/deployment/deploy_player/bundle.spec +++ b/deployment/deploy_player/bundle.spec @@ -69,7 +69,7 @@ if platform.system() == "Darwin": ), hookspath=None, runtime_hooks=None, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) @@ -126,7 +126,7 @@ elif platform.system() == "Linux": + apriltag_hidden_imports, hookspath=None, runtime_hooks=None, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) @@ -214,7 +214,7 @@ elif platform.system() == "Windows": + apriltag_hidden_imports, hookspath=None, runtime_hooks=None, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) diff --git a/deployment/deploy_service/bundle.spec b/deployment/deploy_service/bundle.spec index 5f01431bb7..ba415e6d74 100644 --- a/deployment/deploy_service/bundle.spec +++ b/deployment/deploy_service/bundle.spec @@ -53,7 +53,7 @@ if platform.system() == "Darwin": hiddenimports=[] + av_hidden_imports + pyglui_hidden_imports, hookspath=None, runtime_hooks=None, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) exe = EXE( @@ -99,7 +99,7 @@ elif platform.system() == "Linux": hiddenimports=[] + av_hidden_imports + pyglui_hidden_imports, hookspath=None, runtime_hooks=None, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) @@ -181,7 +181,7 @@ elif platform.system() == "Windows": runtime_hooks=None, win_no_prefer_redirects=False, win_private_assemblies=False, - excludes=["matplotlib", "pyrealsense"], + excludes=["matplotlib"], ) pyz = PYZ(a.pure) diff --git a/docs/dependencies-realsense-r200.md b/docs/dependencies-realsense-r200.md deleted file mode 100644 index 48d08a7afa..0000000000 --- a/docs/dependencies-realsense-r200.md +++ /dev/null @@ -1,41 +0,0 @@ -# Intel RealSense R200 Support - -**Note:** Support for the Intel RealSense R200 is currently not available for **Linux**. This is due to ["librealsense" requiring kernel patches for the "Video4Linux" backend](https://github.com/IntelRealSense/librealsense/blob/66e42069837ed6e0eb46351cc4aa2acca49a4728/doc/installation.md#video4linux-backend-preparation). - -## Dependencies - -### librealsense - -All Intel RealSense cameras require [`librealsense`](https://github.com/pupil-labs/librealsense/) to be installed. Please follow the [install instructions](https://github.com/pupil-labs/librealsense/#table-of-contents) for your operating system. - -### pyrealsense - -[`pyrealsense`](https://github.com/pupil-labs/pyrealsense) provides Python bindings for [`librealsense`](#librealsense). Run the following command in your terminal to install it. - -```sh -pip install git+https://github.com/pupil-labs/pyrealsense -``` - -## Usage - -Select `RealSense 3D` in the Capture Selection menu and activate your RealSense camera. Afterwards you should see the colored video stream of the selected camera. - -Pupil Capture accesses both streams, color and depth, at all times but only previews one at a time. Enable the `Preview Depth` option to see the normalized depth video stream. - -The `Record Depth Stream` option (enabled by default) will save the depth stream during a recording session to the file `depth.mp4` within your recording folder. - -By default, you can choose different resolutions for the color and depth streams. This is advantageous if you want to run both streams at full resolution. The Intel RealSense R200 has a maximum color resolution of `1920 x 1080` pixels and maximum depth resolution of `640 x 480` pixels. `librealsense` also provides the possibility to pixel-align color and depth streams. `Align Streams` enables this functionality. This is required if you want to infer from depth pixels to color pixels and vice versa. - -The `Sensor Settings` menu lists all available device options. These may differ depending on your OS, installed `librealsense` version, and device firmware. - - -**Note:** Not all resolutions support all frame rates. Try different resolutions if your desired frame rate is not listed. - - -### Color Frames - -Pupil Capture accesses the `YUVY` color stream of the RealSense camera. All color frames are accessible through the `events` object using the `frame` key within your plugin's `recent_events` method. - -### Depth Frames - -Depth frame objects are accessible through the `events` object using the `depth_frame` key within your plugin's `recent_events` method. The orginal 16-bit grayscale image of the camera can be accessed using the `depth` attribute of the frame object. The `bgr` attribute provides a colored image that is calculated using [histogram equalization](https://en.wikipedia.org/wiki/Histogram_equalization). These colored images are previewed in Pupil Capture, stored during recordings, and referred to as "normalized depth stream" in the above section. The [`librealsense` examples](https://github.com/IntelRealSense/librealsense/tree/master/examples) use the same coloring method to visualize depth images. diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index ce2b81d6ee..d824bdeb38 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -213,19 +213,21 @@ def get_timestamp(): ] if eye_id == 0: preferred_names += ["HD-6000"] - default_capture_settings = ( - "UVC_Source", - { - "preferred_names": preferred_names, - "frame_size": (320, 240), - "frame_rate": 120, - }, - ) + + default_capture_name = "UVC_Source" + default_capture_settings = { + "preferred_names": preferred_names, + "frame_size": (320, 240), + "frame_rate": 120, + } default_plugins = [ # TODO: extend with plugins - default_capture_settings, + (default_capture_name, default_capture_settings), ("UVC_Manager", {}), + ("NDSI_Manager", {}), + ("HMD_Streaming_Manager", {}), + ("File_Manager", {}), # Detector needs to be loaded first to set `g_pool.pupil_detector` (default_detector_cls.__name__, {}), ("PupilDetectorManager", {}), @@ -413,6 +415,13 @@ def set_window_size(): g_pool.plugins = Plugin_List(g_pool, plugins_to_load) + if not g_pool.capture: + # Make sure we always have a capture running. Important if there was no + # capture stored in session settings. + g_pool.plugins.add( + g_pool.plugin_by_name[default_capture_name], default_capture_settings + ) + g_pool.writer = None # Register callbacks main_window diff --git a/pupil_src/launchables/world.py b/pupil_src/launchables/world.py index 382af3fdb6..b56407cee1 100644 --- a/pupil_src/launchables/world.py +++ b/pupil_src/launchables/world.py @@ -288,6 +288,7 @@ def get_timestamp(): ] g_pool.plugin_by_name = {p.__name__: p for p in plugins} + default_capture_name = "UVC_Source" default_capture_settings = { "preferred_names": [ "Pupil Cam1 ID2", @@ -305,9 +306,12 @@ def get_timestamp(): } default_plugins = [ - ("UVC_Source", default_capture_settings), + (default_capture_name, default_capture_settings), ("Pupil_Data_Relay", {}), ("UVC_Manager", {}), + ("NDSI_Manager", {}), + ("HMD_Streaming_Manager", {}), + ("File_Manager", {}), ("Log_Display", {}), ("Dummy_Gaze_Mapper", {}), ("Display_Recent_Gaze", {}), @@ -573,6 +577,13 @@ def set_window_size(): g_pool, session_settings.get("loaded_plugins", default_plugins) ) + if not g_pool.capture: + # Make sure we always have a capture running. Important if there was no + # capture stored in session settings. + g_pool.plugins.add( + g_pool.plugin_by_name[default_capture_name], default_capture_settings + ) + # Register callbacks main_window glfw.glfwSetFramebufferSizeCallback(main_window, on_resize) glfw.glfwSetKeyCallback(main_window, on_window_key) diff --git a/pupil_src/shared_modules/plugin.py b/pupil_src/shared_modules/plugin.py index 928b79d718..1d6193b795 100644 --- a/pupil_src/shared_modules/plugin.py +++ b/pupil_src/shared_modules/plugin.py @@ -322,6 +322,9 @@ def __init__(self, g_pool, plugin_initializers): expanded_initializers.append((plugin_by_name[name], name, args)) except KeyError: logger.debug(f"Plugin {name} failed to load, not available for import.") + + expanded_initializers.sort(key=lambda data: data[0].order) + # only add plugins that won't be replaced by newer plugins for i, (plugin, name, args) in enumerate(expanded_initializers): for new_plugin, new_name, _ in expanded_initializers[i + 1 :]: diff --git a/pupil_src/shared_modules/remote_recorder.py b/pupil_src/shared_modules/remote_recorder.py index 80f3c10d11..d6fa0f2fd3 100644 --- a/pupil_src/shared_modules/remote_recorder.py +++ b/pupil_src/shared_modules/remote_recorder.py @@ -18,6 +18,11 @@ logger = logging.getLogger(__name__) +# Suppress pyre debug logs (except beacon) +logger.debug("Suppressing pyre debug logs (except zbeacon)") +logging.getLogger("pyre").setLevel(logging.WARNING) +logging.getLogger("pyre.zbeacon").setLevel(logging.DEBUG) + class Remote_Recording_State: __slots__ = ["sensor"] diff --git a/pupil_src/shared_modules/video_capture/__init__.py b/pupil_src/shared_modules/video_capture/__init__.py index c4e1767528..4c075bddc8 100644 --- a/pupil_src/shared_modules/video_capture/__init__.py +++ b/pupil_src/shared_modules/video_capture/__init__.py @@ -37,7 +37,6 @@ InitialisationError, StreamError, ) -from .fake_backend import Fake_Manager, Fake_Source from .file_backend import File_Manager, File_Source, FileSeekError from .hmd_streaming import HMD_Streaming_Source from .uvc_backend import UVC_Manager, UVC_Source @@ -45,8 +44,8 @@ logger = logging.getLogger(__name__) -source_classes = [File_Source, UVC_Source, Fake_Source, HMD_Streaming_Source] -manager_classes = [File_Manager, UVC_Manager, Fake_Manager] +source_classes = [File_Source, UVC_Source, HMD_Streaming_Source] +manager_classes = [File_Manager, UVC_Manager] try: from .ndsi_backend import NDSI_Source, NDSI_Manager @@ -55,21 +54,3 @@ else: source_classes.append(NDSI_Source) manager_classes.append(NDSI_Manager) - -try: - from .realsense_backend import Realsense_Source, Realsense_Manager -except ImportError: - logger.debug("Install pyrealsense to use the Intel RealSense backend") -else: - source_classes.append(Realsense_Source) - manager_classes.append(Realsense_Manager) - -try: - from .realsense2_backend import Realsense2_Source, Realsense2_Manager -except ImportError: - logger.debug( - "Install pyrealsense2 to use the Intel RealSense backend for D400 series cameras" - ) -else: - source_classes.append(Realsense2_Source) - manager_classes.append(Realsense2_Manager) diff --git a/pupil_src/shared_modules/video_capture/base_backend.py b/pupil_src/shared_modules/video_capture/base_backend.py index c70e283b35..8a805fa739 100644 --- a/pupil_src/shared_modules/video_capture/base_backend.py +++ b/pupil_src/shared_modules/video_capture/base_backend.py @@ -10,10 +10,12 @@ """ import logging +import typing as T +from enum import IntEnum, auto from time import monotonic, sleep import numpy as np -from pyglui import cygl +from pyglui import cygl, ui import gl_utils from plugin import Plugin @@ -37,6 +39,12 @@ class NoMoreVideoError(Exception): pass +class SourceMode(IntEnum): + # NOTE: IntEnum is serializable with msgpack + AUTO = auto() + MANUAL = auto() + + class Base_Source(Plugin): """Abstract source class @@ -63,22 +71,150 @@ class Base_Source(Plugin): icon_chr = chr(0xE412) icon_font = "pupil_icons" - def __init__(self, g_pool): + @property + def pretty_class_name(self): + return "Video Source" + + def __init__( + self, g_pool, *, source_mode: T.Optional[SourceMode] = None, **kwargs, + ): super().__init__(g_pool) self.g_pool.capture = self self._recent_frame = None self._intrinsics = None + # Three relevant cases for initializing source_mode: + # - Plugin started at runtime: use existing source mode in g_pool + # - Fresh start without settings: initialize to auto + # - Start with settings: will be passed as parameter, use those + if not hasattr(self.g_pool, "source_mode"): + self.g_pool.source_mode = source_mode or SourceMode.AUTO + + if not hasattr(self.g_pool, "source_managers"): + # If for some reason no manager is loaded, we initialize this ourselves. + self.g_pool.source_managers = [] + def add_menu(self): super().add_menu() self.menu_icon.order = 0.2 + def init_ui(self): + self.add_menu() + self.menu.label = "Video Source" + self.update_menu() + + def deinit_ui(self): + self.remove_menu() + + def source_list(self): + source_type = "Camera" if self.manual_mode else "Device" + entries = [(None, f"Activate {source_type}")] + + for manager in self.g_pool.source_managers: + if self.manual_mode: + sources = manager.get_cameras() + else: + sources = manager.get_devices() + + for info in sources: + entries.append((info, info.label)) + + if len(entries) == 1: + entries.append((None, f"No {source_type}s Found!")) + + return zip(*entries) + + def activate_source(self, source_info): + if source_info is not None: + source_info.activate() + + @property + def manual_mode(self) -> bool: + return self.g_pool.source_mode == SourceMode.MANUAL + + @manual_mode.setter + def manual_mode(self, enable) -> None: + new_mode = SourceMode.MANUAL if enable else SourceMode.AUTO + if new_mode != self.g_pool.source_mode: + logger.debug(f"Setting source mode: {new_mode.name}") + self.notify_all({"subject": "backend.change_mode", "mode": new_mode}) + + def on_notify(self, notification): + subject = notification["subject"] + + if subject == "backend.change_mode": + mode = SourceMode(notification["mode"]) + if mode != self.g_pool.source_mode: + self.g_pool.source_mode = mode + # redraw menu to close potentially open (and now incorrect) dropdown + self.update_menu() + + elif subject == "eye_process.started": + # Make sure to broadcast current source mode once to newly started eyes so + # they are always in sync! + if self.g_pool.app == "capture" and self.g_pool.process == "world": + self.notify_all( + {"subject": "backend.change_mode", "mode": self.g_pool.source_mode} + ) + + def update_menu(self) -> None: + """Update the UI for the source. + + Do not overwrite this in inherited classes. Use ui_elements() instead. + """ + + del self.menu[:] + + if self.manual_mode: + self.menu.append( + ui.Info_Text("Select a camera to use as input for this window.") + ) + else: + self.menu.append( + ui.Info_Text( + "Select a Pupil Core headset from the list." + " Cameras will be automatically selected for world and eye windows." + ) + ) + + self.menu.append( + ui.Selector( + "selected_source", + selection_getter=self.source_list, + getter=lambda: None, + setter=self.activate_source, + label=" ", # TODO: pyglui does not allow using no label at all + ) + ) + + if not self.manual_mode: + self.menu.append( + ui.Info_Text( + "Enable manual camera selection to choose a specific camera" + " as input for every window." + ) + ) + + self.menu.append( + ui.Switch("manual_mode", self, label="Enable Manual Camera Selection") + ) + + source_settings = self.ui_elements() + if source_settings: + settings_menu = ui.Growing_Menu(f"Settings") + settings_menu.extend(source_settings) + self.menu.append(settings_menu) + + def ui_elements(self) -> T.List[ui.UI_element]: + """Returns a list of ui elements with info and settings for the source.""" + return [] + def recent_events(self, events): """Returns None Adds events['frame']=Frame(args) Frame: Object containing image and time information of the current - source frame. See `fake_source.py` for a minimal implementation. + source frame. """ raise NotImplementedError() @@ -110,7 +246,7 @@ def name(self): raise NotImplementedError() def get_init_dict(self): - return {} + return {"source_mode": self.g_pool.source_mode} @property def frame_size(self): @@ -120,10 +256,6 @@ def frame_size(self): """ raise NotImplementedError() - @frame_size.setter - def frame_size(self, new_size): - raise NotImplementedError() - @property def frame_rate(self): """ @@ -132,10 +264,6 @@ def frame_rate(self): """ raise NotImplementedError() - @frame_rate.setter - def frame_rate(self, new_rate): - pass - @property def jpeg_support(self): """ @@ -164,116 +292,56 @@ def intrinsics(self, model): class Base_Manager(Plugin): """Abstract base class for source managers. - Managers are plugins that enumerate and load accessible sources from - different backends, e.g. locally USB-connected cameras. + Managers are plugins that enumerate and load accessible sources from different + backends, e.g. locally USB-connected cameras. - Attributes: - gui_name (str): String used for manager selector labels + Supported sources can be either single cameras or whole devices. Identification and + activation of sources works via SourceInfo (see below). """ - uniqueness = "by_base_class" - gui_name = "Base Manager" - icon_chr = chr(0xEC01) - icon_font = "pupil_icons" + # backend managers are always loaded and need to be loaded before the sources + order = -1 def __init__(self, g_pool): super().__init__(g_pool) - from . import manager_classes + # register all instances in g_pool.source_managers list + if not hasattr(g_pool, "source_managers"): + g_pool.source_managers = [] - self.manager_classes = {m.__name__: m for m in manager_classes} + if self not in g_pool.source_managers: + g_pool.source_managers.append(self) - def on_notify(self, notification): - """ - Reacts to notification: - ``backend.auto_select_manager``: Changes the current Manager to one that's emitted - ``backend.auto_activate_source``: Activates the current source via self.auto_activate_source() - - Emmits notifications (indirectly): - ``start_plugin``: For world thread - ``start_eye_plugin``: For eye thread - ``backend.auto_activate_source`` - """ - - if notification["subject"].startswith("backend.auto_select_manager"): - target_manager_class = self.manager_classes[notification["name"]] - self.replace_backend_manager(target_manager_class, auto_activate=True) - if ( - notification["subject"].startswith("backend.auto_activate_source") - and notification["proc_name"] == self.g_pool.process - ): - self.auto_activate_source() - - def replace_backend_manager(self, manager_class, auto_activate=False): - if not isinstance(self, manager_class): - if self.g_pool.process.startswith("eye"): - self.notify_all( - { - "subject": "start_eye_plugin", - "target": self.g_pool.process, - "name": manager_class.__name__, - } - ) - else: - self.notify_all( - {"subject": "start_plugin", "name": manager_class.__name__} - ) - if auto_activate: - self.notify_all( - { - "subject": "backend.auto_activate_source.{}".format( - self.g_pool.process - ), - "proc_name": self.g_pool.process, - "delay": 0.5, - } - ) + def get_devices(self) -> T.Sequence["SourceInfo"]: + """Return source infos for all devices that the backend supports.""" + return [] - def auto_activate_source(self): - """This function should be implemented in *_Manager classes - to activate the corresponding source with following preferences: - eye0: Pupil Cam1/2/3 ID0 - eye1: Pupil Cam1/2/3 ID1 - world: Pupil Cam1 ID2 + def get_cameras(self) -> T.Sequence["SourceInfo"]: + """Return source infos for all cameras that the backend supports.""" + return [] - See issue #1278 for more details. - """ + def activate(self, key: T.Any) -> None: + """Activate a source (device or camera) by key from source info.""" pass - def auto_select_manager(self): - self.notify_all( - {"subject": "backend.auto_select_manager", "name": self.class_name} - ) - def add_auto_select_button(self): - from pyglui import ui +class SourceInfo: + """SourceInfo is a proxy for a source (camera or device) from a manager. - self.menu.append( - ui.Button("Start with default devices", self.auto_select_manager) - ) - - def add_menu(self): - super().add_menu() - from . import manager_classes - from pyglui import ui + Managers hand out source infos that can be activated from other places in the code. + A manager needs to identify a source uniquely by a key. + """ - self.menu_icon.order = 0.1 + def __init__(self, label: str, manager: Base_Manager, key: T.Any): + self.label = label + self.manager = manager + self.key = key - # We add the capture selection menu - manager_classes.sort(key=lambda x: x.gui_name) - self.menu.append( - ui.Selector( - "capture_manager", - setter=self.replace_backend_manager, - getter=lambda: self.__class__, - selection=manager_classes, - labels=[b.gui_name for b in manager_classes], - label="Manager", - ) - ) + def activate(self) -> None: + self.manager.activate(self.key) - # here is where you add all your menu entries. - self.menu.label = "Backend Manager" + def __str__(self) -> str: + return f"{self.label} - {self.manager.class_name}({self.key})" class Playback_Source(Base_Source): @@ -286,7 +354,7 @@ def __init__(self, g_pool, timing="own", *args, **kwargs): most appropriate frame; does not wait on its own None: Simply returns next frame as fast as possible; used for detectors """ - super().__init__(g_pool) + super().__init__(g_pool, *args, **kwargs) assert timing in ( "external", "own", @@ -321,3 +389,6 @@ def wait(self, timestamp): sleep(target_wait_time) self._recent_wait_ts = timestamp self.finished_sleep = monotonic() + + def get_init_dict(self): + return dict(**super().get_init_dict(), timing=self.timing) diff --git a/pupil_src/shared_modules/video_capture/fake_backend.py b/pupil_src/shared_modules/video_capture/fake_backend.py deleted file mode 100644 index cdf6a3149c..0000000000 --- a/pupil_src/shared_modules/video_capture/fake_backend.py +++ /dev/null @@ -1,384 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2020 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" -from .base_backend import Base_Source, Playback_Source, Base_Manager, EndofVideoError - -import os -import cv2 -import numpy as np -from time import time, sleep -from pyglui import ui -from camera_models import Dummy_Camera -from file_methods import load_object - -# logging -import logging - -logger = logging.getLogger(__name__) - - -class Frame(object): - """docstring of Frame""" - - def __init__(self, timestamp, img, index): - self.timestamp = timestamp - self._img = img - self.bgr = img - self.height, self.width, _ = img.shape - self._gray = None - self.index = index - # indicate that the frame does not have a native yuv or jpeg buffer - self.yuv_buffer = None - self.jpeg_buffer = None - - @property - def img(self): - return self._img - - @property - def gray(self): - if self._gray is None: - self._gray = cv2.cvtColor(self._img, cv2.COLOR_BGR2GRAY) - return self._gray - - def copy(self): - return Frame(self.timestamp, self._img.copy(), self.index) - - -class Fake_Source(Playback_Source, Base_Source): - """Simple source which shows random, static image. - - It is used as falback in case the original source fails. `preferred_source` - contains the necessary information to recover to the original source if - it becomes accessible again. - - Attributes: - current_frame_idx (int): Sequence counter - frame_rate (int) - frame_size (tuple) - """ - - def __init__( - self, - g_pool, - source_path=None, - frame_size=None, - frame_rate=None, - name="Fake Source", - *args, - **kwargs - ): - super().__init__(g_pool, *args, **kwargs) - if self.timing == "external": - self.recent_events = self.recent_events_external_timing - else: - self.recent_events = self.recent_events_own_timing - - if source_path: - meta = load_object(source_path) - frame_size = meta["frame_size"] - frame_rate = meta["frame_rate"] - self.timestamps = np.load( - os.path.splitext(source_path)[0] + "_timestamps.npy" - ) - else: - self.timestamps = None - - self.fps = frame_rate - self._name = name - self.make_img(tuple(frame_size)) - self.source_path = source_path - self.current_frame_idx = 0 - self.target_frame_idx = 0 - - def init_ui(self): - self.add_menu() - self.menu.label = "Static Image Source" - - text = ui.Info_Text("This plugin displays a static image.") - self.menu.append(text) - - self.menu.append( - ui.Text_Input( - "frame_size", - label="Frame size", - setter=lambda x: None, - getter=lambda: "{} x {}".format(*self.frame_size), - ) - ) - - self.menu.append( - ui.Text_Input( - "frame_rate", - label="Frame rate", - setter=lambda x: None, - getter=lambda: "{:.0f} FPS".format(self.frame_rate), - ) - ) - - if self.g_pool.app == "player": - # get_frame_count() is not constant in Capture and would trigger - # a redraw on each frame - self.menu.append( - ui.Text_Input( - "frame_num", - label="Number of frames", - setter=lambda x: None, - getter=lambda: self.get_frame_count(), - ) - ) - - def deinit_ui(self): - self.remove_menu() - - def make_img(self, size): - # Generate Pupil Labs colored gradient - self._img = np.zeros((size[1], size[0], 3), dtype=np.uint8) - self._img[:, :, 0] += np.linspace(91, 157, self.frame_size[0], dtype=np.uint8) - self._img[:, :, 1] += np.linspace(165, 161, self.frame_size[0], dtype=np.uint8) - self._img[:, :, 2] += np.linspace(35, 112, self.frame_size[0], dtype=np.uint8) - - self._intrinsics = Dummy_Camera(size, self.name) - - def recent_events_external_timing(self, events): - try: - last_index = self._recent_frame.index - except AttributeError: - # called once on start when self._recent_frame is None - last_index = -1 - - frame = None - pbt = self.g_pool.seek_control.current_playback_time - ts_idx = self.g_pool.seek_control.ts_idx_from_playback_time(pbt) - if ts_idx == last_index: - frame = self._recent_frame.copy() - if self.play and ts_idx == self.get_frame_count() - 1: - logger.info("Recording has ended.") - self.g_pool.seek_control.play = False - elif ts_idx < last_index or ts_idx > last_index + 1: - # time to seek - self.seek_to_frame(ts_idx) - - # Only call get_frame() if the next frame is actually needed - frame = frame or self.get_frame() - self._recent_frame = frame - events["frame"] = frame - self.g_pool.seek_control.end_of_seek() - - def recent_events_own_timing(self, events): - try: - frame = self.get_frame() - except IndexError: - logger.info("Recording has ended.") - self.play = False - else: - if self.timing: - self.wait(frame.timestamp) - self._recent_frame = frame - events["frame"] = frame - - def get_frame(self): - try: - timestamp = self.timestamps[self.target_frame_idx] - except IndexError: - raise EndofVideoError("Reached end of timestamps list.") - except TypeError: - timestamp = self._recent_wait_ts + 1 / self.fps - - frame = Frame(timestamp, self._img.copy(), self.target_frame_idx) - - frame_txt_font_name = cv2.FONT_HERSHEY_SIMPLEX - frame_txt_font_scale = 1.0 - frame_txt_thickness = 1 - - # first line: frame index - frame_txt = "Fake source frame {}".format(frame.index) - frame_txt_size = cv2.getTextSize( - frame_txt, frame_txt_font_name, frame_txt_font_scale, frame_txt_thickness - )[0] - - frame_txt_loc = ( - self.frame_size[0] // 2 - frame_txt_size[0] // 2, - self.frame_size[1] // 2 - frame_txt_size[1], - ) - - cv2.putText( - frame.img, - frame_txt, - frame_txt_loc, - frame_txt_font_name, - frame_txt_font_scale, - (255, 255, 255), - thickness=frame_txt_thickness, - lineType=cv2.LINE_8, - ) - - # second line: resolution @ fps - frame_txt = "{}x{} @ {} fps".format(*self.frame_size, self.frame_rate) - frame_txt_size = cv2.getTextSize( - frame_txt, frame_txt_font_name, frame_txt_font_scale, frame_txt_thickness - )[0] - - frame_txt_loc = ( - self.frame_size[0] // 2 - frame_txt_size[0] // 2, - self.frame_size[1] // 2 + frame_txt_size[1], - ) - - cv2.putText( - frame.img, - frame_txt, - frame_txt_loc, - frame_txt_font_name, - frame_txt_font_scale, - (255, 255, 255), - thickness=frame_txt_thickness, - lineType=cv2.LINE_8, - ) - - self.current_frame_idx = self.target_frame_idx - self.target_frame_idx += 1 - - return frame - - def get_frame_count(self): - try: - return len(self.timestamps) - except TypeError: - return self.current_frame_idx + 1 - - def seek_to_frame(self, frame_idx): - self.target_frame_idx = max(0, min(frame_idx, self.get_frame_count() - 1)) - self.finished_sleep = 0 - - def get_frame_index(self): - return self.current_frame_idx - - @property - def name(self): - return self._name - - @property - def settings(self): - return {"frame_size": self.frame_size, "frame_rate": self.frame_rate} - - @settings.setter - def settings(self, settings): - self.frame_size = settings.get("frame_size", self.frame_size) - self.frame_rate = settings.get("frame_rate", self.frame_rate) - - @property - def frame_size(self): - return self._img.shape[1], self._img.shape[0] - - @frame_size.setter - def frame_size(self, new_size): - # closest match for size - sizes = [abs(r[0] - new_size[0]) for r in self.frame_sizes] - best_size_idx = sizes.index(min(sizes)) - size = self.frame_sizes[best_size_idx] - if size != new_size: - logger.warning( - "%s resolution capture mode not available. Selected %s." - % (new_size, size) - ) - self.make_img(size) - - @property - def frame_rates(self): - return (30, 60, 90, 120) - - @property - def frame_sizes(self): - return ((640, 480), (1280, 720), (1920, 1080)) - - @property - def frame_rate(self): - return self.fps - - @frame_rate.setter - def frame_rate(self, new_rate): - rates = [abs(r - new_rate) for r in self.frame_rates] - best_rate_idx = rates.index(min(rates)) - rate = self.frame_rates[best_rate_idx] - if rate != new_rate: - logger.warning( - "%sfps capture mode not available at (%s) on 'Fake Source'. Selected %sfps. " - % (new_rate, self.frame_size, rate) - ) - self.fps = rate - - @property - def jpeg_support(self): - return False - - @property - def online(self): - return True - - def get_init_dict(self): - if self.g_pool.app == "capture": - d = super().get_init_dict() - d["frame_size"] = self.frame_size - d["frame_rate"] = self.frame_rate - d["name"] = self.name - return d - else: - raise NotImplementedError() - - -class Fake_Manager(Base_Manager): - """Simple manager to explicitly activate a fake source""" - - gui_name = "Test image" - - def __init__(self, g_pool): - super().__init__(g_pool) - - def init_ui(self): - self.add_menu() - from pyglui import ui - - self.add_auto_select_button() - text = ui.Info_Text("Convenience manager to select a fake source explicitly.") - - activation_button = ui.Button("Activate Fake Capture", self.activate) - self.menu.extend([text, activation_button]) - - def activate(self): - # a capture leaving is a must stop for recording. - self.notify_all({"subject": "recording.should_stop"}) - settings = {} - settings["timing"] = "own" - settings["frame_rate"] = self.g_pool.capture.frame_rate - settings["frame_size"] = self.g_pool.capture.frame_size - settings["name"] = "Fake Source" - # if the user set fake capture, we dont want it to auto jump back to the old capture. - if self.g_pool.process == "world": - self.notify_all( - {"subject": "start_plugin", "name": "Fake_Source", "args": settings} - ) - else: - self.notify_all( - { - "subject": "start_eye_plugin", - "target": self.g_pool.process, - "name": "Fake_Source", - "args": settings, - } - ) - - def auto_activate_source(self): - self.activate() - - def deinit_ui(self): - self.remove_menu() - - def recent_events(self, events): - pass diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index eec7363435..d20cab7dbc 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -13,22 +13,22 @@ import logging import os import os.path -import av -import numpy as np import typing as T - -from multiprocessing import cpu_count from abc import ABC, abstractmethod +from multiprocessing import cpu_count from time import sleep + +import av +import numpy as np +from pyglui import ui + from camera_models import load_intrinsics -from .utils import VideoSet +from pupil_recording import PupilRecording -import player_methods as pm from .base_backend import Base_Manager, Base_Source, EndofVideoError, Playback_Source -from pupil_recording import PupilRecording +from .utils import VideoSet logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) av.logging.set_level(av.logging.ERROR) logging.getLogger("libav").setLevel(logging.ERROR) @@ -190,9 +190,14 @@ def get_frame_iterator(self): yield frame +# NOTE:Base_Source is included as base class for uniqueness:by_base_class to work +# correctly with other Source plugins. class File_Source(Playback_Source, Base_Source): """Simple file capture. + Note that File_Source is special since it is usually not intended to be used as + capture plugin. Therefore it hides it's UI by default. + Playback_Source arguments: timing (str): "external", "own" (default), None @@ -201,6 +206,7 @@ class File_Source(Playback_Source, Base_Source): loop (bool): loop video set if timing!="external" buffered_decoding (bool): use buffered decode fill_gaps (bool): fill gaps with static frames + show_plugin_menu (bool): enable to show regular capture UI with source selection """ def __init__( @@ -210,6 +216,7 @@ def __init__( loop=False, buffered_decoding=False, fill_gaps=False, + show_plugin_menu=False, *args, **kwargs, ): @@ -239,6 +246,8 @@ def __init__( self.reset_video() self._intrinsics = load_intrinsics(rec, set_name, self.frame_size) + self.show_plugin_menu = show_plugin_menu + def get_rec_set_name(self, source_path): """ Return dir and set name by source_path @@ -324,6 +333,10 @@ def _get_streams(self, container, should_buffer): def initialised(self): return not self.videoset.is_empty() + @property + def online(self): + return not self.videoset.is_empty() + @property def frame_size(self): return self.video_stream.frame_size @@ -332,21 +345,30 @@ def frame_size(self): def frame_rate(self): return self._frame_rate + def init_ui(self): + if self.show_plugin_menu: + super().init_ui() + + def deinit_ui(self): + if self.show_plugin_menu: + super().deinit_ui() + def get_init_dict(self): - if self.g_pool.app == "capture": - settings = super().get_init_dict() - settings["source_path"] = self.source_path - settings["loop"] = self.loop - return settings - else: - raise NotImplementedError() + return dict( + **super().get_init_dict(), + source_path=self.source_path, + loop=self.loop, + buffered_decoding=self.buffering, + fill_gaps=self.fill_gaps, + show_plugin_menu=self.show_plugin_menu, + ) @property def name(self): if self.source_path: return os.path.splitext(self.source_path)[0] else: - return "File source in ghost mode" + return "File source (no file loaded)" def get_frame_index(self): return int(self.current_frame_idx) @@ -493,6 +515,7 @@ def seek_to_frame(self, seek_pos): self.target_frame_idx = seek_pos def on_notify(self, notification): + super().on_notify(notification) if ( notification["subject"] == "file_source.seek" and notification.get("source_path") == self.source_path @@ -509,32 +532,33 @@ def on_notify(self, notification): ): self.play = False - def init_ui(self): - self.add_menu() - self.menu.label = "File Source: {}".format(os.path.split(self.source_path)[-1]) - from pyglui import ui - - self.menu.append( - ui.Info_Text( - "The file source plugin loads and " - + "displays video from a given file." - ) + def ui_elements(self): + ui_elements = [] + ui_elements.append( + ui.Info_Text(f"File Source: {os.path.split(self.source_path)[-1]}") ) - if self.g_pool.app == "capture": + if not self.online: + ui_elements.append( + ui.Info_Text( + "Could not playback file! Check if file exists and if" + " corresponding timestamps file is present." + ) + ) + return ui_elements - def toggle_looping(val): - self.loop = val - if val: - self.play = True + def toggle_looping(val): + self.loop = val + if val: + self.play = True - self.menu.append(ui.Switch("loop", self, setter=toggle_looping)) + ui_elements.append(ui.Switch("loop", self, setter=toggle_looping)) - self.menu.append( + ui_elements.append( ui.Text_Input("source_path", self, label="Full path", setter=lambda x: None) ) - self.menu.append( + ui_elements.append( ui.Text_Input( "frame_size", label="Frame size", @@ -543,7 +567,7 @@ def toggle_looping(val): ) ) - self.menu.append( + ui_elements.append( ui.Text_Input( "frame_rate", label="Frame rate", @@ -552,7 +576,7 @@ def toggle_looping(val): ) ) - self.menu.append( + ui_elements.append( ui.Text_Input( "frame_num", label="Number of frames", @@ -560,9 +584,7 @@ def toggle_looping(val): getter=lambda: self.get_frame_count(), ) ) - - def deinit_ui(self): - self.remove_menu() + return ui_elements def cleanup(self): try: @@ -593,73 +615,10 @@ class File_Manager(Base_Manager): Attributes: file_exts (list): File extensions to filter displayed files - root_folder (str): Folder path, which includes file sources """ - gui_name = "Video File Source" file_exts = [".mp4", ".mkv", ".mov", ".mjpeg"] - def __init__(self, g_pool, root_folder=None): - super().__init__(g_pool) - base_dir = self.g_pool.user_dir.rsplit(os.path.sep, 1)[0] - default_rec_dir = os.path.join(base_dir, "recordings") - self.root_folder = root_folder or default_rec_dir - - def init_ui(self): - self.add_menu() - from pyglui import ui - - self.add_auto_select_button() - self.menu.append( - ui.Info_Text( - "Enter a folder to enumerate all eligible video files. " - + "Be aware that entering folders with a lot of files can " - + "slow down Pupil Capture." - ) - ) - - def set_root(folder): - if not os.path.isdir(folder): - logger.error("`%s` is not a valid folder path." % folder) - else: - self.root_folder = folder - - self.menu.append( - ui.Text_Input("root_folder", self, label="Source Folder", setter=set_root) - ) - - def split_enumeration(): - eligible_files = self.enumerate_folder(self.root_folder) - eligible_files.insert(0, (None, "Select to activate")) - return zip(*eligible_files) - - self.menu.append( - # TODO: potential race condition through selection_getter. Should ensure - # that current selection will always be present in the list returned by the - # selection_getter. Highly unlikely though as this needs to happen between - # having clicked the Selector and the next redraw. - # See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b - ui.Selector( - "selected_file", - selection_getter=split_enumeration, - getter=lambda: None, - setter=self.activate, - label="Video File", - ) - ) - - def deinit_ui(self): - self.remove_menu() - - def activate(self, full_path): - if not full_path: - return - settings = {"source_path": full_path, "timing": "own"} - self.activate_source(settings) - - def auto_activate_source(self): - self.activate(None) - def on_drop(self, paths): for p in paths: if os.path.splitext(p)[-1] in self.file_exts: @@ -667,25 +626,15 @@ def on_drop(self, paths): return True return False - def enumerate_folder(self, path): - eligible_files = [] - is_eligible = lambda f: os.path.splitext(f)[-1] in self.file_exts - path = os.path.abspath(os.path.expanduser(path)) - for root, dirs, files in os.walk(path): - - def root_split(file): - full_p = os.path.join(root, file) - disp_p = full_p.replace(path, "") - return (full_p, disp_p) - - eligible_files.extend(map(root_split, filter(is_eligible, files))) - eligible_files.sort(key=lambda x: x[1]) - return eligible_files - - def get_init_dict(self): - return {"root_folder": self.root_folder} + def activate(self, full_path): + if not full_path: + return - def activate_source(self, settings={}): + settings = { + "source_path": full_path, + "timing": "own", + "show_plugin_menu": True, + } if self.g_pool.process == "world": self.notify_all( {"subject": "start_plugin", "name": "File_Source", "args": settings} @@ -699,6 +648,3 @@ def activate_source(self, settings={}): "args": settings, } ) - - def recent_events(self, events): - pass diff --git a/pupil_src/shared_modules/video_capture/hmd_streaming.py b/pupil_src/shared_modules/video_capture/hmd_streaming.py index 6648f0d60b..809850bc02 100644 --- a/pupil_src/shared_modules/video_capture/hmd_streaming.py +++ b/pupil_src/shared_modules/video_capture/hmd_streaming.py @@ -15,8 +15,8 @@ from pyglui import ui import zmq_tools -from camera_models import Radial_Dist_Camera, Dummy_Camera -from video_capture.base_backend import Base_Manager, Base_Source +from camera_models import Dummy_Camera, Radial_Dist_Camera +from video_capture.base_backend import Base_Source logger = logging.getLogger(__name__) @@ -52,17 +52,6 @@ def __init__(self, g_pool, *args, **kwargs): topics=("hmd_streaming.world",), ) - # def get_init_dict(self): - - def init_ui(self): # was gui - self.add_menu() - self.menu.label = "HMD Streaming" - text = ui.Info_Text("HMD Streaming Info") - self.menu.append(text) - - def deinit_ui(self): - self.remove_menu() - def cleanup(self): self.frame_sub = None @@ -139,42 +128,7 @@ def intrinsics(self, model): "HMD Streaming backend does not support setting intrinsics manually" ) - -class HMD_Streaming_Manager(Base_Manager): - """Simple manager to explicitly activate a fake source""" - - gui_name = "HMD Streaming" - - def __init__(self, g_pool): - super().__init__(g_pool) - - # Initiates the UI for starting the webcam. - def init_ui(self): - self.add_menu() - from pyglui import ui - - self.menu.append(ui.Info_Text("Backend for HMD Streaming")) - self.menu.append(ui.Button("Activate HMD Streaming", self.activate_source)) - - def activate_source(self): - settings = {} - # if the user set fake capture, we dont want it to auto jump back to the old capture. - if self.g_pool.process == "world": - self.notify_all( - { - "subject": "start_plugin", - "name": "HMD_Streaming_Source", - "args": settings, - } - ) - else: - logger.warning("HMD Streaming backend is not supported in the eye process.") - - def deinit_ui(self): - self.remove_menu() - - def recent_events(self, events): - pass - - def get_init_dict(self): - return {} + def ui_elements(self): + ui_elements = [] + ui_elements.append(ui.Info_Text(f"HMD Streaming")) + return ui_elements diff --git a/pupil_src/shared_modules/video_capture/ndsi_backend.py b/pupil_src/shared_modules/video_capture/ndsi_backend.py index 886ec48e33..3d8f839c05 100644 --- a/pupil_src/shared_modules/video_capture/ndsi_backend.py +++ b/pupil_src/shared_modules/video_capture/ndsi_backend.py @@ -9,15 +9,17 @@ ---------------------------------------------------------------------------~(*) """ -import time import logging -from packaging.version import Version +import time import ndsi +from packaging.version import Version +from pyglui import ui -from .base_backend import Base_Source, Base_Manager from camera_models import load_intrinsics +from .base_backend import Base_Manager, Base_Source, SourceInfo + try: from ndsi import __version__ @@ -25,8 +27,15 @@ from ndsi import __protocol_version__ except (ImportError, AssertionError): raise Exception("pyndsi version is to old. Please upgrade") from None + + logger = logging.getLogger(__name__) +# Suppress pyre debug logs (except beacon) +logger.debug("Suppressing pyre debug logs (except zbeacon)") +logging.getLogger("pyre").setLevel(logging.WARNING) +logging.getLogger("pyre.zbeacon").setLevel(logging.DEBUG) + class NDSI_Source(Base_Source): """Pupil Mobile video source @@ -41,42 +50,45 @@ def __init__( g_pool, frame_size, frame_rate, - network=None, source_id=None, host_name=None, sensor_name=None, + *args, + **kwargs, ): - super().__init__(g_pool) + super().__init__(g_pool, *args, **kwargs) self.sensor = None self._source_id = source_id self._sensor_name = sensor_name self._host_name = host_name self._frame_size = frame_size self._frame_rate = frame_rate - self.has_ui = False + self.ui_initialized = False self.control_id_ui_mapping = {} self.get_frame_timeout = 100 # ms self.ghost_mode_timeout = 10 # sec self._initial_refresh = True self.last_update = self.g_pool.get_timestamp() + manager = next((p for p in g_pool.plugins if isinstance(p, NDSI_Manager)), None) + if not manager: + logger.error("Error connecting to Pupil Mobile: NDSI Manager not found!") + return + + network = manager.network if not network: - logger.debug( - "No network reference provided. Capture is started " - + "in ghost mode. No images will be supplied." - ) + logger.error("Error connecting to Pupil Mobile: No NDSI network!") return self.recover(network) if not self.sensor or not self.sensor.supports_data_subscription: - logger.error( - "Init failed. Capture is started in ghost mode. " - + "No images will be supplied." - ) + logger.error("Could not connect to device! No images will be supplied.") self.cleanup() - logger.debug("NDSI Source Sensor: %s" % self.sensor) + logger.warning( + "Make sure to enable the Time_Sync plugin for recording with Pupil Mobile!" + ) def recover(self, network): logger.debug( @@ -142,6 +154,13 @@ def recent_events(self, events): ) except ndsi.StreamError: frame = None + except ndsi.sensor.NotDataSubSupportedError: + # NOTE: This (most likely) is a race-condition in NDSI initialization + # that is waiting to be fixed for Pupil Mobile. It happens rarely and + # can be solved by simply reconnecting the headset to the mobile phone. + # Preventing traceback logfloods here and displaying more helpful + # message to the user. + logger.warning("Connection problem! Please reconnect headset to phone!") except Exception: frame = None import traceback @@ -156,13 +175,13 @@ def recent_events(self, events): elif ( self.g_pool.get_timestamp() - self.last_update > self.ghost_mode_timeout ): - logger.info("Entering ghost mode") + logger.info("Device disconnected.") if self.online: self.sensor.unlink() self.sensor = None self._source_id = None self._initial_refresh = True - self.update_control_menu() + self.update_menu() self.last_update = self.g_pool.get_timestamp() else: time.sleep(self.get_frame_timeout / 1e3) @@ -179,15 +198,16 @@ def on_notification(self, sensor, event): logger.warning("Error {}".format(event["error_str"])) if "control_id" in event and event["control_id"] in self.sensor.controls: logger.debug(str(self.sensor.controls[event["control_id"]])) - elif self.has_ui and ( + elif self.ui_initialized and ( event["control_id"] not in self.control_id_ui_mapping or event["changes"].get("dtype") == "strmapping" or event["changes"].get("dtype") == "intmapping" ): - self.update_control_menu() + self.update_menu() # local notifications def on_notify(self, notification): + super().on_notify(notification) subject = notification["subject"] if subject.startswith("remote_recording.") and self.online: if "should_start" in subject and self.online: @@ -246,21 +266,12 @@ def get_init_dict(self): return settings def init_ui(self): - self.add_menu() - self.menu.label = "NDSI Source: {} @ {}".format( - self._sensor_name, self._host_name - ) - - from pyglui import ui - - self.has_ui = True - self.uvc_menu = ui.Growing_Menu("UVC Controls") - self.update_control_menu() + super().init_ui() + self.ui_initialized = True def deinit_ui(self): - self.uvc_menu = None - self.remove_menu() - self.has_ui = False + self.ui_initialized = False + super().deinit_ui() def add_controls_to_menu(self, menu, controls): from pyglui import ui @@ -327,26 +338,18 @@ def initiate_value_change(val): import traceback as tb tb.print_exc() - if len(menu) == 0: - menu.append(ui.Info_Text("No {} settings found".format(menu.label))) return menu - def update_control_menu(self): - if not self.has_ui: - return - from pyglui import ui + def ui_elements(self): + + ui_elements = [] + ui_elements.append( + ui.Info_Text(f"Camera: {self._sensor_name} @ {self._host_name}") + ) - del self.menu[:] - del self.uvc_menu[:] - self.control_id_ui_mapping = {} if not self.sensor: - self.menu.append( - ui.Info_Text( - ("Sensor %s @ %s not available. " + "Running in ghost mode.") - % (self._sensor_name, self._host_name) - ) - ) - return + ui_elements.append(ui.Info_Text("Camera disconnected!")) + return ui_elements uvc_controls = [] other_controls = [] @@ -356,14 +359,22 @@ def update_control_menu(self): else: other_controls.append(entry) - self.add_controls_to_menu(self.menu, other_controls) - self.add_controls_to_menu(self.uvc_menu, uvc_controls) - self.menu.append(self.uvc_menu) + uvc_menu = ui.Growing_Menu("UVC Controls") + self.control_id_ui_mapping = {} + if other_controls: + self.add_controls_to_menu(ui_elements, other_controls) + if uvc_controls: + self.add_controls_to_menu(uvc_menu, uvc_controls) + else: + uvc_menu.append(ui.Info_Text("No UVC settings found.")) + ui_elements.append(uvc_menu) - self.menu.append( + ui_elements.append( ui.Button("Reset to default values", self.sensor.reset_all_control_values) ) + return ui_elements + def cleanup(self): if self.online: self.sensor.unlink() @@ -371,115 +382,58 @@ def cleanup(self): class NDSI_Manager(Base_Manager): - """Enumerates and activates Pupil Mobile video sources - - Attributes: - network (ndsi.Network): NDSI Network backend - selected_host (unicode): Selected host uuid - """ - - gui_name = "Pupil Mobile" + """Enumerates and activates NDSI video sources""" def __init__(self, g_pool): super().__init__(g_pool) self.network = ndsi.Network( - formats={ndsi.DataFormat.V3, ndsi.DataFormat.V4}, - callbacks=(self.on_event,) + formats={ndsi.DataFormat.V3, ndsi.DataFormat.V4}, callbacks=(self.on_event,) ) self.network.start() - self.selected_host = None self._recover_in = 3 self._rejoin_in = 400 - self.should_select_host = None self.cam_selection_lut = { "eye0": ["ID0", "PI right"], "eye1": ["ID1", "PI left"], "world": ["ID2", "Logitech", "PI world"], } - logger.warning("Make sure the `time_sync` plugin is loaded!") def cleanup(self): self.network.stop() - def init_ui(self): - self.add_menu() - self.re_build_ndsi_menu() - - def deinit_ui(self): - self.remove_menu() - - def view_host(self, host_uuid): - if self.selected_host != host_uuid: - self.selected_host = host_uuid - self.re_build_ndsi_menu() - - def host_selection_list(self): - devices = { - s["host_uuid"]: s["host_name"] # removes duplicates + def get_devices(self): + # store hosts in dict to remove duplicates from multiple sensors + active_hosts = { + s["host_uuid"]: s["host_name"] for s in self.network.sensors.values() + if s["sensor_type"] == "video" } - - if devices: - return list(devices.keys()), list(devices.values()) - else: - return [None], ["No hosts found"] - - def source_selection_list(self): - default = (None, "Select to activate") - sources = [default] + [ - (s["sensor_uuid"], s["sensor_name"]) - for s in self.network.sensors.values() - if (s["sensor_type"] == "video" and s["host_uuid"] == self.selected_host) + return [ + SourceInfo(label=host_name, manager=self, key=f"host.{host_uuid}") + for host_uuid, host_name in active_hosts.items() ] - return zip(*sources) - - def re_build_ndsi_menu(self): - del self.menu[1:] - from pyglui import ui - ui_elements = [] - ui_elements.append(ui.Info_Text("Remote Pupil Mobile sources")) - ui_elements.append( - ui.Info_Text("Pupil Mobile Commspec v{}".format(__protocol_version__)) - ) - - host_sel, host_sel_labels = self.host_selection_list() - ui_elements.append( - ui.Selector( - "selected_host", - self, - selection=host_sel, - labels=host_sel_labels, - setter=self.view_host, - label="Remote host", + def get_cameras(self): + return [ + SourceInfo( + label=f"{s['sensor_name']} @ PM {s['host_name']}", + manager=self, + key=f"sensor.{s['sensor_uuid']}", ) - ) - - self.menu.extend(ui_elements) - self.add_auto_select_button() - - if not self.selected_host: - return - ui_elements = [] + for s in self.network.sensors.values() + if s["sensor_type"] == "video" + ] - host_menu = ui.Growing_Menu("Remote Host Information") - ui_elements.append(host_menu) - - src_sel, src_sel_labels = self.source_selection_list() - host_menu.append( - ui.Selector( - "selected_source", - selection=src_sel, - labels=src_sel_labels, - getter=lambda: None, - setter=self.activate, - label="Source", + def activate(self, key): + source_type, uid = key.split(".", maxsplit=1) + if source_type == "host": + self.notify_all( + {"subject": "backend.ndsi.auto_activate_source", "host_uid": uid} ) - ) + elif source_type == "sensor": + self.activate_source(source_uid=uid) - self.menu.extend(ui_elements) - - def activate(self, source_uid): + def activate_source(self, source_uid): if not source_uid: return settings = { @@ -487,40 +441,44 @@ def activate(self, source_uid): "frame_rate": self.g_pool.capture.frame_rate, "source_id": source_uid, } - self.activate_source(settings) - - def auto_select_manager(self): - super().auto_select_manager() - self.notify_all( - { - "subject": "backend.ndsi_do_select_host", - "target_host": self.selected_host, - "delay": 0.4, - } - ) + if self.g_pool.process == "world": + self.notify_all( + {"subject": "start_plugin", "name": "NDSI_Source", "args": settings} + ) + else: + self.notify_all( + { + "subject": "start_eye_plugin", + "target": self.g_pool.process, + "name": "NDSI_Source", + "args": settings, + } + ) - def auto_activate_source(self): - if not self.selected_host: - return + def auto_activate_source(self, host_uid): + host_sensors = [ + sensor + for sensor in self.network.sensors.values() + if (sensor["sensor_type"] == "video" and sensor["host_uuid"] == host_uid) + ] - src_sel, src_sel_labels = self.source_selection_list() - if len(src_sel) < 2: # "Select to Activate" is always presenet as first element - logger.warning("No device is available on the remote host.") + if not host_sensors: + logger.warning("No devices available on the remote host.") return - cam_ids = self.cam_selection_lut[self.g_pool.process] + name_patterns = self.cam_selection_lut[self.g_pool.process] + matching_cams = [ + sensor + for sensor in host_sensors + if any(pattern in sensor["sensor_name"] for pattern in name_patterns) + ] - for cam_id in cam_ids: - try: - source_id = next( - src_sel[i] for i, lab in enumerate(src_sel_labels) if cam_id in lab - ) - self.activate(source_id) - break - except StopIteration: - source_id = None - else: + if not matching_cams: logger.warning("The default device was not found on the remote host.") + return + + cam = matching_cams[0] + self.activate_source(cam["sensor_uuid"]) def poll_events(self): while self.network.has_events: @@ -542,6 +500,7 @@ def recent_events(self, events): if self._rejoin_in <= 0: logger.debug("Rejoining network...") self.network.rejoin() + # frame-timeout independent timer self._rejoin_in = int(10 * 1e3 / self.g_pool.capture.get_frame_timeout) else: self._rejoin_in -= 1 @@ -550,42 +509,27 @@ def on_event(self, caller, event): if event["subject"] == "detach": logger.debug("detached: %s" % event) sensors = [s for s in self.network.sensors.values()] - if self.selected_host == event["host_uuid"]: - if sensors: - self.selected_host = sensors[0]["host_uuid"] - else: - self.selected_host = None - self.re_build_ndsi_menu() elif event["subject"] == "attach": if event["sensor_type"] == "video": logger.debug("attached: {}".format(event)) self.notify_all({"subject": "backend.ndsi_source_found"}) - if not self.selected_host and not self.should_select_host: - self.selected_host = event["host_uuid"] - elif self.should_select_host and event["sensor_type"] == "video": - self.select_host(self.should_select_host) - - self.re_build_ndsi_menu() - - def activate_source(self, settings={}): - settings["network"] = self.network - self.g_pool.plugins.add(NDSI_Source, args=settings) - def recover(self): - self.g_pool.capture.recover(self.network) + if isinstance(self.g_pool.capture, NDSI_Source): + self.g_pool.capture.recover(self.network) def on_notify(self, n): - """Provides UI for the capture selection + """Starts appropriate NDSI sources. Reacts to notification: ``backend.ndsi_source_found``: Check if recovery is possible - ``backend.ndsi_do_select_host``: Switches to selected host from other process + ``backend.ndsi.auto_activate_source``: Auto activate best source for process Emmits notifications: - ``backend.ndsi_source_found`` - ``backend.ndsi_do_select_host` + ``backend.ndsi_source_found``: New NDSI source available + ``backend.ndsi.auto_activate_source``: All NDSI managers should auto activate a source + ``start_(eye_)plugin``: Starts NDSI sources """ super().on_notify(n) @@ -597,20 +541,5 @@ def on_notify(self, n): ): self.recover() - if n["subject"].startswith("backend.ndsi_do_select_host"): - self.select_host(n["target_host"]) - - def select_host(self, selected_host): - host_sel, _ = self.host_selection_list() - if selected_host in host_sel: - self.view_host(selected_host) - self.should_select_host = None - self.re_build_ndsi_menu() - src_sel, _ = self.source_selection_list() - # "Select to Activate" is always presenet as first element - if len(src_sel) >= 2: - self.auto_activate_source() - - else: - self.should_select_host = selected_host - + if n["subject"] == "backend.ndsi.auto_activate_source": + self.auto_activate_source(n["host_uid"]) diff --git a/pupil_src/shared_modules/video_capture/realsense2_backend.py b/pupil_src/shared_modules/video_capture/realsense2_backend.py deleted file mode 100755 index 171b38534c..0000000000 --- a/pupil_src/shared_modules/video_capture/realsense2_backend.py +++ /dev/null @@ -1,938 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2020 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" - -import logging -import time -import cv2 -import os - -import pyrealsense2 as rs - -from version_utils import VersionFormat -from .base_backend import Base_Source, Base_Manager -from av_writer import MPEG_Writer -from camera_models import load_intrinsics - -import glfw -import gl_utils -from OpenGL.GL import * -from OpenGL.GLU import * -from pyglui import cygl -import cython_methods -import numpy as np -from ctypes import * - -# check versions for our own depedencies as they are fast-changing -# assert VersionFormat(rs.__version__) >= VersionFormat("2.2") # FIXME - -# logging -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -TIMEOUT = 500 # ms FIXME -DEFAULT_COLOR_SIZE = (1280, 720) -DEFAULT_COLOR_FPS = 30 -DEFAULT_DEPTH_SIZE = (640, 480) -DEFAULT_DEPTH_FPS = 30 - -# very thin wrapper for rs.frame objects -class ColorFrame(object): - def __init__(self, data, timestamp, index): - self.timestamp = timestamp - self.index = index - - self.data = data[:, :, np.newaxis].view(dtype=np.uint8) - total_size = self.data.size - y_plane = total_size // 2 - u_plane = y_plane // 2 - self._yuv = np.empty(total_size, dtype=np.uint8) - self._yuv[:y_plane] = self.data[:, :, 0].ravel() - self._yuv[y_plane : y_plane + u_plane] = self.data[:, ::2, 1].ravel() - self._yuv[y_plane + u_plane :] = self.data[:, 1::2, 1].ravel() - self._shape = self.data.shape[:2] - - self._bgr = None - self._gray = None - - @property - def height(self): - return self._shape[0] - - @property - def width(self): - return self._shape[1] - - @property - def yuv_buffer(self): - return self._yuv - - @property - def yuv422(self): - Y = self._yuv[: self._yuv.size // 2] - U = self._yuv[self._yuv.size // 2 : 3 * self._yuv.size // 4] - V = self._yuv[3 * self._yuv.size // 4 :] - - Y.shape = self._shape - U.shape = self._shape[0], self._shape[1] // 2 - V.shape = self._shape[0], self._shape[1] // 2 - - return Y, U, V - - @property - def bgr(self): - if self._bgr is None: - self._bgr = cv2.cvtColor(self.data, cv2.COLOR_YUV2BGR_YUYV) - return self._bgr - - @property - def img(self): - return self.bgr - - @property - def gray(self): - if self._gray is None: - self._gray = self._yuv[: self._yuv.size // 2] - self._gray.shape = self._shape - return self._gray - - -class DepthFrame(object): - def __init__(self, data, timestamp, index): - self.timestamp = timestamp - self.index = index - - self._bgr = None - self._gray = None - self.depth = data - self.yuv_buffer = None - - @property - def height(self): - return self.depth.shape[0] - - @property - def width(self): - return self.depth.shape[1] - - @property - def bgr(self): - if self._bgr is None: - self._bgr = cython_methods.cumhist_color_map16(self.depth) - return self._bgr - - @property - def img(self): - return self.bgr - - @property - def gray(self): - if self._gray is None: - self._gray = cv2.cvtColor(self.bgr, cv2.cv2.COLOR_BGR2GRAY) - return self._gray - - -class Realsense2_Source(Base_Source): - def __init__( - self, - g_pool, - device_id=None, - frame_size=DEFAULT_COLOR_SIZE, - frame_rate=DEFAULT_COLOR_FPS, - depth_frame_size=DEFAULT_DEPTH_SIZE, - depth_frame_rate=DEFAULT_DEPTH_FPS, - preview_depth=False, - device_options=(), - record_depth=True, - ): - logger.debug("_init_ started") - super().__init__(g_pool) - self._intrinsics = None - self.color_frame_index = 0 - self.depth_frame_index = 0 - self.context = rs.context() - self.pipeline = rs.pipeline(self.context) - self.pipeline_profile = None - self.preview_depth = preview_depth - self.record_depth = record_depth - self.depth_video_writer = None - self._needs_restart = False - self.frame_size_backup = DEFAULT_COLOR_SIZE - self.depth_frame_size_backup = DEFAULT_DEPTH_SIZE - self.frame_rate_backup = DEFAULT_COLOR_FPS - self.depth_frame_rate_backup = DEFAULT_DEPTH_FPS - - self._initialize_device( - device_id, - frame_size, - frame_rate, - depth_frame_size, - depth_frame_rate, - device_options, - ) - logger.debug("_init_ completed") - - def _initialize_device( - self, - device_id, - color_frame_size, - color_fps, - depth_frame_size, - depth_fps, - device_options=(), - ): - self.stop_pipeline() - self.last_color_frame_ts = None - self.last_depth_frame_ts = None - self._recent_frame = None - self._recent_depth_frame = None - - if device_id is None: - device_id = self.device_id - - if device_id is None: # FIXME these two if blocks look ugly. - return - - # use default streams to filter modes by rs_stream and rs_format - self._available_modes = self._enumerate_formats(device_id) - logger.debug( - "device_id: {} self._available_modes: {}".format( - device_id, str(self._available_modes) - ) - ) - - if ( - color_frame_size is not None - and depth_frame_size is not None - and color_fps is not None - and depth_fps is not None - ): - color_frame_size = tuple(color_frame_size) - depth_frame_size = tuple(depth_frame_size) - - logger.debug( - "Initialize with Color {}@{}\tDepth {}@{}".format( - color_frame_size, color_fps, depth_frame_size, depth_fps - ) - ) - - # make sure the frame rates are compatible with the given frame sizes - color_fps = self._get_valid_frame_rate( - rs.stream.color, color_frame_size, color_fps - ) - depth_fps = self._get_valid_frame_rate( - rs.stream.depth, depth_frame_size, depth_fps - ) - - self.frame_size_backup = color_frame_size - self.depth_frame_size_backup = depth_frame_size - self.frame_rate_backup = color_fps - self.depth_frame_rate_backup = depth_fps - - config = self._prep_configuration( - color_frame_size, color_fps, depth_frame_size, depth_fps - ) - else: - config = self._get_default_config() - self.frame_size_backup = DEFAULT_COLOR_SIZE - self.depth_frame_size_backup = DEFAULT_DEPTH_SIZE - self.frame_rate_backup = DEFAULT_COLOR_FPS - self.depth_frame_rate_backup = DEFAULT_DEPTH_FPS - - try: - self.pipeline_profile = self.pipeline.start(config) - except RuntimeError as re: - logger.error("Cannot start pipeline! " + str(re)) - self.pipeline_profile = None - else: - self.stream_profiles = { - s.stream_type(): s.as_video_stream_profile() - for s in self.pipeline_profile.get_streams() - } - logger.debug("Pipeline started for device " + device_id) - logger.debug("Stream profiles: " + str(self.stream_profiles)) - - self._intrinsics = load_intrinsics( - self.g_pool.user_dir, self.name, self.frame_size - ) - self.update_menu() - self._needs_restart = False - - def _prep_configuration( - self, - color_frame_size=None, - color_fps=None, - depth_frame_size=None, - depth_fps=None, - ): - config = rs.config() - - # only use these two formats - color_format = rs.format.yuyv - depth_format = rs.format.z16 - - config.enable_stream( - rs.stream.depth, - depth_frame_size[0], - depth_frame_size[1], - depth_format, - depth_fps, - ) - - config.enable_stream( - rs.stream.color, - color_frame_size[0], - color_frame_size[1], - color_format, - color_fps, - ) - - return config - - def _get_default_config(self): - config = rs.config() # default config is RGB8, we want YUYV - config.enable_stream( - rs.stream.color, - DEFAULT_COLOR_SIZE[0], - DEFAULT_COLOR_SIZE[1], - rs.format.yuyv, - DEFAULT_COLOR_FPS, - ) - config.enable_stream( - rs.stream.depth, - DEFAULT_DEPTH_SIZE[0], - DEFAULT_DEPTH_SIZE[1], - rs.format.z16, - DEFAULT_DEPTH_FPS, - ) - return config - - def _get_valid_frame_rate(self, stream_type, frame_size, fps): - assert stream_type == rs.stream.color or stream_type == rs.stream.depth - - if not self._available_modes or stream_type not in self._available_modes: - logger.warning( - "_get_valid_frame_rate: self._available_modes not set yet. Returning default fps." - ) - if stream_type == rs.stream.color: - return DEFAULT_COLOR_FPS - elif stream_type == rs.stream.depth: - return DEFAULT_DEPTH_FPS - else: - raise ValueError("Unexpected `stream_type`: {}".format(stream_type)) - - if frame_size not in self._available_modes[stream_type]: - logger.error( - "Frame size not supported for {}: {}. Returning default fps".format( - stream_type, frame_size - ) - ) - if stream_type == rs.stream.color: - return DEFAULT_COLOR_FPS - elif stream_type == rs.stream.depth: - return DEFAULT_DEPTH_FPS - - if fps not in self._available_modes[stream_type][frame_size]: - old_fps = fps - rates = [ - abs(r - fps) for r in self._available_modes[stream_type][frame_size] - ] - best_rate_idx = rates.index(min(rates)) - fps = self._available_modes[stream_type][frame_size][best_rate_idx] - logger.warning( - "{} fps is not supported for ({}) for Color Stream. Fallback to {} fps".format( - old_fps, frame_size, fps - ) - ) - - return fps - - def _enumerate_formats(self, device_id): - """Enumerate formats into hierachical structure: - - streams: - resolutions: - framerates - """ - formats = {} - - if self.context is None: - return formats - - devices = self.context.query_devices() - current_device = None - - for d in devices: - try: - serial = d.get_info(rs.camera_info.serial_number) - except RuntimeError as re: - logger.error("Device no longer available " + str(re)) - else: - if device_id == serial: - current_device = d - - if current_device is None: - return formats - logger.debug("Found the current device: " + device_id) - - sensors = current_device.query_sensors() - for s in sensors: - stream_profiles = s.get_stream_profiles() - for sp in stream_profiles: - vp = sp.as_video_stream_profile() - stream_type = vp.stream_type() - - if stream_type not in (rs.stream.color, rs.stream.depth): - continue - elif vp.format() not in (rs.format.z16, rs.format.yuyv): - continue - - formats.setdefault(stream_type, {}) - stream_resolution = (vp.width(), vp.height()) - formats[stream_type].setdefault(stream_resolution, []).append(vp.fps()) - - return formats - - def stop_pipeline(self): - if self.online: - try: - self.pipeline_profile = None - self.stream_profiles = None - self.pipeline.stop() - logger.debug("Pipeline stopped.") - except RuntimeError as re: - logger.error("Cannot stop the pipeline: " + str(re)) - - def cleanup(self): - if self.depth_video_writer is not None: - self.stop_depth_recording() - self.stop_pipeline() - - def get_init_dict(self): - return { - "frame_size": self.frame_size, - "frame_rate": self.frame_rate, - "depth_frame_size": self.depth_frame_size, - "depth_frame_rate": self.depth_frame_rate, - "preview_depth": self.preview_depth, - "record_depth": self.record_depth, - } - - def get_frames(self): - if self.online: - try: - frames = self.pipeline.wait_for_frames(TIMEOUT) - except RuntimeError as e: - logger.error("get_frames: Timeout!") - raise RuntimeError(e) - else: - current_time = self.g_pool.get_timestamp() - - color = None - # if we're expecting color frames - if rs.stream.color in self.stream_profiles: - color_frame = frames.get_color_frame() - last_color_frame_ts = color_frame.get_timestamp() - if self.last_color_frame_ts != last_color_frame_ts: - self.last_color_frame_ts = last_color_frame_ts - color = ColorFrame( - np.asanyarray(color_frame.get_data()), - current_time, - self.color_frame_index, - ) - self.color_frame_index += 1 - - depth = None - # if we're expecting depth frames - if rs.stream.depth in self.stream_profiles: - depth_frame = frames.get_depth_frame() - last_depth_frame_ts = depth_frame.get_timestamp() - if self.last_depth_frame_ts != last_depth_frame_ts: - self.last_depth_frame_ts = last_depth_frame_ts - depth = DepthFrame( - np.asanyarray(depth_frame.get_data()), - current_time, - self.depth_frame_index, - ) - self.depth_frame_index += 1 - - return color, depth - return None, None - - def recent_events(self, events): - if self._needs_restart or not self.online: - logger.debug("recent_events -> restarting device") - self.restart_device() - time.sleep(0.01) - return - - try: - color_frame, depth_frame = self.get_frames() - except RuntimeError as re: - logger.warning("Realsense failed to provide frames." + str(re)) - self._recent_frame = None - self._recent_depth_frame = None - self._needs_restart = True - else: - if color_frame is not None: - self._recent_frame = color_frame - events["frame"] = color_frame - - if depth_frame is not None: - self._recent_depth_frame = depth_frame - events["depth_frame"] = depth_frame - - if self.depth_video_writer is not None: - self.depth_video_writer.write_video_frame(depth_frame) - - def deinit_ui(self): - self.remove_menu() - - def init_ui(self): - self.add_menu() - self.menu.label = "Local USB Video Source" - self.update_menu() - - def update_menu(self): - logger.debug("update_menu") - try: - del self.menu[:] - except AttributeError: - return - - from pyglui import ui - - if not self.online: - self.menu.append(ui.Info_Text("Capture initialization failed.")) - return - - self.menu.append(ui.Switch("record_depth", self, label="Record Depth Stream")) - self.menu.append(ui.Switch("preview_depth", self, label="Preview Depth")) - - if self._available_modes is not None: - - def frame_size_selection_getter(): - if self.device_id: - frame_size = sorted( - self._available_modes[rs.stream.color], reverse=True - ) - labels = ["({}, {})".format(t[0], t[1]) for t in frame_size] - return frame_size, labels - else: - return [self.frame_size_backup], [str(self.frame_size_backup)] - - # TODO: potential race condition through selection_getter. Should ensure - # that current selection will always be present in the list returned by the - # selection_getter. Highly unlikely though as this needs to happen between - # having clicked the Selector and the next redraw. - # See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b - selector = ui.Selector( - "frame_size", - self, - selection_getter=frame_size_selection_getter, - label="Color Resolution", - ) - self.menu.append(selector) - - def frame_rate_selection_getter(): - if self.device_id: - avail_fps = [ - fps - for fps in self._available_modes[rs.stream.color][ - self.frame_size - ] - ] - return avail_fps, [str(fps) for fps in avail_fps] - else: - return [self.frame_rate_backup], [str(self.frame_rate_backup)] - - # TODO: potential race condition through selection_getter. Should ensure - # that current selection will always be present in the list returned by the - # selection_getter. Highly unlikely though as this needs to happen between - # having clicked the Selector and the next redraw. - # See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b - selector = ui.Selector( - "frame_rate", - self, - selection_getter=frame_rate_selection_getter, - label="Color Frame Rate", - ) - self.menu.append(selector) - - def depth_frame_size_selection_getter(): - if self.device_id: - depth_sizes = sorted( - self._available_modes[rs.stream.depth], reverse=True - ) - labels = ["({}, {})".format(t[0], t[1]) for t in depth_sizes] - return depth_sizes, labels - else: - return ( - [self.depth_frame_size_backup], - [str(self.depth_frame_size_backup)], - ) - - # TODO: potential race condition through selection_getter. Should ensure - # that current selection will always be present in the list returned by the - # selection_getter. Highly unlikely though as this needs to happen between - # having clicked the Selector and the next redraw. - # See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b - selector = ui.Selector( - "depth_frame_size", - self, - selection_getter=depth_frame_size_selection_getter, - label="Depth Resolution", - ) - self.menu.append(selector) - - def depth_frame_rate_selection_getter(): - if self.device_id: - avail_fps = [ - fps - for fps in self._available_modes[rs.stream.depth][ - self.depth_frame_size - ] - ] - return avail_fps, [str(fps) for fps in avail_fps] - else: - return ( - [self.depth_frame_rate_backup], - [str(self.depth_frame_rate_backup)], - ) - - # TODO: potential race condition through selection_getter. Should ensure - # that current selection will always be present in the list returned by the - # selection_getter. Highly unlikely though as this needs to happen between - # having clicked the Selector and the next redraw. - # See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b - selector = ui.Selector( - "depth_frame_rate", - self, - selection_getter=depth_frame_rate_selection_getter, - label="Depth Frame Rate", - ) - self.menu.append(selector) - - def reset_options(): - logger.debug("reset_options") - self.reset_device(self.device_id) - - sensor_control = ui.Growing_Menu(label="Sensor Settings") - sensor_control.append( - ui.Button("Reset device options to default", reset_options) - ) - self.menu.append(sensor_control) - else: - logger.debug("update_menu: self._available_modes is None") - - def gl_display(self): - - if self.preview_depth and self._recent_depth_frame is not None: - self.g_pool.image_tex.update_from_ndarray(self._recent_depth_frame.bgr) - gl_utils.glFlush() - gl_utils.make_coord_system_norm_based() - self.g_pool.image_tex.draw() - elif self._recent_frame is not None: - self.g_pool.image_tex.update_from_yuv_buffer( - self._recent_frame.yuv_buffer, - self._recent_frame.width, - self._recent_frame.height, - ) - gl_utils.glFlush() - gl_utils.make_coord_system_norm_based() - self.g_pool.image_tex.draw() - - if not self.online: - super().gl_display() - - gl_utils.make_coord_system_pixel_based( - (self.frame_size[1], self.frame_size[0], 3) - ) - - def reset_device(self, device_id): - logger.debug("reset_device") - if device_id is None: - device_id = self.device_id - - self.notify_all( - { - "subject": "realsense2_source.restart", - "device_id": device_id, - "color_frame_size": None, - "color_fps": None, - "depth_frame_size": None, - "depth_fps": None, - "device_options": [], # FIXME - } - ) - - def restart_device( - self, - color_frame_size=None, - color_fps=None, - depth_frame_size=None, - depth_fps=None, - device_options=None, - ): - if color_frame_size is None: - color_frame_size = self.frame_size - if color_fps is None: - color_fps = self.frame_rate - if depth_frame_size is None: - depth_frame_size = self.depth_frame_size - if depth_fps is None: - depth_fps = self.depth_frame_rate - if device_options is None: - device_options = [] # FIXME - - self.notify_all( - { - "subject": "realsense2_source.restart", - "device_id": None, - "color_frame_size": color_frame_size, - "color_fps": color_fps, - "depth_frame_size": depth_frame_size, - "depth_fps": depth_fps, - "device_options": device_options, - } - ) - logger.debug("self.restart_device --> self.notify_all") - - def on_notify(self, notification): - logger.debug( - 'self.on_notify, notification["subject"]: ' + notification["subject"] - ) - if notification["subject"] == "realsense2_source.restart": - kwargs = notification.copy() - del kwargs["subject"] - del kwargs["topic"] - self._initialize_device(**kwargs) - elif notification["subject"] == "recording.started": - self.start_depth_recording( - notification["rec_path"], notification["start_time_synced"] - ) - elif notification["subject"] == "recording.stopped": - self.stop_depth_recording() - - def start_depth_recording(self, rec_loc, start_time_synced): - if not self.record_depth: - return - - if self.depth_video_writer is not None: - logger.warning("Depth video recording has been started already") - return - - video_path = os.path.join(rec_loc, "depth.mp4") - self.depth_video_writer = MPEG_Writer(video_path, start_time_synced) - - def stop_depth_recording(self): - if self.depth_video_writer is None: - logger.warning("Depth video recording was not running") - return - - self.depth_video_writer.close() - self.depth_video_writer = None - - @property - def device_id(self): - if self.online: # already running - return self.pipeline_profile.get_device().get_info( - rs.camera_info.serial_number - ) - else: - # set the first available device - devices = self.context.query_devices() - if devices: - logger.info("device_id: first device by default.") - return devices[0].get_info(rs.camera_info.serial_number) - else: - logger.debug("device_id: No device connected.") - return None - - @property - def frame_size(self): - try: - stream_profile = self.stream_profiles[rs.stream.color] - # TODO check width & height is in self.available modes - return stream_profile.width(), stream_profile.height() - except AttributeError: - return self.frame_size_backup - except KeyError: - return self.frame_size_backup - except TypeError: - return self.frame_size_backup - - @frame_size.setter - def frame_size(self, new_size): - if new_size != self.frame_size: - self.restart_device(color_frame_size=new_size) - - @property - def frame_rate(self): - try: - stream_profile = self.stream_profiles[rs.stream.color] - # TODO check FPS is in self.available modes - return stream_profile.fps() - except AttributeError: - return self.frame_rate_backup - except KeyError: - return self.frame_rate_backup - except TypeError: - return self.frame_rate_backup - - @frame_rate.setter - def frame_rate(self, new_rate): - if new_rate != self.frame_rate: - self.restart_device(color_fps=new_rate) - - @property - def depth_frame_size(self): - try: - stream_profile = self.stream_profiles[rs.stream.depth] - # TODO check width & height is in self.available modes - return stream_profile.width(), stream_profile.height() - except AttributeError: - return self.depth_frame_size_backup - except KeyError: - return self.depth_frame_size_backup - except TypeError: - return self.depth_frame_size_backup - - @depth_frame_size.setter - def depth_frame_size(self, new_size): - if new_size != self.depth_frame_size: - self.restart_device(depth_frame_size=new_size) - - @property - def depth_frame_rate(self): - try: - stream_profile = self.stream_profiles[rs.stream.depth] - return stream_profile.fps() - except AttributeError: - return self.depth_frame_rate_backup - except KeyError: - return self.depth_frame_rate_backup - except TypeError: - return self.depth_frame_rate_backup - - @depth_frame_rate.setter - def depth_frame_rate(self, new_rate): - if new_rate != self.depth_frame_rate: - self.restart_device(depth_fps=new_rate) - - @property - def jpeg_support(self): - return False - - @property - def online(self): - return self.pipeline_profile is not None and self.pipeline is not None - - @property - def name(self): - if self.online: - return self.pipeline_profile.get_device().get_info(rs.camera_info.name) - else: - logger.debug( - "self.name: Realsense2 not online. Falling back to Ghost capture" - ) - return "Ghost capture" - - -class Realsense2_Manager(Base_Manager): - """Manages Intel RealSense D400 sources - - Attributes: - check_intervall (float): Intervall in which to look for new UVC devices - """ - - gui_name = "RealSense D400" - - def get_init_dict(self): - return {} - - def init_ui(self): - self.add_menu() - from pyglui import ui - - self.menu.append(ui.Info_Text("Intel RealSense D400 sources")) - - def is_streaming(device_id): - try: - c = rs.config() - c.enable_device(device_id) # device_id is in fact the serial_number - p = rs.pipeline() - p.start(c) - p.stop() - return False - except RuntimeError: - return True - - def get_device_info(d): - name = d.get_info(rs.camera_info.name) # FIXME is camera in use? - device_id = d.get_info(rs.camera_info.serial_number) - - fmt = "- " if is_streaming(device_id) else "" - fmt += name - - return device_id, fmt - - def dev_selection_list(): - default = (None, "Select to activate") - try: - ctx = rs.context() # FIXME cannot use "with rs.context() as ctx:" - # got "AttributeError: __enter__" - # see https://stackoverflow.com/questions/5093382/object-becomes-none-when-using-a-context-manager - dev_pairs = [default] + [get_device_info(d) for d in ctx.devices] - except Exception: # FIXME - dev_pairs = [default] - - return zip(*dev_pairs) - - def activate(source_uid): - if source_uid is None: - return - - settings = { - "frame_size": self.g_pool.capture.frame_size, - "frame_rate": self.g_pool.capture.frame_rate, - "device_id": source_uid, - } - if self.g_pool.process == "world": - self.notify_all( - { - "subject": "start_plugin", - "name": "Realsense2_Source", - "args": settings, - } - ) - else: - self.notify_all( - { - "subject": "start_eye_plugin", - "target": self.g_pool.process, - "name": "Realsense2_Source", - "args": settings, - } - ) - - self.menu.append( - ui.Selector( - "selected_source", - selection_getter=dev_selection_list, - getter=lambda: None, - setter=activate, - label="Activate source", - ) - ) - - def deinit_ui(self): - self.remove_menu() diff --git a/pupil_src/shared_modules/video_capture/realsense_backend.py b/pupil_src/shared_modules/video_capture/realsense_backend.py deleted file mode 100755 index 567b4c8a99..0000000000 --- a/pupil_src/shared_modules/video_capture/realsense_backend.py +++ /dev/null @@ -1,947 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2020 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" - -import logging -import time -import cv2 -import os - -import pyrealsense as pyrs -from pyrealsense.stream import ColorStream, DepthStream, DACStream, PointStream -from pyrealsense.constants import rs_stream, rs_option, rs_preset -from pyrealsense.extlib import rsutilwrapper - -from version_utils import VersionFormat -from .base_backend import Base_Source, Base_Manager -from av_writer import MPEG_Writer -from camera_models import load_intrinsics - -import glfw -import gl_utils -from OpenGL.GL import * -from OpenGL.GLU import * -from pyglui import cygl -import cython_methods -import numpy as np -from ctypes import * - -# check versions for our own depedencies as they are fast-changing -assert VersionFormat(pyrs.__version__) >= VersionFormat("2.2") - -# logging -logging.getLogger("pyrealsense").setLevel(logging.ERROR + 1) -logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) - - -class ColorFrame(object): - def __init__(self, device): - # we need to keep this since there is no cv2 conversion for our planar format - self._yuv422 = device.color - self._shape = self._yuv422.shape[:2] - self._yuv = np.empty(self._yuv422.size, dtype=np.uint8) - y_plane = self._yuv422.size // 2 - u_plane = y_plane // 2 - self._yuv[:y_plane] = self._yuv422[:, :, 0].flatten() - self._yuv[y_plane : y_plane + u_plane] = self._yuv422[:, ::2, 1].flatten() - self._yuv[y_plane + u_plane :] = self._yuv422[:, 1::2, 1].flatten() - self._bgr = None - self._gray = None - - @property - def height(self): - return self._shape[0] - - @property - def width(self): - return self._shape[1] - - @property - def yuv_buffer(self): - return self._yuv - - @property - def yuv422(self): - Y = self._yuv[: self._yuv.size // 2] - U = self._yuv[self._yuv.size // 2 : 3 * self._yuv.size // 4] - V = self._yuv[3 * self._yuv.size // 4 :] - - Y.shape = self._shape - U.shape = self._shape[0], self._shape[1] // 2 - V.shape = self._shape[0], self._shape[1] // 2 - - return Y, U, V - - @property - def bgr(self): - if self._bgr is None: - self._bgr = cv2.cvtColor(self._yuv422, cv2.COLOR_YUV2BGR_YUYV) - return self._bgr - - @property - def img(self): - return self.bgr - - @property - def gray(self): - if self._gray is None: - self._gray = self._yuv[: self._yuv.size // 2] - self._gray.shape = self._shape - return self._gray - - -class DepthFrame(object): - def __init__(self, device): - self._bgr = None - self._gray = None - self.depth = device.depth - self.yuv_buffer = None - - @property - def height(self): - return self.depth.shape[0] - - @property - def width(self): - return self.depth.shape[1] - - @property - def bgr(self): - if self._bgr is None: - self._bgr = cython_methods.cumhist_color_map16(self.depth) - return self._bgr - - @property - def img(self): - return self.bgr - - @property - def gray(self): - if self._gray is None: - self._gray = cv2.cvtColor(self.bgr, cv2.cv2.COLOR_BGR2GRAY) - return self._gray - - -class Control(object): - def __init__(self, device, opt_range, value): - self._dev = device - self._value = value - self.range = opt_range - self.label = rs_option.name_for_value[opt_range.option] - self.label = self.label.replace("RS_OPTION_", "") - self.label = self.label.replace("R200_", "") - self.label = self.label.replace("_", " ") - self.label = self.label.title() - self.description = self._dev.get_device_option_description(opt_range.option) - - @property - def value(self): - return self._value - - @value.setter - def value(self, val): - try: - self._dev.set_device_option(self.range.option, val) - except pyrs.RealsenseError as err: - logger.error('Setting option "{}" failed'.format(self.label)) - logger.debug("Reason: {}".format(err)) - else: - self._value = val - - def refresh(self): - self._value = self._dev.get_device_option(self.range.option) - - -class Realsense_Controls(dict): - def __init__(self, device, presets=()): - if not device: - super().__init__() - return - if presets: - # presets: list of (option, value)-tuples - try: - device.set_device_options(*zip(*presets)) - except pyrs.RealsenseError as err: - logger.error("Setting device option presets failed") - logger.debug("Reason: {}".format(err)) - controls = {} - for opt_range, value in device.get_available_options(): - if opt_range.min < opt_range.max: - controls[opt_range.option] = Control(device, opt_range, value) - super().__init__(controls) - - def export_presets(self): - return [(opt, ctrl.value) for opt, ctrl in self.items()] - - def refresh(self): - for ctrl in self.values(): - ctrl.refresh() - - -class Realsense_Source(Base_Source): - """ - Camera Capture is a class that encapsualtes pyrs.Device: - """ - - def __init__( - self, - g_pool, - device_id=0, - frame_size=(1920, 1080), - frame_rate=30, - depth_frame_size=(640, 480), - depth_frame_rate=60, - align_streams=False, - preview_depth=False, - device_options=(), - record_depth=True, - stream_preset=None, - ): - super().__init__(g_pool) - self._intrinsics = None - self.color_frame_index = 0 - self.depth_frame_index = 0 - self.device = None - self.service = pyrs.Service() - self.align_streams = align_streams - self.preview_depth = preview_depth - self.record_depth = record_depth - self.depth_video_writer = None - self.controls = None - self.pitch = 0 - self.yaw = 0 - self.mouse_drag = False - self.last_pos = (0, 0) - self.depth_window = None - self._needs_restart = False - self.stream_preset = stream_preset - self._initialize_device( - device_id, - frame_size, - frame_rate, - depth_frame_size, - depth_frame_rate, - device_options, - ) - - def _initialize_device( - self, - device_id, - color_frame_size, - color_fps, - depth_frame_size, - depth_fps, - device_options=(), - ): - devices = tuple(self.service.get_devices()) - color_frame_size = tuple(color_frame_size) - depth_frame_size = tuple(depth_frame_size) - - self.streams = [ColorStream(), DepthStream(), PointStream()] - self.last_color_frame_ts = None - self.last_depth_frame_ts = None - self._recent_frame = None - self._recent_depth_frame = None - - if not devices: - if not self._needs_restart: - logger.error("Camera failed to initialize. No cameras connected.") - self.device = None - self.update_menu() - return - - if self.device is not None: - self.device.stop() # only call Device.stop() if its context - - if device_id >= len(devices): - logger.error( - "Camera with id {} not found. Initializing default camera.".format( - device_id - ) - ) - device_id = 0 - - # use default streams to filter modes by rs_stream and rs_format - self._available_modes = self._enumerate_formats(device_id) - - # make sure that given frame sizes and rates are available - color_modes = self._available_modes[rs_stream.RS_STREAM_COLOR] - if color_frame_size not in color_modes: - # automatically select highest resolution - color_frame_size = sorted(color_modes.keys(), reverse=True)[0] - - if color_fps not in color_modes[color_frame_size]: - # automatically select highest frame rate - color_fps = color_modes[color_frame_size][0] - - depth_modes = self._available_modes[rs_stream.RS_STREAM_DEPTH] - if self.align_streams: - depth_frame_size = color_frame_size - else: - if depth_frame_size not in depth_modes: - # automatically select highest resolution - depth_frame_size = sorted(depth_modes.keys(), reverse=True)[0] - - if depth_fps not in depth_modes[depth_frame_size]: - # automatically select highest frame rate - depth_fps = depth_modes[depth_frame_size][0] - - colorstream = ColorStream( - width=color_frame_size[0], - height=color_frame_size[1], - fps=color_fps, - color_format="yuv", - preset=self.stream_preset, - ) - depthstream = DepthStream( - width=depth_frame_size[0], - height=depth_frame_size[1], - fps=depth_fps, - preset=self.stream_preset, - ) - pointstream = PointStream( - width=depth_frame_size[0], height=depth_frame_size[1], fps=depth_fps - ) - - self.streams = [colorstream, depthstream, pointstream] - if self.align_streams: - dacstream = DACStream( - width=depth_frame_size[0], height=depth_frame_size[1], fps=depth_fps - ) - dacstream.name = "depth" # rename data accessor - self.streams.append(dacstream) - - # update with correctly initialized streams - # always initiliazes color + depth, adds rectified/aligned versions as necessary - - self.device = self.service.Device(device_id, streams=self.streams) - self.controls = Realsense_Controls(self.device, device_options) - self._intrinsics = load_intrinsics( - self.g_pool.user_dir, self.name, self.frame_size - ) - - self.update_menu() - self._needs_restart = False - - def _enumerate_formats(self, device_id): - """Enumerate formats into hierachical structure: - - streams: - resolutions: - framerates - """ - formats = {} - # only lists modes for native streams (RS_STREAM_COLOR/RS_STREAM_DEPTH) - for mode in self.service.get_device_modes(device_id): - if mode.stream in (rs_stream.RS_STREAM_COLOR, rs_stream.RS_STREAM_DEPTH): - # check if frame size dict is available - if mode.stream not in formats: - formats[mode.stream] = {} - stream_obj = next((s for s in self.streams if s.stream == mode.stream)) - if mode.format == stream_obj.format: - size = mode.width, mode.height - # check if framerate list is already available - if size not in formats[mode.stream]: - formats[mode.stream][size] = [] - formats[mode.stream][size].append(mode.fps) - - if self.align_streams: - depth_sizes = formats[rs_stream.RS_STREAM_DEPTH].keys() - color_sizes = formats[rs_stream.RS_STREAM_COLOR].keys() - # common_sizes = depth_sizes & color_sizes - discarded_sizes = depth_sizes ^ color_sizes - for size in discarded_sizes: - for sizes in formats.values(): - if size in sizes: - del sizes[size] - - return formats - - def cleanup(self): - if self.depth_video_writer is not None: - self.stop_depth_recording() - if self.device is not None: - self.device.stop() - self.service.stop() - - def get_init_dict(self): - return { - "device_id": self.device.device_id if self.device is not None else 0, - "frame_size": self.frame_size, - "frame_rate": self.frame_rate, - "depth_frame_size": self.depth_frame_size, - "depth_frame_rate": self.depth_frame_rate, - "preview_depth": self.preview_depth, - "record_depth": self.record_depth, - "align_streams": self.align_streams, - "device_options": self.controls.export_presets() - if self.controls is not None - else (), - "stream_preset": self.stream_preset, - } - - def get_frames(self): - if self.device: - self.device.wait_for_frames() - current_time = self.g_pool.get_timestamp() - - last_color_frame_ts = self.device.get_frame_timestamp( - self.streams[0].stream - ) - if self.last_color_frame_ts != last_color_frame_ts: - self.last_color_frame_ts = last_color_frame_ts - color = ColorFrame(self.device) - color.timestamp = current_time - color.index = self.color_frame_index - self.color_frame_index += 1 - else: - color = None - - last_depth_frame_ts = self.device.get_frame_timestamp( - self.streams[1].stream - ) - if self.last_depth_frame_ts != last_depth_frame_ts: - self.last_depth_frame_ts = last_depth_frame_ts - depth = DepthFrame(self.device) - depth.timestamp = current_time - depth.index = self.depth_frame_index - self.depth_frame_index += 1 - else: - depth = None - - return color, depth - return None, None - - def recent_events(self, events): - if self._needs_restart: - self.restart_device() - time.sleep(0.05) - elif not self.online: - time.sleep(0.05) - return - - try: - color_frame, depth_frame = self.get_frames() - except (pyrs.RealsenseError, TimeoutError) as err: - logger.warning("Realsense failed to provide frames. Attempting to reinit.") - self._recent_frame = None - self._recent_depth_frame = None - self._needs_restart = True - else: - if color_frame and depth_frame: - self._recent_frame = color_frame - events["frame"] = color_frame - - if depth_frame: - self._recent_depth_frame = depth_frame - events["depth_frame"] = depth_frame - - if self.depth_video_writer is not None: - self.depth_video_writer.write_video_frame(depth_frame) - - def deinit_ui(self): - self.remove_menu() - - def init_ui(self): - self.add_menu() - self.menu.label = "Local USB Video Source" - self.update_menu() - - def update_menu(self): - try: - del self.menu[:] - except AttributeError: - return - - from pyglui import ui - - if self.device is None: - self.menu.append(ui.Info_Text("Capture initialization failed.")) - return - - def align_and_restart(val): - self.align_streams = val - self.restart_device() - - self.menu.append(ui.Switch("record_depth", self, label="Record Depth Stream")) - self.menu.append(ui.Switch("preview_depth", self, label="Preview Depth")) - self.menu.append( - ui.Switch( - "align_streams", self, label="Align Streams", setter=align_and_restart - ) - ) - - def toggle_depth_display(): - def on_depth_mouse_button(window, button, action, mods): - if button == glfw.GLFW_MOUSE_BUTTON_LEFT and action == glfw.GLFW_PRESS: - self.mouse_drag = True - if ( - button == glfw.GLFW_MOUSE_BUTTON_LEFT - and action == glfw.GLFW_RELEASE - ): - self.mouse_drag = False - - if self.depth_window is None: - self.pitch = 0 - self.yaw = 0 - - win_size = glfw.glfwGetWindowSize(self.g_pool.main_window) - self.depth_window = glfw.glfwCreateWindow( - win_size[0], win_size[1], "3D Point Cloud" - ) - glfw.glfwSetMouseButtonCallback( - self.depth_window, on_depth_mouse_button - ) - active_window = glfw.glfwGetCurrentContext() - glfw.glfwMakeContextCurrent(self.depth_window) - gl_utils.basic_gl_setup() - gl_utils.make_coord_system_norm_based() - - # refresh speed settings - glfw.glfwSwapInterval(0) - - glfw.glfwMakeContextCurrent(active_window) - - native_presets = [ - ("None", None), - ("Best Quality", rs_preset.RS_PRESET_BEST_QUALITY), - ("Largest image", rs_preset.RS_PRESET_LARGEST_IMAGE), - ("Highest framerate", rs_preset.RS_PRESET_HIGHEST_FRAMERATE), - ] - - def set_stream_preset(val): - if self.stream_preset != val: - self.stream_preset = val - self.restart_device() - - self.menu.append( - ui.Selector( - "stream_preset", - self, - setter=set_stream_preset, - labels=[preset[0] for preset in native_presets], - selection=[preset[1] for preset in native_presets], - label="Stream preset", - ) - ) - color_sizes = sorted( - self._available_modes[rs_stream.RS_STREAM_COLOR], reverse=True - ) - selector = ui.Selector( - "frame_size", - self, - # setter=, - selection=color_sizes, - label="Resolution" if self.align_streams else "Color Resolution", - ) - selector.read_only = self.stream_preset is not None - self.menu.append(selector) - - def color_fps_getter(): - avail_fps = [ - fps - for fps in self._available_modes[rs_stream.RS_STREAM_COLOR][ - self.frame_size - ] - if self.depth_frame_rate % fps == 0 - ] - return avail_fps, [str(fps) for fps in avail_fps] - - # TODO: potential race condition through selection_getter. Should ensure that - # current selection will always be present in the list returned by the - # selection_getter. Highly unlikely though as this needs to happen between - # having clicked the Selector and the next redraw. - # See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b - selector = ui.Selector( - "frame_rate", - self, - # setter=, - selection_getter=color_fps_getter, - label="Color Frame Rate", - ) - selector.read_only = self.stream_preset is not None - self.menu.append(selector) - - if not self.align_streams: - depth_sizes = sorted( - self._available_modes[rs_stream.RS_STREAM_DEPTH], reverse=True - ) - selector = ui.Selector( - "depth_frame_size", - self, - # setter=, - selection=depth_sizes, - label="Depth Resolution", - ) - selector.read_only = self.stream_preset is not None - self.menu.append(selector) - - def depth_fps_getter(): - avail_fps = [ - fps - for fps in self._available_modes[rs_stream.RS_STREAM_DEPTH][ - self.depth_frame_size - ] - if fps % self.frame_rate == 0 - ] - return avail_fps, [str(fps) for fps in avail_fps] - - # TODO: potential race condition through selection_getter. Should ensure that - # current selection will always be present in the list returned by the - # selection_getter. Highly unlikely though as this needs to happen between - # having clicked the Selector and the next redraw. - # See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b - selector = ui.Selector( - "depth_frame_rate", - self, - selection_getter=depth_fps_getter, - label="Depth Frame Rate", - ) - selector.read_only = self.stream_preset is not None - self.menu.append(selector) - - def reset_options(): - if self.device: - try: - self.device.reset_device_options_to_default(self.controls.keys()) - except pyrs.RealsenseError as err: - logger.info("Resetting some device options failed") - logger.debug("Reason: {}".format(err)) - finally: - self.controls.refresh() - - self.menu.append(ui.Button("Point Cloud Window", toggle_depth_display)) - sensor_control = ui.Growing_Menu(label="Sensor Settings") - sensor_control.append( - ui.Button("Reset device options to default", reset_options) - ) - for ctrl in sorted(self.controls.values(), key=lambda x: x.range.option): - # sensor_control.append(ui.Info_Text(ctrl.description)) - if ( - ctrl.range.min == 0.0 - and ctrl.range.max == 1.0 - and ctrl.range.step == 1.0 - ): - sensor_control.append( - ui.Switch("value", ctrl, label=ctrl.label, off_val=0.0, on_val=1.0) - ) - else: - sensor_control.append( - ui.Slider( - "value", - ctrl, - label=ctrl.label, - min=ctrl.range.min, - max=ctrl.range.max, - step=ctrl.range.step, - ) - ) - self.menu.append(sensor_control) - - def gl_display(self): - from math import floor - - if self.depth_window is not None and glfw.glfwWindowShouldClose( - self.depth_window - ): - glfw.glfwDestroyWindow(self.depth_window) - self.depth_window = None - - if self.depth_window is not None and self._recent_depth_frame is not None: - active_window = glfw.glfwGetCurrentContext() - glfw.glfwMakeContextCurrent(self.depth_window) - - win_size = glfw.glfwGetFramebufferSize(self.depth_window) - gl_utils.adjust_gl_view(win_size[0], win_size[1]) - pos = glfw.glfwGetCursorPos(self.depth_window) - if self.mouse_drag: - self.pitch = np.clip(self.pitch + (pos[1] - self.last_pos[1]), -80, 80) - self.yaw = np.clip(self.yaw - (pos[0] - self.last_pos[0]), -120, 120) - self.last_pos = pos - - glClearColor(0, 0, 0, 0) - glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT) - glMatrixMode(GL_PROJECTION) - glLoadIdentity() - gluPerspective(60, win_size[0] / win_size[1], 0.01, 20.0) - glMatrixMode(GL_MODELVIEW) - glLoadIdentity() - gluLookAt(0, 0, 0, 0, 0, 1, 0, -1, 0) - glTranslatef(0, 0, 0.5) - glRotated(self.pitch, 1, 0, 0) - glRotated(self.yaw, 0, 1, 0) - glTranslatef(0, 0, -0.5) - - # glPointSize(2) - glEnable(GL_DEPTH_TEST) - extrinsics = self.device.get_device_extrinsics( - rs_stream.RS_STREAM_DEPTH, rs_stream.RS_STREAM_COLOR - ) - depth_frame = self._recent_depth_frame - color_frame = self._recent_frame - depth_scale = self.device.depth_scale - - glEnableClientState(GL_VERTEX_ARRAY) - - pointcloud = self.device.pointcloud - glVertexPointer(3, GL_FLOAT, 0, pointcloud) - glEnableClientState(GL_COLOR_ARRAY) - depth_to_color = np.zeros( - depth_frame.height * depth_frame.width * 3, np.uint8 - ) - rsutilwrapper.project_pointcloud_to_pixel( - depth_to_color, - self.device.depth_intrinsics, - self.device.color_intrinsics, - extrinsics, - pointcloud, - self._recent_frame.bgr, - ) - glColorPointer(3, GL_UNSIGNED_BYTE, 0, depth_to_color) - glDrawArrays(GL_POINTS, 0, depth_frame.width * depth_frame.height) - gl_utils.glFlush() - glDisable(GL_DEPTH_TEST) - # gl_utils.make_coord_system_norm_based() - glfw.glfwSwapBuffers(self.depth_window) - glfw.glfwMakeContextCurrent(active_window) - - if self.preview_depth and self._recent_depth_frame is not None: - self.g_pool.image_tex.update_from_ndarray(self._recent_depth_frame.bgr) - gl_utils.glFlush() - gl_utils.make_coord_system_norm_based() - self.g_pool.image_tex.draw() - elif self._recent_frame is not None: - self.g_pool.image_tex.update_from_yuv_buffer( - self._recent_frame.yuv_buffer, - self._recent_frame.width, - self._recent_frame.height, - ) - gl_utils.glFlush() - gl_utils.make_coord_system_norm_based() - self.g_pool.image_tex.draw() - - if not self.online: - super().gl_display() - - gl_utils.make_coord_system_pixel_based( - (self.frame_size[1], self.frame_size[0], 3) - ) - - def restart_device( - self, - device_id=None, - color_frame_size=None, - color_fps=None, - depth_frame_size=None, - depth_fps=None, - device_options=None, - ): - if device_id is None: - if self.device is not None: - device_id = self.device.device_id - else: - device_id = 0 - if color_frame_size is None: - color_frame_size = self.frame_size - if color_fps is None: - color_fps = self.frame_rate - if depth_frame_size is None: - depth_frame_size = self.depth_frame_size - if depth_fps is None: - depth_fps = self.depth_frame_rate - if device_options is None: - device_options = self.controls.export_presets() - if self.device is not None: - self.device.stop() - self.device = None - self.service.stop() - self.service.start() - self.notify_all( - { - "subject": "realsense_source.restart", - "device_id": device_id, - "color_frame_size": color_frame_size, - "color_fps": color_fps, - "depth_frame_size": depth_frame_size, - "depth_fps": depth_fps, - "device_options": device_options, - } - ) - - def on_click(self, pos, button, action): - if button == glfw.GLFW_MOUSE_BUTTON_LEFT and action == glfw.GLFW_PRESS: - self.mouse_drag = True - if button == glfw.GLFW_MOUSE_BUTTON_LEFT and action == glfw.GLFW_RELEASE: - self.mouse_drag = False - - def on_notify(self, notification): - if notification["subject"] == "realsense_source.restart": - kwargs = notification.copy() - del kwargs["subject"] - del kwargs["topic"] - self._initialize_device(**kwargs) - elif notification["subject"] == "recording.started": - self.start_depth_recording( - notification["rec_path"], notification["start_time_synced"] - ) - elif notification["subject"] == "recording.stopped": - self.stop_depth_recording() - - def start_depth_recording(self, rec_loc, start_time_synced): - if not self.record_depth: - return - - if self.depth_video_writer is not None: - logger.warning("Depth video recording has been started already") - return - - video_path = os.path.join(rec_loc, "depth.mp4") - self.depth_video_writer = MPEG_Writer(video_path, start_time_synced) - - def stop_depth_recording(self): - if self.depth_video_writer is None: - logger.warning("Depth video recording was not running") - return - - self.depth_video_writer.close() - self.depth_video_writer = None - - @property - def frame_size(self): - stream = self.streams[0] - return stream.width, stream.height - - @frame_size.setter - def frame_size(self, new_size): - if self.device is not None and new_size != self.frame_size: - self.restart_device(color_frame_size=new_size) - - @property - def frame_rate(self): - return self.streams[0].fps - - @frame_rate.setter - def frame_rate(self, new_rate): - if self.device is not None and new_rate != self.frame_rate: - self.restart_device(color_fps=new_rate) - - @property - def depth_frame_size(self): - stream = self.streams[1] - return stream.width, stream.height - - @depth_frame_size.setter - def depth_frame_size(self, new_size): - if self.device is not None and new_size != self.depth_frame_size: - self.restart_device(depth_frame_size=new_size) - - @property - def depth_frame_rate(self): - return self.streams[1].fps - - @depth_frame_rate.setter - def depth_frame_rate(self, new_rate): - if self.device is not None and new_rate != self.depth_frame_rate: - self.restart_device(depth_fps=new_rate) - - @property - def jpeg_support(self): - return False - - @property - def online(self): - return self.device and self.device.is_streaming() - - @property - def name(self): - # not the same as `if self.device:`! - if self.device is not None: - return self.device.name - else: - return "Ghost capture" - - -class Realsense_Manager(Base_Manager): - """Manages Intel RealSense R200 sources - - Attributes: - check_intervall (float): Intervall in which to look for new UVC devices - """ - - gui_name = "RealSense R200" - - def get_init_dict(self): - return {} - - def init_ui(self): - self.add_menu() - from pyglui import ui - - self.menu.append(ui.Info_Text("Intel RealSense R200 sources")) - - def pair(d): - fmt = "- " if d["is_streaming"] else "" - fmt += d["name"] - return d["id"], fmt - - def dev_selection_list(): - default = (None, "Select to activate") - try: - with pyrs.Service() as service: - dev_pairs = [default] + [pair(d) for d in service.get_devices()] - except pyrs.RealsenseError: - dev_pairs = [default] - - return zip(*dev_pairs) - - def activate(source_uid): - if source_uid is None: - return - - # with pyrs.Service() as service: - # if not service.is_device_streaming(source_uid): - # logger.error("The selected camera is already in use or blocked.") - # return - settings = { - "frame_size": self.g_pool.capture.frame_size, - "frame_rate": self.g_pool.capture.frame_rate, - "device_id": source_uid, - } - if self.g_pool.process == "world": - self.notify_all( - { - "subject": "start_plugin", - "name": "Realsense_Source", - "args": settings, - } - ) - else: - self.notify_all( - { - "subject": "start_eye_plugin", - "target": self.g_pool.process, - "name": "Realsense_Source", - "args": settings, - } - ) - - self.menu.append( - ui.Selector( - "selected_source", - selection_getter=dev_selection_list, - getter=lambda: None, - setter=activate, - label="Activate source", - ) - ) - - def deinit_ui(self): - self.remove_menu() diff --git a/pupil_src/shared_modules/video_capture/uvc_backend.py b/pupil_src/shared_modules/video_capture/uvc_backend.py index d1ba181c03..7b15fe8582 100644 --- a/pupil_src/shared_modules/video_capture/uvc_backend.py +++ b/pupil_src/shared_modules/video_capture/uvc_backend.py @@ -16,14 +16,14 @@ import time import numpy as np -from pyglui import cygl +from pyglui import cygl, ui import gl_utils import uvc from camera_models import load_intrinsics from version_utils import VersionFormat -from .base_backend import Base_Manager, Base_Source, InitialisationError +from .base_backend import Base_Manager, Base_Source, InitialisationError, SourceInfo from .utils import Check_Frame_Stripes, Exposure_Time # check versions for our own depedencies as they are fast-changing @@ -31,7 +31,7 @@ # logging logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) +logger.setLevel(logging.DEBUG) class TJSAMP(enum.IntEnum): @@ -61,9 +61,11 @@ def __init__( uvc_controls={}, check_stripes=True, exposure_mode="manual", + *args, + **kwargs, ): - super().__init__(g_pool) + super().__init__(g_pool, *args, **kwargs) self.uvc_capture = None self._restart_in = 3 assert name or preferred_names or uid @@ -124,9 +126,7 @@ def __init__( # check if we were sucessfull if not self.uvc_capture: - logger.error( - "Init failed. Capture is started in ghost mode. No images will be supplied." - ) + logger.error("Could not connect to device! No images will be supplied.") self.name_backup = preferred_names self.frame_size_backup = frame_size self.frame_rate_backup = frame_rate @@ -382,12 +382,13 @@ def _restart_logic(self): ) except (InitialisationError, uvc.InitError): time.sleep(0.02) - self.update_menu() self._restart_in = int(5 / 0.02) else: self._restart_in -= 1 def recent_events(self, events): + was_online = self.online + try: frame = self.uvc_capture.get_frame(0.05) @@ -426,6 +427,9 @@ def recent_events(self, events): events["frame"] = frame self._restart_in = 3 + if was_online != self.online: + self.update_menu() + def _get_uvc_controls(self): d = {} if self.uvc_capture: @@ -451,7 +455,7 @@ def name(self): if self.uvc_capture: return self.uvc_capture.name else: - return "Ghost capture" + return "(disconnected)" @property def frame_size(self): @@ -563,19 +567,14 @@ def jpeg_support(self): def online(self): return bool(self.uvc_capture) - def deinit_ui(self): - self.remove_menu() - - def init_ui(self): - self.add_menu() - self.menu.label = "Local USB Source: {}".format(self.name) - self.update_menu() + def ui_elements(self): + ui_elements = [] - def update_menu(self): - del self.menu[:] - from pyglui import ui + if self.uvc_capture is None: + ui_elements.append(ui.Info_Text("Local USB: camera disconnected!")) + return ui_elements - ui_elements = [] + ui_elements.append(ui.Info_Text(f"Camera: {self.name} @ Local USB")) # lets define some helper functions: def gui_load_defaults(): @@ -596,12 +595,6 @@ def set_frame_rate(new_rate): self.frame_rate = new_rate self.update_menu() - if self.uvc_capture is None: - ui_elements.append(ui.Info_Text("Capture initialization failed.")) - self.menu.extend(ui_elements) - return - - ui_elements.append(ui.Info_Text("{} Controls".format(self.name))) sensor_control = ui.Growing_Menu(label="Sensor Settings") sensor_control.append( ui.Info_Text("Do not change these during calibration or recording!") @@ -791,7 +784,8 @@ def set_check_stripes(enable_stripe_checks): label="Check Stripes", ) ) - self.menu.extend(ui_elements) + + return ui_elements def cleanup(self): self.devices.cleanup() @@ -830,13 +824,7 @@ def gl_display(self): class UVC_Manager(Base_Manager): - """Manages local USB sources - - Attributes: - check_intervall (float): Intervall in which to look for new UVC devices - """ - - gui_name = "Local USB" + """Manages local USB sources.""" def __init__(self, g_pool): super().__init__(g_pool) @@ -847,45 +835,33 @@ def __init__(self, g_pool): "world": ["ID2", "Logitech"], } - def get_init_dict(self): - return {} - - def init_ui(self): - self.add_menu() + def get_devices(self): + self.devices.update() + if len(self.devices) == 0: + return [] + else: + return [SourceInfo(label="Local USB", manager=self, key="usb")] - from pyglui import ui + def get_cameras(self): + self.devices.update() + return [ + SourceInfo( + label=f"{device['name']} @ Local USB", + manager=self, + key=f"cam.{device['uid']}", + ) + for device in self.devices + ] - self.add_auto_select_button() - ui_elements = [] - ui_elements.append(ui.Info_Text("Local UVC sources")) - - def dev_selection_list(): - default = (None, "Select to activate") - self.devices.update() - dev_pairs = [default] + [ - (d["uid"], d["name"]) - for d in self.devices - if "RealSense" not in d["name"] - ] - return zip(*dev_pairs) + def activate(self, key): + if key == "usb": + self.notify_all({"subject": "backend.uvc.auto_activate_source"}) + return - # TODO: potential race condition through selection_getter. Should ensure that - # current selection will always be present in the list returned by the - # selection_getter. Highly unlikely though as this needs to happen between - # having clicked the Selector and the next redraw. - # See https://github.com/pupil-labs/pyglui/pull/112/commits/587818e9556f14bfedd8ff8d093107358745c29b - ui_elements.append( - ui.Selector( - "selected_source", - selection_getter=dev_selection_list, - getter=lambda: None, - setter=self.activate, - label="Activate source", - ) - ) - self.menu.extend(ui_elements) + source_uid = key[4:] + self.activate_source(source_uid) - def activate(self, source_uid): + def activate_source(self, source_uid): if not source_uid: return @@ -916,29 +892,45 @@ def activate(self, source_uid): } ) + def on_notify(self, notification): + """Starts appropriate UVC sources. + + Emits notifications: + ``backend.uvc.auto_activate_source``: All UVC managers should auto activate a source + ``start_(eye_)plugin``: Starts UVC sources + + Reacts to notifications: + ``backend.uvc.auto_activate_source``: Auto activate best source for process + """ + if notification["subject"] == "backend.uvc.auto_activate_source": + self.auto_activate_source() + def auto_activate_source(self): + logger.debug("Auto activating USB source.") + self.devices.update() if not self.devices or len(self.devices) == 0: logger.warning("No default device is available.") return - cam_ids = self.cam_selection_lut[self.g_pool.process] + name_patterns = self.cam_selection_lut[self.g_pool.process] + matching_cams = [ + device + for device in self.devices + if any(pattern in device["name"] for pattern in name_patterns) + ] - for cam_id in cam_ids: - try: - source_id = next(d["uid"] for d in self.devices if cam_id in d["name"]) - self.activate(source_id) - break - except StopIteration: - source_id = None - else: - logger.warning("The default device is not found.") + if not matching_cams: + logger.warning("Could not find default device.") + return - def deinit_ui(self): - self.remove_menu() + # Sorting cams by bus_number increases chances of selecting only cams from the + # same headset when having multiple headsets connected. Note that two headsets + # might have the same bus_number when they share an internal USB bus. + cam = min( + matching_cams, key=lambda device: device.get("bus_number", float("inf")) + ) + self.activate_source(cam["uid"]) def cleanup(self): self.devices.cleanup() self.devices = None - - def recent_events(self, events): - pass