diff --git a/deployment/bundle_macos.sh b/deployment/bundle_macos.sh index 0d0400fb25..56aeead6f7 100755 --- a/deployment/bundle_macos.sh +++ b/deployment/bundle_macos.sh @@ -1,8 +1,11 @@ -#!/bin/bash +#!/usr/local/bin/zsh pl_codesign () { sign="Developer ID Application: Pupil Labs UG (haftungsbeschrankt) (R55K9ESN6B)" codesign \ + --all-architectures \ + --force \ + --strict=all \ --options runtime \ --entitlements entitlements.plist \ --continue \ @@ -42,6 +45,7 @@ mv dist/*.$ext ../$release_dir cd .. printf "\n##########\nSigning applications\n##########\n" +pl_codesign $release_dir/*.$ext/Contents/Resources/**/.dylibs/*.dylib pl_codesign $release_dir/*.$ext printf "\n##########\nCreating dmg file\n##########\n" diff --git a/deployment/deploy_capture/bundle.spec b/deployment/deploy_capture/bundle.spec index 494903681f..e8da02e1fa 100644 --- a/deployment/deploy_capture/bundle.spec +++ b/deployment/deploy_capture/bundle.spec @@ -52,7 +52,7 @@ if platform.system() == "Darwin": pathex=["../../pupil_src/shared_modules/"], hiddenimports=hidden_imports, hookspath=None, - runtime_hooks=None, + runtime_hooks=["../find_opengl_bigsur.py"], excludes=["matplotlib"], datas=data_files_pye3d, ) diff --git a/deployment/deploy_player/bundle.spec b/deployment/deploy_player/bundle.spec index 9e4b2cd912..2fb5afa28a 100644 --- a/deployment/deploy_player/bundle.spec +++ b/deployment/deploy_player/bundle.spec @@ -53,7 +53,7 @@ if platform.system() == "Darwin": pathex=["../../pupil_src/shared_modules/"], hiddenimports=hidden_imports, hookspath=None, - runtime_hooks=None, + runtime_hooks=["../find_opengl_bigsur.py"], excludes=["matplotlib"], datas=data_files_pye3d, ) diff --git a/deployment/deploy_service/bundle.spec b/deployment/deploy_service/bundle.spec index f91fb8bd2b..905271f3d6 100644 --- a/deployment/deploy_service/bundle.spec +++ b/deployment/deploy_service/bundle.spec @@ -43,7 +43,7 @@ if platform.system() == "Darwin": pathex=["../../pupil_src/shared_modules/"], hiddenimports=hidden_imports, hookspath=None, - runtime_hooks=None, + runtime_hooks=["../find_opengl_bigsur.py"], excludes=["matplotlib"], datas=data_files_pye3d, ) diff --git a/deployment/find_opengl_bigsur.py b/deployment/find_opengl_bigsur.py new file mode 100755 index 0000000000..292eaf04a4 --- /dev/null +++ b/deployment/find_opengl_bigsur.py @@ -0,0 +1,23 @@ +import ctypes.util +import functools + + +print("Attempting to import OpenGL using patched `ctypes.util.find_library`...") +_find_library_original = ctypes.util.find_library + +@functools.wraps(_find_library_original) +def _find_library_patched(name): + if name == "OpenGL": + return "/System/Library/Frameworks/OpenGL.framework/OpenGL" + else: + return _find_library_original(name) + +ctypes.util.find_library = _find_library_patched + +import OpenGL.GL + +print("OpenGL import successful!") +print("Restoring original `ctypes.util.find_library`...") +ctypes.util.find_library = _find_library_original +del _find_library_patched +print("Original `ctypes.util.find_library` restored.") diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index 6ec1c9b451..b0bc9384c2 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -126,8 +126,9 @@ def eye( # display import glfw + from gl_utils import GLFWErrorReporting - glfw.ERROR_REPORTING = "raise" + GLFWErrorReporting.set_default() from pyglui import ui, graph, cygl from pyglui.cygl.utils import draw_points, RGBA, draw_polyline diff --git a/pupil_src/launchables/player.py b/pupil_src/launchables/player.py index 4604a49ac1..afdcaceedd 100644 --- a/pupil_src/launchables/player.py +++ b/pupil_src/launchables/player.py @@ -73,8 +73,9 @@ def player( # display import glfw + from gl_utils import GLFWErrorReporting - glfw.ERROR_REPORTING = "raise" + GLFWErrorReporting.set_default() # check versions for our own depedencies as they are fast-changing from pyglui import __version__ as pyglui_version @@ -137,7 +138,7 @@ def player( ) assert parse_version(pyglui_version) >= parse_version( - "1.28" + "1.29" ), "pyglui out of date, please upgrade to newest version" process_was_interrupted = False @@ -374,7 +375,6 @@ def get_dt(): glfw.init() glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) main_window = glfw.create_window(width, height, window_name, None, None) - window_position_manager = gl_utils.WindowPositionManager() window_pos = window_position_manager.new_window_position( window=main_window, @@ -808,8 +808,9 @@ def player_drop( try: import glfw + from gl_utils import GLFWErrorReporting - glfw.ERROR_REPORTING = "raise" + GLFWErrorReporting.set_default() import gl_utils from OpenGL.GL import glClearColor diff --git a/pupil_src/launchables/world.py b/pupil_src/launchables/world.py index 3008f1afc5..78e9784402 100644 --- a/pupil_src/launchables/world.py +++ b/pupil_src/launchables/world.py @@ -121,8 +121,9 @@ def detection_enabled_setter(is_on: bool): # display import glfw + from gl_utils import GLFWErrorReporting - glfw.ERROR_REPORTING = "raise" + GLFWErrorReporting.set_default() from version_utils import parse_version from pyglui import ui, cygl, __version__ as pyglui_version diff --git a/pupil_src/shared_modules/audio_utils.py b/pupil_src/shared_modules/audio_utils.py index 20f65626b3..6b613a125e 100644 --- a/pupil_src/shared_modules/audio_utils.py +++ b/pupil_src/shared_modules/audio_utils.py @@ -81,7 +81,11 @@ def _load_audio_single(file_path, return_pts_based_timestamps=False): ), dtype=float, ) - container.seek(0) + try: + container.seek(0) + except av.AVError as err: + logger.debug(f"{err}") + return None return LoadedAudio(container, stream, timestamps) diff --git a/pupil_src/shared_modules/calibration_choreography/controller/gui_monitor.py b/pupil_src/shared_modules/calibration_choreography/controller/gui_monitor.py index 0e2e3b25d8..a89741a8ec 100644 --- a/pupil_src/shared_modules/calibration_choreography/controller/gui_monitor.py +++ b/pupil_src/shared_modules/calibration_choreography/controller/gui_monitor.py @@ -2,8 +2,9 @@ import typing as T import glfw +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() try: from typing import OrderedDict as T_OrderedDict # Python 3.7.2 diff --git a/pupil_src/shared_modules/calibration_choreography/controller/gui_window.py b/pupil_src/shared_modules/calibration_choreography/controller/gui_window.py index 159038da83..0b3bab1a16 100644 --- a/pupil_src/shared_modules/calibration_choreography/controller/gui_window.py +++ b/pupil_src/shared_modules/calibration_choreography/controller/gui_window.py @@ -5,10 +5,10 @@ import OpenGL.GL as gl import glfw - -glfw.ERROR_REPORTING = "raise" - import gl_utils +from gl_utils import GLFWErrorReporting + +GLFWErrorReporting.set_default() from pyglui.cygl.utils import draw_polyline diff --git a/pupil_src/shared_modules/calibration_choreography/natural_feature_plugin.py b/pupil_src/shared_modules/calibration_choreography/natural_feature_plugin.py index 09c5acec27..6957bc3087 100644 --- a/pupil_src/shared_modules/calibration_choreography/natural_feature_plugin.py +++ b/pupil_src/shared_modules/calibration_choreography/natural_feature_plugin.py @@ -16,8 +16,9 @@ import numpy as np import glfw +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() from methods import normalize from pyglui import ui diff --git a/pupil_src/shared_modules/calibration_choreography/screen_marker_plugin.py b/pupil_src/shared_modules/calibration_choreography/screen_marker_plugin.py index 19e70aa91b..c7170fa319 100644 --- a/pupil_src/shared_modules/calibration_choreography/screen_marker_plugin.py +++ b/pupil_src/shared_modules/calibration_choreography/screen_marker_plugin.py @@ -17,8 +17,9 @@ import OpenGL.GL as gl import glfw +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() from circle_detector import CircleTracker from platform import system diff --git a/pupil_src/shared_modules/calibration_choreography/single_marker_plugin.py b/pupil_src/shared_modules/calibration_choreography/single_marker_plugin.py index 3876a0019f..1e534edd1f 100644 --- a/pupil_src/shared_modules/calibration_choreography/single_marker_plugin.py +++ b/pupil_src/shared_modules/calibration_choreography/single_marker_plugin.py @@ -27,8 +27,9 @@ import OpenGL.GL as gl import glfw +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() from .mixin import MonitorSelectionMixin from .controller import ( diff --git a/pupil_src/shared_modules/camera_intrinsics_estimation.py b/pupil_src/shared_modules/camera_intrinsics_estimation.py index 01ecbf8ea6..d0fd3730bd 100644 --- a/pupil_src/shared_modules/camera_intrinsics_estimation.py +++ b/pupil_src/shared_modules/camera_intrinsics_estimation.py @@ -27,10 +27,10 @@ from pyglui.ui import get_opensans_font_path import glfw - -glfw.ERROR_REPORTING = "raise" - import gl_utils +from gl_utils import GLFWErrorReporting + +GLFWErrorReporting.set_default() from plugin import Plugin diff --git a/pupil_src/shared_modules/gaze_producer/controller/reference_location_controllers.py b/pupil_src/shared_modules/gaze_producer/controller/reference_location_controllers.py index 14c10d5f41..5315066b02 100644 --- a/pupil_src/shared_modules/gaze_producer/controller/reference_location_controllers.py +++ b/pupil_src/shared_modules/gaze_producer/controller/reference_location_controllers.py @@ -14,8 +14,9 @@ import numpy as np import glfw +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() import tasklib from gaze_producer import model, worker diff --git a/pupil_src/shared_modules/gl_utils/__init__.py b/pupil_src/shared_modules/gl_utils/__init__.py index 3dd65f571f..30587d0008 100644 --- a/pupil_src/shared_modules/gl_utils/__init__.py +++ b/pupil_src/shared_modules/gl_utils/__init__.py @@ -28,5 +28,6 @@ make_coord_system_norm_based, make_coord_system_pixel_based, window_coordinate_to_framebuffer_coordinate, + GLFWErrorReporting, ) from .window_position_manager import WindowPositionManager diff --git a/pupil_src/shared_modules/gl_utils/utils.py b/pupil_src/shared_modules/gl_utils/utils.py index 83c9f5cfa5..faa85fbf73 100644 --- a/pupil_src/shared_modules/gl_utils/utils.py +++ b/pupil_src/shared_modules/gl_utils/utils.py @@ -8,6 +8,7 @@ See COPYING and COPYING.LESSER for license details. ---------------------------------------------------------------------------~(*) """ +import contextlib import logging import math import typing as T @@ -41,8 +42,6 @@ ) from OpenGL.GLU import gluErrorString, gluPerspective -glfw.ERROR_REPORTING = "raise" - # OpenGL.FULL_LOGGING = True OpenGL.ERROR_LOGGING = False @@ -330,3 +329,58 @@ def get_window_title_bar_rect(window) -> _Rectangle: return _Rectangle( x=frame_rect.x, y=frame_rect.y, width=frame_rect.width, height=frame_edges.top ) + + +_GLFWErrorReportingDict = T.Dict[T.Union[None, int], str] + + +class GLFWErrorReporting: + @classmethod + @contextlib.contextmanager + def error_code_handling( + cls, + *_, + ignore: T.Optional[T.Tuple[int]] = None, + debug: T.Optional[T.Tuple[int]] = None, + warn: T.Optional[T.Tuple[int]] = None, + raise_: T.Optional[T.Tuple[int]] = None, + ): + old_reporting = glfw.ERROR_REPORTING + + if isinstance(old_reporting, dict): + new_reporting: _GLFWErrorReportingDict = dict(old_reporting) + else: + new_reporting = cls.__default_error_reporting() + + new_reporting.update({err_code: "ignore" for err_code in ignore or ()}) + new_reporting.update({err_code: "log" for err_code in debug or ()}) + new_reporting.update({err_code: "warn" for err_code in warn or ()}) + new_reporting.update({err_code: "raise" for err_code in raise_ or ()}) + + glfw.ERROR_REPORTING = new_reporting + + try: + yield + finally: + glfw.ERROR_REPORTING = old_reporting + + @classmethod + def set_default(cls): + glfw.ERROR_REPORTING = cls.__default_error_reporting() + + ### Private + + @staticmethod + def __default_error_reporting() -> _GLFWErrorReportingDict: + ignore = [ + # GLFWError: (65544) b'Cocoa: Failed to find service port for display' + # This happens on macOS Big Sur running on Apple Silicone hardware + 65544, + ] + return { + None: "raise", + **{code: "log" for code in ignore}, + } + + +GLFWErrorReporting.set_default() diff --git a/pupil_src/shared_modules/gl_utils/window_position_manager.py b/pupil_src/shared_modules/gl_utils/window_position_manager.py index 3c5fbfaf90..ee93b57390 100644 --- a/pupil_src/shared_modules/gl_utils/window_position_manager.py +++ b/pupil_src/shared_modules/gl_utils/window_position_manager.py @@ -13,10 +13,10 @@ import typing as T import glfw - -glfw.ERROR_REPORTING = "raise" - import gl_utils +from .utils import GLFWErrorReporting + +GLFWErrorReporting.set_default() class WindowPositionManager: diff --git a/pupil_src/shared_modules/head_pose_tracker/ui/gl_window.py b/pupil_src/shared_modules/head_pose_tracker/ui/gl_window.py index 3625da2be5..9b37b30304 100644 --- a/pupil_src/shared_modules/head_pose_tracker/ui/gl_window.py +++ b/pupil_src/shared_modules/head_pose_tracker/ui/gl_window.py @@ -16,8 +16,9 @@ import gl_utils import glfw +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() from observable import Observable diff --git a/pupil_src/shared_modules/log_display.py b/pupil_src/shared_modules/log_display.py index 6526623e14..fe48023312 100644 --- a/pupil_src/shared_modules/log_display.py +++ b/pupil_src/shared_modules/log_display.py @@ -16,10 +16,10 @@ from pyglui.pyfontstash import fontstash from pyglui.ui import get_opensans_font_path import glfw - -glfw.ERROR_REPORTING = "raise" - import gl_utils +from gl_utils import GLFWErrorReporting + +GLFWErrorReporting.set_default() def color_from_level(lvl): diff --git a/pupil_src/shared_modules/marker_auto_trim_marks.py b/pupil_src/shared_modules/marker_auto_trim_marks.py index 192a902e8f..0096557f44 100644 --- a/pupil_src/shared_modules/marker_auto_trim_marks.py +++ b/pupil_src/shared_modules/marker_auto_trim_marks.py @@ -32,8 +32,9 @@ ) import glfw +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() import numpy as np from itertools import groupby diff --git a/pupil_src/shared_modules/pupil_detector_plugins/detector_2d_plugin.py b/pupil_src/shared_modules/pupil_detector_plugins/detector_2d_plugin.py index 04687c475d..89b092c3c5 100644 --- a/pupil_src/shared_modules/pupil_detector_plugins/detector_2d_plugin.py +++ b/pupil_src/shared_modules/pupil_detector_plugins/detector_2d_plugin.py @@ -16,15 +16,17 @@ import glfw -glfw.ERROR_REPORTING = "raise" - from gl_utils import ( adjust_gl_view, basic_gl_setup, clear_gl_screen, make_coord_system_norm_based, make_coord_system_pixel_based, + GLFWErrorReporting, ) + +GLFWErrorReporting.set_default() + from methods import normalize from plugin import Plugin diff --git a/pupil_src/shared_modules/pupil_recording/update/invisible.py b/pupil_src/shared_modules/pupil_recording/update/invisible.py index 10ae77cb74..fa681371ee 100644 --- a/pupil_src/shared_modules/pupil_recording/update/invisible.py +++ b/pupil_src/shared_modules/pupil_recording/update/invisible.py @@ -11,22 +11,23 @@ import logging import re +import shutil +import tempfile from pathlib import Path -import numpy as np - +import av import file_methods as fm import methods as m +import numpy as np +from version_utils import parse_version from video_capture.utils import pi_gaze_items from ..info import RecordingInfoFile from ..info import recording_info_utils as utils from ..recording import PupilRecording -from ..recording_utils import InvalidRecordingException +from ..recording_utils import VALID_VIDEO_EXTENSIONS, InvalidRecordingException from . import update_utils -from version_utils import parse_version - logger = logging.getLogger(__name__) NEWEST_SUPPORTED_VERSION = parse_version("1.4") @@ -61,6 +62,11 @@ def _transform_invisible_v1_0_to_pprf_2_1(rec_dir: str): recording = PupilRecording(rec_dir) + # Fix broken first frame issue, if affected. + # This needs to happend before anything else + # to make sure the rest of the pipeline is processed correctly. + BrokenFirstFrameRecordingIssue.patch_recording_if_affected(recording) + # patch world.intrinsics # NOTE: could still be worldless at this point update_utils._try_patch_world_instrinsics_file( @@ -101,11 +107,16 @@ def _generate_pprf_2_1_info_file(rec_dir: str) -> RecordingInfoFile: def _rename_pi_files(recording: PupilRecording): - for path in recording.files(): + for pi_path, core_path in _pi_path_core_path_pairs(recording): + pi_path.replace(core_path) # rename with overwrite + + +def _pi_path_core_path_pairs(recording: PupilRecording): + for pi_path in recording.files(): # replace prefix based on cam_type, need to reformat part number match = re.match( r"^(?PPI (?Pleft|right|world) v\d+ ps(?P\d+))", - path.name, + pi_path.name, ) if match: replacement_for_cam_type = { @@ -120,8 +131,9 @@ def _rename_pi_files(recording: PupilRecording): # NOTE: recordings for PI start at part 1, mobile start at part 0 replacement += f"_{part_number - 1:03}" - new_name = path.name.replace(match.group("prefix"), replacement) - path.replace(path.with_name(new_name)) # rename with overwrite + core_name = pi_path.name.replace(match.group("prefix"), replacement) + core_path = pi_path.with_name(core_name) + yield pi_path, core_path def _rewrite_timestamps(recording: PupilRecording): @@ -167,3 +179,140 @@ def android_system_info(info_json: dict) -> str: f"Android device name: {android_device_name}; " f"Android device model: {android_device_model}" ) + + +class BrokenFirstFrameRecordingIssue: + @classmethod + def is_recording_affected(cls, recording: PupilRecording) -> bool: + # If there are any world video and timestamps pairs affected - return True + # Otherwise - False + for _ in cls._pi_world_video_and_raw_time_affected_paths(recording): + return True + return False + + @classmethod + def patch_recording_if_affected(cls, recording: PupilRecording): + if not cls.is_recording_affected(recording=recording): + return + + with tempfile.TemporaryDirectory() as temp_dir: + for v_path, t_path in cls._pi_world_video_and_raw_time_affected_paths( + recording + ): + temp_t_path = Path(temp_dir) / t_path.name + temp_v_path = Path(temp_dir) / v_path.name + + # Save video, dropping first frame, to temp file + in_container = av.open(str(v_path)) + out_container = av.open(str(temp_v_path), "w") + + # input -> output stream mapping + stream_mapping = { + in_stream: out_container.add_stream(template=in_stream) + for in_stream in in_container.streams + } + + # Keep track of streams that should skip frame + stream_should_skip_frame = { + in_stream: in_stream.codec_context.type == "video" + or in_stream.codec_context.type == "audio" + for in_stream in stream_mapping.keys() + } + + for packet in in_container.demux(): + if stream_should_skip_frame[packet.stream]: + # Once the stream skipped the first frame, don't skip anymore + stream_should_skip_frame[packet.stream] = False + continue + packet.stream = stream_mapping[packet.stream] + out_container.mux(packet) + out_container.close() + + # Save raw time file, dropping first timestamp, to temp file + ts = cls._pi_raw_time_load(t_path) + cls._pi_raw_time_save(temp_t_path, ts[1:]) + + # Overwrite old files with new ones + v_path = v_path.with_name(v_path.stem).with_suffix(v_path.suffix) + + # pathlib.Path.replace raises an `OSError: [Errno 18] Cross-device link` + # if the temp file is on a different device than the original. This + # https://stackoverflow.com/a/43967659/5859392 recommends using + # shutil.move instead (only supports pathlike in python>=3.9). + shutil.move(str(temp_v_path), str(v_path)) + shutil.move(str(temp_t_path), str(t_path)) + + @classmethod + def _pi_world_video_and_raw_time_affected_paths(cls, recording: PupilRecording): + # Check if the first timestamp is greater than the second timestamp from world timestamps; + # this is a symptom of Pupil Invisible recording with broken first frame. + # If the first timestamp is greater, remove it from the timestamps and overwrite the file. + for v_path, ts_path in cls._pi_world_video_and_raw_time_paths(recording): + + in_container = av.open(str(v_path)) + packets = in_container.demux(video=0) + + # Try to demux the first frame. + # This is expected to raise an error. + # If no error is raised, ignore this video. + try: + _ = next(packets).decode() + except av.AVError: + pass # Expected + except StopIteration: + continue # Not expected + else: + continue # Not expected + + # Try to demux the second frame. + # This is not expected to raise an error. + # If an error is raised, ignore this video. + try: + _ = next(packets).decode() + except av.AVError: + continue # Not expected + except StopIteration: + continue # Not expected + else: + pass # Expected + + # Check there are 2 or more raw timestamps. + raw_time = cls._pi_raw_time_load(ts_path) + if len(raw_time) < 2: + continue + + yield v_path, ts_path + + @classmethod + def _pi_world_video_and_raw_time_paths(cls, recording: PupilRecording): + for pi_path, core_path in _pi_path_core_path_pairs(recording): + if not cls._is_pi_world_video_path(pi_path): + continue + + video_path = pi_path + raw_time_path = video_path.with_suffix(".time") + + assert raw_time_path.is_file(), f"Expected file at path: {raw_time_path}" + + yield video_path, raw_time_path + + @staticmethod + def _is_pi_world_video_path(path): + def match_any(target, *patterns): + return any([re.search(pattern, str(target)) for pattern in patterns]) + + is_pi_world = match_any(path.name, r"^PI world v(\d+) ps(\d+)") + + is_video = match_any( + path.name, *[rf"\.{ext}$" for ext in VALID_VIDEO_EXTENSIONS] + ) + + return is_pi_world and is_video + + @staticmethod + def _pi_raw_time_load(path): + return np.fromfile(str(path), dtype=" except av.AVError: continue + if container.streams.video[0].format is None: + continue + for camera in cm.default_intrinsics: if camera in video.name: camera_hint = camera diff --git a/pupil_src/shared_modules/roi.py b/pupil_src/shared_modules/roi.py index 03b5900726..10696e95cf 100644 --- a/pupil_src/shared_modules/roi.py +++ b/pupil_src/shared_modules/roi.py @@ -19,8 +19,9 @@ from pyglui.cygl.utils import draw_polyline as cygl_draw_polyline import glfw +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() from methods import denormalize, normalize from observable import Observable diff --git a/pupil_src/shared_modules/service_ui.py b/pupil_src/shared_modules/service_ui.py index 48b862b83c..45097c1413 100644 --- a/pupil_src/shared_modules/service_ui.py +++ b/pupil_src/shared_modules/service_ui.py @@ -18,10 +18,11 @@ from OpenGL.GL import GL_COLOR_BUFFER_BIT import glfw +import gl_utils +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() -import gl_utils from pyglui import ui, cygl from plugin import System_Plugin_Base @@ -57,6 +58,7 @@ def __init__( glfw.window_hint(glfw.SCALE_TO_MONITOR, glfw.TRUE) if g_pool.hide_ui: glfw.window_hint(glfw.VISIBLE, 0) # hide window + main_window = glfw.create_window(*window_size, "Pupil Service", None, None) window_position_manager = gl_utils.WindowPositionManager() diff --git a/pupil_src/shared_modules/surface_tracker/gui.py b/pupil_src/shared_modules/surface_tracker/gui.py index 3094d046e6..1189ab898d 100644 --- a/pupil_src/shared_modules/surface_tracker/gui.py +++ b/pupil_src/shared_modules/surface_tracker/gui.py @@ -21,8 +21,9 @@ import gl_utils import glfw +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() from .surface_marker import Surface_Marker_Type diff --git a/pupil_src/shared_modules/system_graphs.py b/pupil_src/shared_modules/system_graphs.py index 4a2a4af074..53b015c30c 100644 --- a/pupil_src/shared_modules/system_graphs.py +++ b/pupil_src/shared_modules/system_graphs.py @@ -12,10 +12,11 @@ import os import psutil import glfw +import gl_utils +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() -import gl_utils from pyglui import ui, graph from pyglui.cygl.utils import RGBA, mix_smooth from plugin import System_Plugin_Base diff --git a/pupil_src/shared_modules/video_capture/utils.py b/pupil_src/shared_modules/video_capture/utils.py index b056997548..8cc58df785 100644 --- a/pupil_src/shared_modules/video_capture/utils.py +++ b/pupil_src/shared_modules/video_capture/utils.py @@ -20,6 +20,10 @@ import cv2 import numpy as np + +import player_methods as pm + + logger = logging.getLogger(__name__) VIDEO_EXTS = ("mp4", "mjpeg", "h264", "mkv", "avi", "fake") @@ -483,22 +487,46 @@ def _setup_lookup(self, timestamps: np.ndarray) -> np.recarray: def pi_gaze_items(root_dir): + def find_timestamps_200hz_path(timestamps_path): + path = timestamps_path.with_name("gaze_200hz_timestamps.npy") + if path.is_file(): + return path + else: + return None + def find_raw_path(timestamps_path): - raw_name = timestamps_path.name.replace("_timestamps", "") - raw_path = timestamps_path.with_name(raw_name).with_suffix(".raw") - assert raw_path.exists(), f"The file does not exist at path: {raw_path}" - return raw_path + name = timestamps_path.name.replace("_timestamps", "") + path = timestamps_path.with_name(name).with_suffix(".raw") + assert path.is_file(), f"The file does not exist at path: {path}" + return path + + def find_raw_200hz_path(timestamps_path): + path = timestamps_path.with_name("gaze_200hz.raw") + if path.is_file(): + return path + else: + return None def find_worn_path(timestamps_path): - worn_name = timestamps_path.name - worn_name = worn_name.replace("gaze", "worn") - worn_name = worn_name.replace("_timestamps", "") - worn_path = timestamps_path.with_name(worn_name).with_suffix(".raw") - if worn_path.exists(): - return worn_path + name = timestamps_path.name + name = name.replace("gaze", "worn") + name = name.replace("_timestamps", "") + path = timestamps_path.with_name(name).with_suffix(".raw") + if path.is_file(): + return path + else: + return None + + def find_worn_200hz_path(timestamps_path): + path = timestamps_path.with_name("worn_200hz.raw") + if path.is_file(): + return path else: return None + def is_200hz(path: Path) -> bool: + return "200hz" in path.name + def load_timestamps_data(path): timestamps = np.load(str(path)) return timestamps @@ -525,9 +553,51 @@ def load_worn_data(path): ) for timestamps_path in gaze_timestamp_paths: - raw_path = find_raw_path(timestamps_path) - timestamps = load_timestamps_data(timestamps_path) - raw_data = load_raw_data(raw_path) + # Use 200hz data only if both gaze data and timestamps are available at 200hz + is_200hz_data_available = (find_raw_200hz_path(timestamps_path)) and ( + find_timestamps_200hz_path(timestamps_path) is not None + ) + + if is_200hz_data_available: + raw_data = load_raw_data(find_raw_200hz_path(timestamps_path)) + else: + raw_data = load_raw_data(find_raw_path(timestamps_path)) + + if is_200hz_data_available: + timestamps = load_timestamps_data( + find_timestamps_200hz_path(timestamps_path) + ) + else: + timestamps = load_timestamps_data(timestamps_path) + + if is_200hz_data_available: + ts_ = load_timestamps_data(timestamps_path) + ts_200hz_ = load_timestamps_data( + find_timestamps_200hz_path(timestamps_path) + ) + densification_idc = pm.find_closest(ts_, ts_200hz_) + else: + densification_idc = np.asarray(range(len(raw_data))) + + # Load confidence data when both 200hz gaze and 200hz confidence data is available + if ( + is_200hz_data_available + and find_worn_200hz_path(timestamps_path) is not None + ): + conf_data = load_worn_data(find_worn_200hz_path(timestamps_path)) + # Load and densify confidence data when 200hz gaze is available, but only non-200hz confidence is available + elif is_200hz_data_available and find_worn_path(timestamps_path) is not None: + conf_data = load_worn_data(find_worn_path(timestamps_path)) + conf_data = conf_data[densification_idc] + # Load confidence data when both non-200hz gaze and non-200hz confidence is available + elif ( + not is_200hz_data_available and find_worn_path(timestamps_path) is not None + ): + conf_data = load_worn_data(find_worn_path(timestamps_path)) + # Otherwise, don't load confidence data + else: + conf_data = None + if len(raw_data) != len(timestamps): logger.warning( f"There is a mismatch between the number of raw data ({len(raw_data)}) " @@ -537,16 +607,16 @@ def load_worn_data(path): raw_data = raw_data[:size] timestamps = timestamps[:size] - conf_data = load_worn_data(find_worn_path(timestamps_path)) if conf_data is not None and len(conf_data) != len(timestamps): logger.warning( f"There is a mismatch between the number of confidence data ({len(conf_data)}) " f"and the number of timestamps ({len(timestamps)})! Not using confidence data." ) + conf_data = None if conf_data is None: - conf_data = (1.0 for _ in range(len(timestamps))) + conf_data = (1.0 for _ in range(len(raw_data))) yield from zip(raw_data, timestamps, conf_data) diff --git a/pupil_src/shared_modules/video_overlay/ui/interactions.py b/pupil_src/shared_modules/video_overlay/ui/interactions.py index 259cf96b22..2b88d2ad78 100644 --- a/pupil_src/shared_modules/video_overlay/ui/interactions.py +++ b/pupil_src/shared_modules/video_overlay/ui/interactions.py @@ -9,10 +9,11 @@ ---------------------------------------------------------------------------~(*) """ import glfw +import gl_utils +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() -import gl_utils from methods import normalize, denormalize diff --git a/pupil_src/shared_modules/vis_watermark.py b/pupil_src/shared_modules/vis_watermark.py index e2e0fcbc5e..eb04b8eb1c 100644 --- a/pupil_src/shared_modules/vis_watermark.py +++ b/pupil_src/shared_modules/vis_watermark.py @@ -18,8 +18,9 @@ from pyglui import ui import glfw +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() from methods import normalize, denormalize import logging diff --git a/pupil_src/shared_modules/visualizer.py b/pupil_src/shared_modules/visualizer.py index 612052a881..0e6d0a9b31 100644 --- a/pupil_src/shared_modules/visualizer.py +++ b/pupil_src/shared_modules/visualizer.py @@ -9,10 +9,11 @@ ---------------------------------------------------------------------------~(*) """ import glfw +import gl_utils +from gl_utils import GLFWErrorReporting -glfw.ERROR_REPORTING = "raise" +GLFWErrorReporting.set_default() -import gl_utils from OpenGL.GL import ( GL_BLEND, GL_COLOR_BUFFER_BIT, diff --git a/requirements.txt b/requirements.txt index df50e7f215..04e31ccb8c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,9 +33,9 @@ av @ https://github.com/pupil-labs/PyAV/releases/download/v0.4.6/av-0.4.6-cp36-c uvc @ git+https://github.com/pupil-labs/pyuvc@v0.14.1 ; platform_system != "Windows" # Minor patch fixes build issues in pyproject.toml uvc @ https://github.com/pupil-labs/pyuvc/releases/download/v0.14/uvc-0.14-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" -# pupil-labs/pyglui 1.28 -pyglui @ git+https://github.com/pupil-labs/pyglui@v1.28 ; platform_system != "Windows" -pyglui @ https://github.com/pupil-labs/pyglui/releases/download/v1.28/pyglui-1.28-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" +# pupil-labs/pyglui 1.29 +pyglui @ git+https://github.com/pupil-labs/pyglui@v1.29 ; platform_system != "Windows" +pyglui @ https://github.com/pupil-labs/pyglui/releases/download/v1.29/pyglui-1.29-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" # pupil-labs/pyndsi 1.4 ndsi @ git+https://github.com/pupil-labs/pyndsi@v1.4 ; platform_system != "Windows"