From aa2f33327f99ac5404d0e682830dfa18803515cd Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Mon, 22 Feb 2021 19:28:00 +0100 Subject: [PATCH 01/32] Add Plugin class method to signal plugin availability given g_pool context --- pupil_src/shared_modules/plugin.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pupil_src/shared_modules/plugin.py b/pupil_src/shared_modules/plugin.py index 548e18db64..bd8ef3cb86 100644 --- a/pupil_src/shared_modules/plugin.py +++ b/pupil_src/shared_modules/plugin.py @@ -253,6 +253,13 @@ def pretty_class_name(self): def parse_pretty_class_name(cls) -> str: return cls.__name__.replace("_", " ") + @classmethod + def is_available_within_context(cls, g_pool) -> bool: + """ + Returns `True` if the plugin class is available within the `g_pool` context. + """ + return True + def add_menu(self): """ This fn is called when the plugin ui is initialized. Do not change! From 33381af0ba58845acea9b3b75383088c621b619b Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Mon, 22 Feb 2021 19:29:02 +0100 Subject: [PATCH 02/32] Update Plugin_List to skip adding Plugins that are not available given g_pool context --- pupil_src/shared_modules/plugin.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pupil_src/shared_modules/plugin.py b/pupil_src/shared_modules/plugin.py index bd8ef3cb86..a2ab5adf71 100644 --- a/pupil_src/shared_modules/plugin.py +++ b/pupil_src/shared_modules/plugin.py @@ -400,6 +400,12 @@ def add(self, new_plugin_cls, args={}): """ add a plugin instance to the list. """ + + # Check if the plugin class is supported within the current g_pool context + if not new_plugin_cls.is_available_within_context(self.g_pool): + logger.error(f"Plugin {new_plugin_cls.__name__} not available; skip adding it to plugin list.") + return + self._find_and_remove_duplicates(new_plugin_cls) plugin_instance = new_plugin_cls(self.g_pool, **args) From be0c57c8446c75709b593e89ae331aad695f8f34 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Mon, 22 Feb 2021 19:29:22 +0100 Subject: [PATCH 03/32] Update Plugin_Manager to skip adding Plugins that are not available given g_pool context --- pupil_src/shared_modules/plugin_manager.py | 26 ++++++++++++++++------ 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/pupil_src/shared_modules/plugin_manager.py b/pupil_src/shared_modules/plugin_manager.py index edfd07434e..8dcd731624 100644 --- a/pupil_src/shared_modules/plugin_manager.py +++ b/pupil_src/shared_modules/plugin_manager.py @@ -8,6 +8,7 @@ See COPYING and COPYING.LESSER for license details. ---------------------------------------------------------------------------~(*) """ +import logging from plugin import System_Plugin_Base from pyglui import ui @@ -16,6 +17,9 @@ from video_capture import Base_Manager, Base_Source +logger = logging.getLogger(__name__) + + class Plugin_Manager(System_Plugin_Base): icon_chr = chr(0xE8C0) icon_font = "pupil_icons" @@ -29,13 +33,21 @@ def __init__(self, g_pool): CalibrationChoreographyPlugin, GazerBase, ) - self.user_plugins = [ - p - for p in sorted( - g_pool.plugin_by_name.values(), key=lambda p: p.__name__.lower() - ) - if not issubclass(p, non_user_plugins) - ] + all_available_plugins = sorted( + g_pool.plugin_by_name.values(), key=lambda p: p.__name__.lower() + ) + + available_and_supported_user_plugins = [] + + for plugin in all_available_plugins: + if issubclass(plugin, non_user_plugins): + continue + if not plugin.is_available_within_context(g_pool): + logger.debug(f"Plugin {plugin.__name__} not available; skip adding it to plugin list.") + continue + available_and_supported_user_plugins.append(plugin) + + self.user_plugins = available_and_supported_user_plugins def init_ui(self): self.add_menu() From 01e285daf12d914814037cefbb3ffe3fc97c8d45 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Tue, 23 Feb 2021 12:56:30 +0100 Subject: [PATCH 04/32] Export RecordingInfo from pupil_recording --- pupil_src/shared_modules/pupil_recording/__init__.py | 2 +- pupil_src/shared_modules/pupil_recording/info/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/pupil_recording/__init__.py b/pupil_src/shared_modules/pupil_recording/__init__.py index 74e0b7af53..032d1aa0fd 100644 --- a/pupil_src/shared_modules/pupil_recording/__init__.py +++ b/pupil_src/shared_modules/pupil_recording/__init__.py @@ -8,6 +8,6 @@ See COPYING and COPYING.LESSER for license details. ---------------------------------------------------------------------------~(*) """ -from .info import RecordingInfoFile +from .info import RecordingInfoFile, RecordingInfo from .recording import PupilRecording from .recording_utils import InvalidRecordingException, assert_valid_recording_type diff --git a/pupil_src/shared_modules/pupil_recording/info/__init__.py b/pupil_src/shared_modules/pupil_recording/info/__init__.py index a53b7d8a49..3b74bf5637 100644 --- a/pupil_src/shared_modules/pupil_recording/info/__init__.py +++ b/pupil_src/shared_modules/pupil_recording/info/__init__.py @@ -8,7 +8,7 @@ See COPYING and COPYING.LESSER for license details. ---------------------------------------------------------------------------~(*) """ -from .recording_info import RecordingInfoFile +from .recording_info import RecordingInfoFile, RecordingInfo from .recording_info_2_0 import _RecordingInfoFile_2_0 from .recording_info_2_1 import _RecordingInfoFile_2_1 from .recording_info_2_2 import _RecordingInfoFile_2_2 From 3c1a3172f52354c2e3000d75deb65bd2a28a2822 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Tue, 23 Feb 2021 15:17:30 +0100 Subject: [PATCH 05/32] Mark Offline_Blink_Detection unavailable for Pupil Invisible recordings --- pupil_src/shared_modules/blink_detection.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pupil_src/shared_modules/blink_detection.py b/pupil_src/shared_modules/blink_detection.py index 35826eac94..5252ae8fe2 100644 --- a/pupil_src/shared_modules/blink_detection.py +++ b/pupil_src/shared_modules/blink_detection.py @@ -28,6 +28,7 @@ import player_methods as pm from observable import Observable from plugin import Plugin +from pupil_recording import PupilRecording, RecordingInfo logger = logging.getLogger(__name__) @@ -185,6 +186,17 @@ def get_init_dict(self): class Offline_Blink_Detection(Observable, Blink_Detection): + + @classmethod + def is_available_within_context(cls, g_pool) -> bool: + if g_pool.app == "player": + recording = PupilRecording(rec_dir=g_pool.rec_dir) + meta_info = recording.meta_info + if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE: + # Disable blink detector in Player if Pupil Invisible recording + return False + return super().is_available_within_context(g_pool) + def __init__( self, g_pool, From d3bfe2b7333a48550e3208386095c5615f2af9cc Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Tue, 23 Feb 2021 15:18:15 +0100 Subject: [PATCH 06/32] Mark Offline_Fixation_Detector unavailable for Pupil Invisible recordings --- pupil_src/shared_modules/fixation_detector.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pupil_src/shared_modules/fixation_detector.py b/pupil_src/shared_modules/fixation_detector.py index 115c96b86c..a9de2054a0 100644 --- a/pupil_src/shared_modules/fixation_detector.py +++ b/pupil_src/shared_modules/fixation_detector.py @@ -50,6 +50,7 @@ import player_methods as pm from methods import denormalize from plugin import Plugin +from pupil_recording import PupilRecording, RecordingInfo logger = logging.getLogger(__name__) @@ -269,6 +270,16 @@ class Offline_Fixation_Detector(Observable, Fixation_Detector_Base): fixations will have their method field set to "gaze". """ + @classmethod + def is_available_within_context(cls, g_pool) -> bool: + if g_pool.app == "player": + recording = PupilRecording(rec_dir=g_pool.rec_dir) + meta_info = recording.meta_info + if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE: + # Disable fixation detector in Player if Pupil Invisible recording + return False + return super().is_available_within_context(g_pool) + def __init__( self, g_pool, From 7e14c94224861f22a9f2be085db74cc916f2eea5 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 24 Feb 2021 09:37:04 +0100 Subject: [PATCH 07/32] Mark Offline_Pupil_Detection unavailable for Pupil Invisible recordings --- pupil_src/shared_modules/pupil_producers.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pupil_src/shared_modules/pupil_producers.py b/pupil_src/shared_modules/pupil_producers.py index 2e1557ed12..e2d84d3ec6 100644 --- a/pupil_src/shared_modules/pupil_producers.py +++ b/pupil_src/shared_modules/pupil_producers.py @@ -27,6 +27,7 @@ import zmq_tools from observable import Observable from plugin import System_Plugin_Base +from pupil_recording import PupilRecording, RecordingInfo from pyglui import ui from pyglui.pyfontstash import fontstash as fs from video_capture.utils import VideoSet @@ -312,6 +313,16 @@ class Offline_Pupil_Detection(Pupil_Producer_Base): session_data_version = 4 session_data_name = "offline_pupil" + @classmethod + def is_available_within_context(cls, g_pool) -> bool: + if g_pool.app == "player": + recording = PupilRecording(rec_dir=g_pool.rec_dir) + meta_info = recording.meta_info + if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE: + # Disable post-hoc pupil detector in Player if Pupil Invisible recording + return False + return super().is_available_within_context(g_pool) + @classmethod def plugin_menu_label(cls) -> str: return "Post-Hoc Pupil Detection" From f2c61dfad3e1ed0da6ca368169b577e2a0c3454f Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 24 Feb 2021 09:37:45 +0100 Subject: [PATCH 08/32] Mark GazeFromOfflineCalibration unavailable for Pupil Invisible recordings --- .../gaze_producer/gaze_from_offline_calibration.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/gaze_producer/gaze_from_offline_calibration.py b/pupil_src/shared_modules/gaze_producer/gaze_from_offline_calibration.py index 7a128dd58e..18b9cd44eb 100644 --- a/pupil_src/shared_modules/gaze_producer/gaze_from_offline_calibration.py +++ b/pupil_src/shared_modules/gaze_producer/gaze_from_offline_calibration.py @@ -13,7 +13,7 @@ from gaze_producer import ui as plugin_ui from gaze_producer.gaze_producer_base import GazeProducerBase from plugin_timeline import PluginTimeline -from pupil_recording import PupilRecording +from pupil_recording import PupilRecording, RecordingInfo from tasklib.manager import UniqueTaskManager @@ -23,6 +23,16 @@ class GazeFromOfflineCalibration(GazeProducerBase): icon_chr = chr(0xEC14) icon_font = "pupil_icons" + @classmethod + def is_available_within_context(cls, g_pool) -> bool: + if g_pool.app == "player": + recording = PupilRecording(rec_dir=g_pool.rec_dir) + meta_info = recording.meta_info + if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE: + # Disable post-hoc gaze calibration in Player if Pupil Invisible recording + return False + return super().is_available_within_context(g_pool) + @classmethod def plugin_menu_label(cls) -> str: return "Post-Hoc Gaze Calibration" From 164f55942635f33546c28c32d0652e69c4dd4e0a Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 24 Feb 2021 09:38:32 +0100 Subject: [PATCH 09/32] Mark Audio_Playback unavailable for Pupil Mobile recordings --- pupil_src/shared_modules/audio_playback.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pupil_src/shared_modules/audio_playback.py b/pupil_src/shared_modules/audio_playback.py index 22cd749b5f..13d06f8d66 100644 --- a/pupil_src/shared_modules/audio_playback.py +++ b/pupil_src/shared_modules/audio_playback.py @@ -26,6 +26,7 @@ import gl_utils from audio_utils import Audio_Viz_Transform, NoAudioLoadedError, load_audio from plugin import System_Plugin_Base +from pupil_recording import PupilRecording, RecordingInfo from version_utils import parse_version @@ -54,6 +55,16 @@ class Audio_Playback(System_Plugin_Base): icon_chr = chr(0xE050) icon_font = "pupil_icons" + @classmethod + def is_available_within_context(cls, g_pool) -> bool: + if g_pool.app == "player": + recording = PupilRecording(rec_dir=g_pool.rec_dir) + meta_info = recording.meta_info + if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_MOBILE: + # Disable audio playback in Player if Pupil Mobile recording + return False + return super().is_available_within_context(g_pool) + def __init__(self, g_pool): super().__init__(g_pool) From 2ffb44ece3cada17c38d6d973f31aca92afe8288 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 24 Feb 2021 09:39:34 +0100 Subject: [PATCH 10/32] Mark Pupil_From_Recording unavailable for Pupil Mobile recordings --- pupil_src/shared_modules/pupil_producers.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pupil_src/shared_modules/pupil_producers.py b/pupil_src/shared_modules/pupil_producers.py index e2d84d3ec6..896bd8fa53 100644 --- a/pupil_src/shared_modules/pupil_producers.py +++ b/pupil_src/shared_modules/pupil_producers.py @@ -286,6 +286,17 @@ def _legend_font(self, scale): class Pupil_From_Recording(Pupil_Producer_Base): + + @classmethod + def is_available_within_context(cls, g_pool) -> bool: + if g_pool.app == "player": + recording = PupilRecording(rec_dir=g_pool.rec_dir) + meta_info = recording.meta_info + if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_MOBILE: + # Disable pupil from recording in Player if Pupil Mobile recording + return False + return super().is_available_within_context(g_pool) + @classmethod def plugin_menu_label(cls) -> str: return "Pupil Data From Recording" From 2ae560a8b266eb6e4ea88580c4216b0747045f98 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 24 Feb 2021 09:39:50 +0100 Subject: [PATCH 11/32] Mark GazeFromRecording unavailable for Pupil Mobile recordings --- .../gaze_producer/gaze_from_recording.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pupil_src/shared_modules/gaze_producer/gaze_from_recording.py b/pupil_src/shared_modules/gaze_producer/gaze_from_recording.py index 01d9bd9e26..07d1f294e5 100644 --- a/pupil_src/shared_modules/gaze_producer/gaze_from_recording.py +++ b/pupil_src/shared_modules/gaze_producer/gaze_from_recording.py @@ -13,9 +13,21 @@ import file_methods as fm import player_methods as pm from gaze_producer.gaze_producer_base import GazeProducerBase +from pupil_recording import PupilRecording, RecordingInfo class GazeFromRecording(GazeProducerBase): + + @classmethod + def is_available_within_context(cls, g_pool) -> bool: + if g_pool.app == "player": + recording = PupilRecording(rec_dir=g_pool.rec_dir) + meta_info = recording.meta_info + if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_MOBILE: + # Disable gaze from recording in Player if Pupil Mobile recording + return False + return super().is_available_within_context(g_pool) + @classmethod def plugin_menu_label(cls) -> str: return "Gaze Data From Recording" From a31127ffb91613be5377dcc9ead5fa241073f592 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 24 Feb 2021 09:41:11 +0100 Subject: [PATCH 12/32] Don't show unavailable pupil/gaze producers in selection dropdown --- pupil_src/shared_modules/gaze_producer/gaze_producer_base.py | 5 +++++ pupil_src/shared_modules/pupil_producers.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/pupil_src/shared_modules/gaze_producer/gaze_producer_base.py b/pupil_src/shared_modules/gaze_producer/gaze_producer_base.py index bc0f41c8a1..a8890dc1c4 100644 --- a/pupil_src/shared_modules/gaze_producer/gaze_producer_base.py +++ b/pupil_src/shared_modules/gaze_producer/gaze_producer_base.py @@ -56,6 +56,11 @@ def _create_plugin_selector(self): for p in self.g_pool.plugin_by_name.values() if issubclass(p, GazeProducerBase) ] + # Skip gaze producers that are not available within g_pool context + gaze_producer_plugins = [ + p for p in gaze_producer_plugins + if p.is_available_within_context(self.g_pool) + ] gaze_producer_plugins.sort(key=lambda p: p.gaze_data_source_selection_label()) gaze_producer_plugins.sort(key=lambda p: p.gaze_data_source_selection_order()) gaze_producer_labels = [ diff --git a/pupil_src/shared_modules/pupil_producers.py b/pupil_src/shared_modules/pupil_producers.py index 896bd8fa53..b287a27d21 100644 --- a/pupil_src/shared_modules/pupil_producers.py +++ b/pupil_src/shared_modules/pupil_producers.py @@ -81,6 +81,11 @@ def init_ui(self): for p in self.g_pool.plugin_by_name.values() if issubclass(p, Pupil_Producer_Base) ] + # Skip pupil producers that are not available within g_pool context + pupil_producer_plugins = [ + p for p in pupil_producer_plugins + if p.is_available_within_context(self.g_pool) + ] pupil_producer_plugins.sort(key=lambda p: p.pupil_data_source_selection_label()) pupil_producer_plugins.sort(key=lambda p: p.pupil_data_source_selection_order()) pupil_producer_labels = [ From 8525ec5fd06a32ee4014a35f7cdaf54fefcccffe Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 24 Feb 2021 09:42:02 +0100 Subject: [PATCH 13/32] Add all pupil/gaze producers to default plugin list in Player --- pupil_src/launchables/player.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/pupil_src/launchables/player.py b/pupil_src/launchables/player.py index 949bf3c6d4..0fddbe0f78 100644 --- a/pupil_src/launchables/player.py +++ b/pupil_src/launchables/player.py @@ -572,6 +572,16 @@ def set_window_size(): g_pool.gui.append(g_pool.quickbar) # we always load these plugins + _pupil_producer_plugins = [ + # In priority order (first is default) + ("Pupil_From_Recording", {}), + ("Offline_Pupil_Detection", {}), + ] + _gaze_producer_plugins = [ + # In priority order (first is default) + ("GazeFromRecording", {}), + ("GazeFromOfflineCalibration", {}), + ] default_plugins = [ ("Plugin_Manager", {}), ("Seek_Control", {}), @@ -582,8 +592,8 @@ def set_window_size(): ("System_Graphs", {}), ("System_Timelines", {}), ("World_Video_Exporter", {}), - ("Pupil_From_Recording", {}), - ("GazeFromRecording", {}), + *reversed(_pupil_producer_plugins), + *reversed(_gaze_producer_plugins), ("Audio_Playback", {}), ] From 85760da48ec9d6ed28a20458add49e96240818cf Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 24 Feb 2021 09:42:49 +0100 Subject: [PATCH 14/32] Filter out unavailable plugins on Plugin_List.__init__ before attempting to add --- pupil_src/shared_modules/plugin.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pupil_src/shared_modules/plugin.py b/pupil_src/shared_modules/plugin.py index a2ab5adf71..19825c403a 100644 --- a/pupil_src/shared_modules/plugin.py +++ b/pupil_src/shared_modules/plugin.py @@ -371,6 +371,14 @@ def __init__(self, g_pool, plugin_initializers): expanded_initializers.sort(key=lambda data: data[0].order) + # skip plugins that are not available within g_pool context + # not removing them here will break the uniqueness logic bellow + expanded_initializers = [ + (plugin, name, args) + for (plugin, name, args) in expanded_initializers + if plugin.is_available_within_context(self.g_pool) + ] + # only add plugins that won't be replaced by newer plugins for i, (plugin, name, args) in enumerate(expanded_initializers): for new_plugin, new_name, _ in expanded_initializers[i + 1 :]: From deb7a44451b61f63b71b6943d0881c885f77c5d2 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 24 Feb 2021 10:00:26 +0100 Subject: [PATCH 15/32] Change plugin availability guard log message level Co-authored-by: Pablo Prietz --- pupil_src/shared_modules/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/plugin.py b/pupil_src/shared_modules/plugin.py index 19825c403a..f3c903f768 100644 --- a/pupil_src/shared_modules/plugin.py +++ b/pupil_src/shared_modules/plugin.py @@ -411,7 +411,7 @@ def add(self, new_plugin_cls, args={}): # Check if the plugin class is supported within the current g_pool context if not new_plugin_cls.is_available_within_context(self.g_pool): - logger.error(f"Plugin {new_plugin_cls.__name__} not available; skip adding it to plugin list.") + logger.debug(f"Plugin {new_plugin_cls.__name__} not available; skip adding it to plugin list.") return self._find_and_remove_duplicates(new_plugin_cls) From b1d22a2cb57793af1288c0f76fea7eb59d7fdcb6 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 24 Feb 2021 20:03:18 +0100 Subject: [PATCH 16/32] Apply black formatting --- pupil_src/shared_modules/audio_playback.py | 5 ++++- pupil_src/shared_modules/blink_detection.py | 6 ++++-- pupil_src/shared_modules/fixation_detector.py | 5 ++++- .../gaze_producer/gaze_from_offline_calibration.py | 5 ++++- .../gaze_producer/gaze_from_recording.py | 6 ++++-- .../gaze_producer/gaze_producer_base.py | 3 ++- pupil_src/shared_modules/plugin.py | 4 +++- pupil_src/shared_modules/plugin_manager.py | 4 +++- pupil_src/shared_modules/pupil_producers.py | 14 ++++++++++---- 9 files changed, 38 insertions(+), 14 deletions(-) diff --git a/pupil_src/shared_modules/audio_playback.py b/pupil_src/shared_modules/audio_playback.py index 13d06f8d66..50b9b660fc 100644 --- a/pupil_src/shared_modules/audio_playback.py +++ b/pupil_src/shared_modules/audio_playback.py @@ -60,7 +60,10 @@ def is_available_within_context(cls, g_pool) -> bool: if g_pool.app == "player": recording = PupilRecording(rec_dir=g_pool.rec_dir) meta_info = recording.meta_info - if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_MOBILE: + if ( + meta_info.recording_software_name + == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_MOBILE + ): # Disable audio playback in Player if Pupil Mobile recording return False return super().is_available_within_context(g_pool) diff --git a/pupil_src/shared_modules/blink_detection.py b/pupil_src/shared_modules/blink_detection.py index 5252ae8fe2..5abddeff4f 100644 --- a/pupil_src/shared_modules/blink_detection.py +++ b/pupil_src/shared_modules/blink_detection.py @@ -186,13 +186,15 @@ def get_init_dict(self): class Offline_Blink_Detection(Observable, Blink_Detection): - @classmethod def is_available_within_context(cls, g_pool) -> bool: if g_pool.app == "player": recording = PupilRecording(rec_dir=g_pool.rec_dir) meta_info = recording.meta_info - if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE: + if ( + meta_info.recording_software_name + == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE + ): # Disable blink detector in Player if Pupil Invisible recording return False return super().is_available_within_context(g_pool) diff --git a/pupil_src/shared_modules/fixation_detector.py b/pupil_src/shared_modules/fixation_detector.py index a9de2054a0..4853f89915 100644 --- a/pupil_src/shared_modules/fixation_detector.py +++ b/pupil_src/shared_modules/fixation_detector.py @@ -275,7 +275,10 @@ def is_available_within_context(cls, g_pool) -> bool: if g_pool.app == "player": recording = PupilRecording(rec_dir=g_pool.rec_dir) meta_info = recording.meta_info - if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE: + if ( + meta_info.recording_software_name + == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE + ): # Disable fixation detector in Player if Pupil Invisible recording return False return super().is_available_within_context(g_pool) diff --git a/pupil_src/shared_modules/gaze_producer/gaze_from_offline_calibration.py b/pupil_src/shared_modules/gaze_producer/gaze_from_offline_calibration.py index 18b9cd44eb..22ec99aa06 100644 --- a/pupil_src/shared_modules/gaze_producer/gaze_from_offline_calibration.py +++ b/pupil_src/shared_modules/gaze_producer/gaze_from_offline_calibration.py @@ -28,7 +28,10 @@ def is_available_within_context(cls, g_pool) -> bool: if g_pool.app == "player": recording = PupilRecording(rec_dir=g_pool.rec_dir) meta_info = recording.meta_info - if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE: + if ( + meta_info.recording_software_name + == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE + ): # Disable post-hoc gaze calibration in Player if Pupil Invisible recording return False return super().is_available_within_context(g_pool) diff --git a/pupil_src/shared_modules/gaze_producer/gaze_from_recording.py b/pupil_src/shared_modules/gaze_producer/gaze_from_recording.py index 07d1f294e5..84fda54ad3 100644 --- a/pupil_src/shared_modules/gaze_producer/gaze_from_recording.py +++ b/pupil_src/shared_modules/gaze_producer/gaze_from_recording.py @@ -17,13 +17,15 @@ class GazeFromRecording(GazeProducerBase): - @classmethod def is_available_within_context(cls, g_pool) -> bool: if g_pool.app == "player": recording = PupilRecording(rec_dir=g_pool.rec_dir) meta_info = recording.meta_info - if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_MOBILE: + if ( + meta_info.recording_software_name + == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_MOBILE + ): # Disable gaze from recording in Player if Pupil Mobile recording return False return super().is_available_within_context(g_pool) diff --git a/pupil_src/shared_modules/gaze_producer/gaze_producer_base.py b/pupil_src/shared_modules/gaze_producer/gaze_producer_base.py index a8890dc1c4..fc0b342a1e 100644 --- a/pupil_src/shared_modules/gaze_producer/gaze_producer_base.py +++ b/pupil_src/shared_modules/gaze_producer/gaze_producer_base.py @@ -58,7 +58,8 @@ def _create_plugin_selector(self): ] # Skip gaze producers that are not available within g_pool context gaze_producer_plugins = [ - p for p in gaze_producer_plugins + p + for p in gaze_producer_plugins if p.is_available_within_context(self.g_pool) ] gaze_producer_plugins.sort(key=lambda p: p.gaze_data_source_selection_label()) diff --git a/pupil_src/shared_modules/plugin.py b/pupil_src/shared_modules/plugin.py index f3c903f768..0c9dee1505 100644 --- a/pupil_src/shared_modules/plugin.py +++ b/pupil_src/shared_modules/plugin.py @@ -411,7 +411,9 @@ def add(self, new_plugin_cls, args={}): # Check if the plugin class is supported within the current g_pool context if not new_plugin_cls.is_available_within_context(self.g_pool): - logger.debug(f"Plugin {new_plugin_cls.__name__} not available; skip adding it to plugin list.") + logger.debug( + f"Plugin {new_plugin_cls.__name__} not available; skip adding it to plugin list." + ) return self._find_and_remove_duplicates(new_plugin_cls) diff --git a/pupil_src/shared_modules/plugin_manager.py b/pupil_src/shared_modules/plugin_manager.py index 8dcd731624..b9626aff62 100644 --- a/pupil_src/shared_modules/plugin_manager.py +++ b/pupil_src/shared_modules/plugin_manager.py @@ -43,7 +43,9 @@ def __init__(self, g_pool): if issubclass(plugin, non_user_plugins): continue if not plugin.is_available_within_context(g_pool): - logger.debug(f"Plugin {plugin.__name__} not available; skip adding it to plugin list.") + logger.debug( + f"Plugin {plugin.__name__} not available; skip adding it to plugin list." + ) continue available_and_supported_user_plugins.append(plugin) diff --git a/pupil_src/shared_modules/pupil_producers.py b/pupil_src/shared_modules/pupil_producers.py index b287a27d21..afb7b83f42 100644 --- a/pupil_src/shared_modules/pupil_producers.py +++ b/pupil_src/shared_modules/pupil_producers.py @@ -83,7 +83,8 @@ def init_ui(self): ] # Skip pupil producers that are not available within g_pool context pupil_producer_plugins = [ - p for p in pupil_producer_plugins + p + for p in pupil_producer_plugins if p.is_available_within_context(self.g_pool) ] pupil_producer_plugins.sort(key=lambda p: p.pupil_data_source_selection_label()) @@ -291,13 +292,15 @@ def _legend_font(self, scale): class Pupil_From_Recording(Pupil_Producer_Base): - @classmethod def is_available_within_context(cls, g_pool) -> bool: if g_pool.app == "player": recording = PupilRecording(rec_dir=g_pool.rec_dir) meta_info = recording.meta_info - if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_MOBILE: + if ( + meta_info.recording_software_name + == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_MOBILE + ): # Disable pupil from recording in Player if Pupil Mobile recording return False return super().is_available_within_context(g_pool) @@ -334,7 +337,10 @@ def is_available_within_context(cls, g_pool) -> bool: if g_pool.app == "player": recording = PupilRecording(rec_dir=g_pool.rec_dir) meta_info = recording.meta_info - if meta_info.recording_software_name == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE: + if ( + meta_info.recording_software_name + == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_INVISIBLE + ): # Disable post-hoc pupil detector in Player if Pupil Invisible recording return False return super().is_available_within_context(g_pool) From c2e32c63dc8da653911678db632e588227bd3c6c Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 24 Feb 2021 20:04:37 +0100 Subject: [PATCH 17/32] Assign list instead of iterable to _pupil_producer_plugins/_gaze_producer_plugins --- pupil_src/launchables/player.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pupil_src/launchables/player.py b/pupil_src/launchables/player.py index 0fddbe0f78..6797ec222c 100644 --- a/pupil_src/launchables/player.py +++ b/pupil_src/launchables/player.py @@ -577,11 +577,13 @@ def set_window_size(): ("Pupil_From_Recording", {}), ("Offline_Pupil_Detection", {}), ] + _pupil_producer_plugins = list(reversed(_pupil_producer_plugins)) _gaze_producer_plugins = [ # In priority order (first is default) ("GazeFromRecording", {}), ("GazeFromOfflineCalibration", {}), ] + _gaze_producer_plugins = list(reversed(_gaze_producer_plugins)) default_plugins = [ ("Plugin_Manager", {}), ("Seek_Control", {}), @@ -592,8 +594,8 @@ def set_window_size(): ("System_Graphs", {}), ("System_Timelines", {}), ("World_Video_Exporter", {}), - *reversed(_pupil_producer_plugins), - *reversed(_gaze_producer_plugins), + *_pupil_producer_plugins, + *_gaze_producer_plugins, ("Audio_Playback", {}), ] From 27543a671e55dbb9ff844539f34d89ddb258e00e Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 24 Feb 2021 20:16:57 +0100 Subject: [PATCH 18/32] Add plugins required for core functionality but not shown in Plugin Manager to loaded_plugins --- pupil_src/launchables/player.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/pupil_src/launchables/player.py b/pupil_src/launchables/player.py index 6797ec222c..97069b43ec 100644 --- a/pupil_src/launchables/player.py +++ b/pupil_src/launchables/player.py @@ -598,10 +598,21 @@ def set_window_size(): *_gaze_producer_plugins, ("Audio_Playback", {}), ] - - g_pool.plugins = Plugin_List( - g_pool, session_settings.get("loaded_plugins", default_plugins) - ) + _plugins_to_load = session_settings.get("loaded_plugins", None) + if _plugins_to_load is None: + # If no plugins are available from a previous session, + # then use the default plugin list + _plugins_to_load = default_plugins + else: + # If there are plugins available from a previous session, + # then prepend plugins that are required, but might have not been available before + _plugins_to_load = [ + *_pupil_producer_plugins, + *_gaze_producer_plugins, + *_plugins_to_load, + ] + + g_pool.plugins = Plugin_List(g_pool, _plugins_to_load) # Manually add g_pool.capture to the plugin list g_pool.plugins._plugins.append(g_pool.capture) From 579632ed22fadde48929cfb768b7fbabba790a82 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Thu, 25 Feb 2021 10:02:02 +0100 Subject: [PATCH 19/32] Handle OSError expections raised by PortAudio --- pupil_src/shared_modules/audio_playback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/audio_playback.py b/pupil_src/shared_modules/audio_playback.py index 22cd749b5f..4e0cbf2120 100644 --- a/pupil_src/shared_modules/audio_playback.py +++ b/pupil_src/shared_modules/audio_playback.py @@ -164,7 +164,7 @@ def _setup_output_audio(self): self.audio_sync = self.pa_stream.get_output_latency() self.audio_reported_latency = self.pa_stream.get_output_latency() - except ValueError: + except (ValueError, OSError): self.pa_stream = None def _setup_audio_vis(self): From a818094bb37cca7303a56af32ff0eb9031971b79 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Thu, 25 Feb 2021 10:02:58 +0100 Subject: [PATCH 20/32] Guard audio_timeline from being None before calling refresh --- pupil_src/shared_modules/audio_playback.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/audio_playback.py b/pupil_src/shared_modules/audio_playback.py index 4e0cbf2120..6b2cf8fdc9 100644 --- a/pupil_src/shared_modules/audio_playback.py +++ b/pupil_src/shared_modules/audio_playback.py @@ -321,7 +321,7 @@ def update_audio_viz(self): self.audio_viz_data, finished = self.audio_viz_trans.get_data( log_scale=self.log_scale ) - if not finished: + if not finished and self.audio_timeline: self.audio_timeline.refresh() def setup_pyaudio_output_if_necessary(self): From 72b6ed8ac2ebab2f1ea7607f072b22ef27d921a7 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Thu, 25 Feb 2021 10:11:18 +0100 Subject: [PATCH 21/32] Undo marking Audio_Playback unavailable for Pupil Mobile recordings --- pupil_src/shared_modules/audio_playback.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/pupil_src/shared_modules/audio_playback.py b/pupil_src/shared_modules/audio_playback.py index 50b9b660fc..22cd749b5f 100644 --- a/pupil_src/shared_modules/audio_playback.py +++ b/pupil_src/shared_modules/audio_playback.py @@ -26,7 +26,6 @@ import gl_utils from audio_utils import Audio_Viz_Transform, NoAudioLoadedError, load_audio from plugin import System_Plugin_Base -from pupil_recording import PupilRecording, RecordingInfo from version_utils import parse_version @@ -55,19 +54,6 @@ class Audio_Playback(System_Plugin_Base): icon_chr = chr(0xE050) icon_font = "pupil_icons" - @classmethod - def is_available_within_context(cls, g_pool) -> bool: - if g_pool.app == "player": - recording = PupilRecording(rec_dir=g_pool.rec_dir) - meta_info = recording.meta_info - if ( - meta_info.recording_software_name - == RecordingInfo.RECORDING_SOFTWARE_NAME_PUPIL_MOBILE - ): - # Disable audio playback in Player if Pupil Mobile recording - return False - return super().is_available_within_context(g_pool) - def __init__(self, g_pool): super().__init__(g_pool) From 4f4b87d50e1215525a2321416e76ceb607f2e263 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Tue, 2 Mar 2021 22:20:56 +0100 Subject: [PATCH 22/32] Handle OSError in Audio_Playback._setup_output_audio separately --- pupil_src/shared_modules/audio_playback.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/audio_playback.py b/pupil_src/shared_modules/audio_playback.py index 6b2cf8fdc9..787b9e4be0 100644 --- a/pupil_src/shared_modules/audio_playback.py +++ b/pupil_src/shared_modules/audio_playback.py @@ -164,8 +164,14 @@ def _setup_output_audio(self): self.audio_sync = self.pa_stream.get_output_latency() self.audio_reported_latency = self.pa_stream.get_output_latency() - except (ValueError, OSError): + except ValueError: self.pa_stream = None + except OSError: + self.pa_stream = None + import traceback + + logger.warning("Audio found, but playback failed (#2103)") + logger.debug(traceback.format_exc()) def _setup_audio_vis(self): self.audio_timeline = None From 5409fffce0fd61b485c7c5958b9fa9e94a75655b Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 9 Mar 2021 14:50:36 +0100 Subject: [PATCH 23/32] Require pye3d v0.0.6 --- pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py b/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py index 3930f4df8b..34ad04a596 100644 --- a/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py +++ b/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py @@ -22,7 +22,7 @@ logger = logging.getLogger(__name__) version_installed = getattr(pye3d, "__version__", "0.0.1") -version_supported = "0.0.5" +version_supported = "0.0.6" if version_installed != version_supported: logger.info( diff --git a/requirements.txt b/requirements.txt index 60bcda1316..12ea29d8af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,7 +23,7 @@ opencv-python==3.* ; platform_system == "Windows" ### pupil-apriltags==1.0.4 pupil-detectors==2.0.* -pye3d==0.0.5 +pye3d==0.0.6 # pupil-labs/PyAV 0.4.6 av @ git+https://github.com/pupil-labs/PyAV@v0.4.6 ; platform_system != "Windows" From bce63fbd24ef536d8ffc3dfe5c6d1f56baa5ec97 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Wed, 3 Mar 2021 16:52:52 +0100 Subject: [PATCH 24/32] Fixes laggy playback of PI recordings after seeking. TODO: - [ ] Fix new frame drop every few seconds --- pupil_src/shared_modules/audio_playback.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pupil_src/shared_modules/audio_playback.py b/pupil_src/shared_modules/audio_playback.py index 787b9e4be0..5a6a0a4ec6 100644 --- a/pupil_src/shared_modules/audio_playback.py +++ b/pupil_src/shared_modules/audio_playback.py @@ -406,7 +406,7 @@ def fill_audio_queue(self): audio_buffer = bytes(audio_frame.planes[0]) audio_part_start_ts = self.audio.timestamps[0] - audio_part_progress = audio_frame.pts * self.audio.stream.time_base + audio_part_progress = audio_frame.pts * audio_frame_p.time_base audio_playback_time = audio_part_start_ts + audio_part_progress self.audio_bytes_fifo.append((audio_buffer, audio_playback_time)) @@ -472,12 +472,11 @@ def check_ts_consistency(self, reference_frame): ) if ( self.audio.timestamps[fnum] - != self.audio.timestamps[0] + af.pts * self.audio.stream.time_base + != self.audio.timestamps[0] + af.pts * af.time_base ): print( "ts[0] + af.pts = {} fnum = {} timestamp = {}".format( - self.audio.timestamps[0] - + af.pts * self.audio.stream.time_base, + self.audio.timestamps[0] + af.pts * af.time_base, fnum, self.audio.timestamps[fnum], ) From ac5f1dca6114337ea1e2c1440c0ec4d3a2168454 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Thu, 4 Mar 2021 16:47:24 +0100 Subject: [PATCH 25/32] Cache audio pts on load --- pupil_src/shared_modules/audio_utils.py | 34 +++++++++++++------------ 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/pupil_src/shared_modules/audio_utils.py b/pupil_src/shared_modules/audio_utils.py index 86df37cb22..0a8204185d 100644 --- a/pupil_src/shared_modules/audio_utils.py +++ b/pupil_src/shared_modules/audio_utils.py @@ -25,7 +25,7 @@ class NoAudioLoadedError(Exception): LoadedAudio = collections.namedtuple( - "LoadedAudio", ["container", "stream", "timestamps"] + "LoadedAudio", ["container", "stream", "timestamps", "pts"] ) @@ -71,23 +71,25 @@ def _load_audio_single(file_path, return_pts_based_timestamps=False): except IOError: return None + start = timestamps[0] + packet_pts = np.array( + [p.pts for p in container.demux(stream) if p is not None and p.pts is not None], + ) + if return_pts_based_timestamps: - start = timestamps[0] - timestamps = np.fromiter( - ( - float(start + p.pts * p.time_base) - for p in container.demux(audio=0) - if p is not None and p.pts is not None and p.time_base is not None - ), - dtype=float, - ) - try: - container.seek(0) - except av.AVError as err: - logger.debug(f"{err}") - return None + timestamps = start + packet_pts * stream.time_base + + # pts seeking requires Python ints; convert after `packet_pts * stream.time_base` + # to leverage numpy element-wise function application + packet_pts = packet_pts.tolist() + + try: + container.seek(0) + except av.AVError as err: + logger.debug(f"{err}") + return None - return LoadedAudio(container, stream, timestamps) + return LoadedAudio(container, stream, timestamps, packet_pts) class Audio_Viz_Transform: From ee58ead43d4b39967cf29cff6186f24b347adb3c Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Thu, 4 Mar 2021 16:51:58 +0100 Subject: [PATCH 26/32] Lookup pts in cache instead of calculating it from sampling rate Audio pts rate is not consistent with pts distribution, causing seeks to be off. --- pupil_src/shared_modules/audio_playback.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/pupil_src/shared_modules/audio_playback.py b/pupil_src/shared_modules/audio_playback.py index 5a6a0a4ec6..295126c5bb 100644 --- a/pupil_src/shared_modules/audio_playback.py +++ b/pupil_src/shared_modules/audio_playback.py @@ -123,17 +123,10 @@ def _setup_input_audio_part(self, part_idx): self.audio_paused = False self.audio.stream.seek(0) - first_frame = next(self.audio_frame_iterator) - self.audio_pts_rate = first_frame.samples - self.audio_start_pts = first_frame.pts - - logger.debug( - "audio_pts_rate = {} start_pts = {}".format( - self.audio_pts_rate, self.audio_start_pts - ) - ) - self.check_ts_consistency(reference_frame=first_frame) - self.seek_to_audio_frame(0) + if self.should_check_ts_consistency: + first_frame = next(self.audio_frame_iterator) + self.check_ts_consistency(reference_frame=first_frame) + self.seek_to_audio_frame(0) logger.debug( "Audio file format {} chans {} rate {} framesize {}".format( @@ -260,13 +253,11 @@ def get_audio_frame_iterator(self): yield frame def audio_idx_to_pts(self, idx): - return idx * self.audio_pts_rate + return self.audio.pts[idx] def seek_to_audio_frame(self, seek_pos): try: - self.audio.stream.seek( - self.audio_start_pts + self.audio_idx_to_pts(seek_pos) - ) + self.audio.stream.seek(self.audio_idx_to_pts(seek_pos)) except av.AVError: raise FileSeekError() else: From f76514baf9009d22ffeb4b159e7a8163dfb2a7fb Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Wed, 3 Mar 2021 16:52:52 +0100 Subject: [PATCH 27/32] Revert ts calculation based on frame timebase This reverts commit ef690a461bfb52a7b255b20e1bbb6ab5d29f0da3. --- pupil_src/shared_modules/audio_playback.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pupil_src/shared_modules/audio_playback.py b/pupil_src/shared_modules/audio_playback.py index 295126c5bb..6a394ade7f 100644 --- a/pupil_src/shared_modules/audio_playback.py +++ b/pupil_src/shared_modules/audio_playback.py @@ -397,7 +397,7 @@ def fill_audio_queue(self): audio_buffer = bytes(audio_frame.planes[0]) audio_part_start_ts = self.audio.timestamps[0] - audio_part_progress = audio_frame.pts * audio_frame_p.time_base + audio_part_progress = audio_frame.pts * self.audio.stream.time_base audio_playback_time = audio_part_start_ts + audio_part_progress self.audio_bytes_fifo.append((audio_buffer, audio_playback_time)) @@ -463,11 +463,12 @@ def check_ts_consistency(self, reference_frame): ) if ( self.audio.timestamps[fnum] - != self.audio.timestamps[0] + af.pts * af.time_base + != self.audio.timestamps[0] + af.pts * self.audio.stream.time_base ): print( "ts[0] + af.pts = {} fnum = {} timestamp = {}".format( - self.audio.timestamps[0] + af.pts * af.time_base, + self.audio.timestamps[0] + + af.pts * self.audio.stream.time_base, fnum, self.audio.timestamps[fnum], ) From 04951a194604bde3d00747198ec424968d2f5afa Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 9 Mar 2021 11:11:04 +0100 Subject: [PATCH 28/32] Add more detail to inline comment --- pupil_src/shared_modules/audio_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/audio_utils.py b/pupil_src/shared_modules/audio_utils.py index 0a8204185d..f5be657743 100644 --- a/pupil_src/shared_modules/audio_utils.py +++ b/pupil_src/shared_modules/audio_utils.py @@ -79,8 +79,9 @@ def _load_audio_single(file_path, return_pts_based_timestamps=False): if return_pts_based_timestamps: timestamps = start + packet_pts * stream.time_base - # pts seeking requires Python ints; convert after `packet_pts * stream.time_base` - # to leverage numpy element-wise function application + # pts seeking requires primitive Python integers and does not accept numpy int types; + # `.tolist()` converts numpy integers to primitive Python integers; do conversion after + # `packet_pts * stream.time_base` to leverage numpy element-wise function application packet_pts = packet_pts.tolist() try: From 9b233399b3eb325a2c8a2c19641d843bae0eae6c Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 9 Mar 2021 16:28:25 +0100 Subject: [PATCH 29/32] Accuracy Visualizer: Inherit CalculationResult from T.NamedTuple --- pupil_src/shared_modules/accuracy_visualizer.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pupil_src/shared_modules/accuracy_visualizer.py b/pupil_src/shared_modules/accuracy_visualizer.py index fb4d6f0a17..9f4d070aaa 100644 --- a/pupil_src/shared_modules/accuracy_visualizer.py +++ b/pupil_src/shared_modules/accuracy_visualizer.py @@ -36,9 +36,10 @@ logger = logging.getLogger(__name__) -Calculation_Result = namedtuple( - "Calculation_Result", ["result", "num_used", "num_total"] -) +class CalculationResult(T.NamedTuple): + result: float + num_used: int + num_total: int class ValidationInput: @@ -394,8 +395,8 @@ def calc_acc_prec_errlines( [(*e["ref"]["norm_pos"], *e["pupil"]["norm_pos"]) for e in correlated] ) if locations.size == 0: - accuracy_result = Calculation_Result(0.0, 0, 0) - precision_result = Calculation_Result(0.0, 0, 0) + accuracy_result = CalculationResult(0.0, 0, 0) + precision_result = CalculationResult(0.0, 0, 0) error_lines = np.array([]) return accuracy_result, precision_result, error_lines error_lines = locations.copy() # n x 4 @@ -426,7 +427,7 @@ def calc_acc_prec_errlines( -1, 2 ) # shape: num_used x 2 accuracy = np.rad2deg(np.arccos(selected_samples.clip(-1.0, 1.0).mean())) - accuracy_result = Calculation_Result(accuracy, num_used, num_total) + accuracy_result = CalculationResult(accuracy, num_used, num_total) # lets calculate precision: (RMS of distance of succesive samples.) # This is a little rough as we do not compensate headmovements in this test. @@ -457,7 +458,7 @@ def calc_acc_prec_errlines( precision = np.sqrt( np.mean(np.rad2deg(np.arccos(succesive_distances.clip(-1.0, 1.0))) ** 2) ) - precision_result = Calculation_Result(precision, num_used, num_total) + precision_result = CalculationResult(precision, num_used, num_total) return accuracy_result, precision_result, error_lines From 5735d0e9ba54502273e4a2ca7fdfb10e10714f40 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 9 Mar 2021 19:47:33 +0100 Subject: [PATCH 30/32] Accuracy Visualizer: Extract correlation and coordinate transformation --- .../shared_modules/accuracy_visualizer.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/pupil_src/shared_modules/accuracy_visualizer.py b/pupil_src/shared_modules/accuracy_visualizer.py index 9f4d070aaa..bbc882b6c3 100644 --- a/pupil_src/shared_modules/accuracy_visualizer.py +++ b/pupil_src/shared_modules/accuracy_visualizer.py @@ -36,12 +36,34 @@ logger = logging.getLogger(__name__) + class CalculationResult(T.NamedTuple): result: float num_used: int num_total: int +class CorrelatedAndCoordinateTransformedResult(T.NamedTuple): + """Holds result from correlating reference and gaze data and their respective + transformations into norm, image, and camera coordinate systems. + """ + + norm_space: np.ndarray # shape: 2*n, 2 + image_space: np.ndarray # shape: 2*n, 2 + camera_space: np.ndarray # shape: 2*n, 3 + + @staticmethod + def empty() -> "CorrelatedAndCoordinateTransformedResult": + return CorrelatedAndCoordinateTransformedResult( + norm_space=np.ndarray([]), + image_space=np.ndarray([]), + camera_space=np.ndarray([]), + ) + + +class CorrelationError(ValueError): + pass + class ValidationInput: def __init__(self): self.clear() @@ -461,6 +483,83 @@ def calc_acc_prec_errlines( precision_result = CalculationResult(precision, num_used, num_total) return accuracy_result, precision_result, error_lines + @staticmethod + def correlate_and_coordinate_transform( + gaze_pos, ref_pos, intrinsics + ) -> CorrelatedAndCoordinateTransformedResult: + # reuse closest_matches_monocular to correlate one label to each prediction + # correlated['ref']: prediction, correlated['pupil']: label location + # NOTE the switch of the ref and pupil keys! This effects mostly hmd data. + correlated = closest_matches_monocular(gaze_pos, ref_pos) + # [[pred.x, pred.y, label.x, label.y], ...], shape: n x 4 + if not correlated: + raise CorrelationError("No correlation possible") + + try: + return Accuracy_Visualizer._coordinate_transform_ref_in_norm_space( + correlated, intrinsics + ) + except KeyError as err: + if "norm_pos" in err.args: + return Accuracy_Visualizer._coordinate_transform_ref_in_camera_space( + correlated, intrinsics + ) + else: + raise + + @staticmethod + def _coordinate_transform_ref_in_norm_space( + correlated, intrinsics + ) -> CorrelatedAndCoordinateTransformedResult: + width, height = intrinsics.resolution + locations_norm = np.array( + [(*e["ref"]["norm_pos"], *e["pupil"]["norm_pos"]) for e in correlated] + ) + locations_image = locations_norm.copy() # n x 4 + locations_image[:, ::2] *= width + locations_image[:, 1::2] = (1.0 - locations_image[:, 1::2]) * height + locations_image.shape = -1, 2 + locations_norm.shape = -1, 2 + locations_camera = intrinsics.unprojectPoints(locations_image, normalize=True) + return CorrelatedAndCoordinateTransformedResult( + locations_norm, locations_image, locations_camera + ) + + @staticmethod + def _coordinate_transform_ref_in_camera_space( + correlated, intrinsics + ) -> CorrelatedAndCoordinateTransformedResult: + width, height = intrinsics.resolution + locations_mixed = np.array( + # NOTE: This looks incorrect, but is actually correct. The switch comes from + # using closest_matches_monocular() above with switched arguments. + [(*e["ref"]["norm_pos"], *e["pupil"]["mm_pos"]) for e in correlated] + ) # n x 5 + pupil_norm = locations_mixed[:, 0:2] # n x 2 + pupil_image = pupil_norm.copy() + pupil_image[:, 0] *= width + pupil_image[:, 1] = (1.0 - pupil_image[:, 1]) * height + pupil_camera = intrinsics.unprojectPoints(pupil_image, normalize=True) # n x 3 + + ref_camera = locations_mixed[:, 2:5] # n x 3 + ref_camera /= np.linalg.norm(ref_camera, axis=1, keepdims=True) + ref_image = intrinsics.projectPoints(ref_camera) # n x 2 + ref_norm = ref_image.copy() + ref_norm[:, 0] /= width + ref_norm[:, 1] = 1.0 - (ref_norm[:, 1] / height) + + locations_norm = np.hstack([pupil_norm, ref_norm]) # n x 4 + locations_norm.shape = -1, 2 + + locations_image = np.hstack([pupil_image, ref_image]) # n x 4 + locations_image.shape = -1, 2 + + locations_camera = np.hstack([pupil_camera, ref_camera]) # n x 6 + locations_camera.shape = -1, 3 + + return CorrelatedAndCoordinateTransformedResult( + locations_norm, locations_image, locations_camera + ) def gl_display(self): if self.vis_mapping_error and self.error_lines is not None: From 4ea857d8ba3a137ce0be078097721f2b57910398 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 9 Mar 2021 19:57:04 +0100 Subject: [PATCH 31/32] Use extracted method to correlate and transform data --- .../shared_modules/accuracy_visualizer.py | 60 +++++++++++-------- .../gaze_producer/worker/validate_gaze.py | 4 +- 2 files changed, 36 insertions(+), 28 deletions(-) diff --git a/pupil_src/shared_modules/accuracy_visualizer.py b/pupil_src/shared_modules/accuracy_visualizer.py index bbc882b6c3..17e1aafce1 100644 --- a/pupil_src/shared_modules/accuracy_visualizer.py +++ b/pupil_src/shared_modules/accuracy_visualizer.py @@ -64,6 +64,23 @@ def empty() -> "CorrelatedAndCoordinateTransformedResult": class CorrelationError(ValueError): pass + +class AccuracyPrecisionResult(T.NamedTuple): + accuracy: CalculationResult + precision: CalculationResult + error_lines: np.ndarray + correlation: CorrelatedAndCoordinateTransformedResult + + @staticmethod + def failed() -> "AccuracyPrecisionResult": + return AccuracyPrecisionResult( + accuracy=CalculationResult(0.0, 0, 0), + precision=CalculationResult(0.0, 0, 0), + error_lines=np.array([]), + correlation=CorrelatedAndCoordinateTransformedResult.empty(), + ) + + class ValidationInput: def __init__(self): self.clear() @@ -360,7 +377,7 @@ def recalculate(self): succession_threshold=self.succession_threshold, ) - accuracy = results[0].result + accuracy = results.accuracy.result if np.isnan(accuracy): self.accuracy = None logger.warning( @@ -372,7 +389,7 @@ def recalculate(self): "Angular accuracy: {}. Used {} of {} samples.".format(*results[0]) ) - precision = results[1].result + precision = results.precision.result if np.isnan(precision): self.precision = None logger.warning( @@ -384,9 +401,8 @@ def recalculate(self): "Angular precision: {}. Used {} of {} samples.".format(*results[1]) ) - self.error_lines = results[2] - - ref_locations = [loc["norm_pos"] for loc in self.recent_input.ref_list] + self.error_lines = results.error_lines + ref_locations = results.correlation.norm_space[1::2, :] if len(ref_locations) >= 3: hull = ConvexHull(ref_locations) # requires at least 3 points self.calibration_area = hull.points[hull.vertices, :] @@ -401,36 +417,25 @@ def calc_acc_prec_errlines( intrinsics, outlier_threshold, succession_threshold=np.cos(np.deg2rad(0.5)), - ): + ) -> AccuracyPrecisionResult: gazer = gazer_class(g_pool, params=gazer_params) gaze_pos = gazer.map_pupil_to_gaze(pupil_list) ref_pos = ref_list - width, height = intrinsics.resolution - - # reuse closest_matches_monocular to correlate one label to each prediction - # correlated['ref']: prediction, correlated['pupil']: label location - correlated = closest_matches_monocular(gaze_pos, ref_pos) - # [[pred.x, pred.y, label.x, label.y], ...], shape: n x 4 - locations = np.array( - [(*e["ref"]["norm_pos"], *e["pupil"]["norm_pos"]) for e in correlated] - ) - if locations.size == 0: - accuracy_result = CalculationResult(0.0, 0, 0) - precision_result = CalculationResult(0.0, 0, 0) - error_lines = np.array([]) - return accuracy_result, precision_result, error_lines - error_lines = locations.copy() # n x 4 - locations[:, ::2] *= width - locations[:, 1::2] = (1.0 - locations[:, 1::2]) * height - locations.shape = -1, 2 + try: + correlation_result = Accuracy_Visualizer.correlate_and_coordinate_transform( + gaze_pos, ref_pos, intrinsics + ) + error_lines = correlation_result.norm_space.reshape(-1, 4) + undistorted_3d = correlation_result.camera_space + except CorrelationError: + return AccuracyPrecisionResult.failed() # Accuracy is calculated as the average angular # offset (distance) (in degrees of visual angle) # between fixations locations and the corresponding # locations of the fixation targets. - undistorted_3d = intrinsics.unprojectPoints(locations, normalize=True) # Cosine distance of A and B: (A @ B) / (||A|| * ||B||) # No need to calculate norms, since A and B are normalized in our case. @@ -482,7 +487,10 @@ def calc_acc_prec_errlines( ) precision_result = CalculationResult(precision, num_used, num_total) - return accuracy_result, precision_result, error_lines + return AccuracyPrecisionResult( + accuracy_result, precision_result, error_lines, correlation_result + ) + @staticmethod def correlate_and_coordinate_transform( gaze_pos, ref_pos, intrinsics diff --git a/pupil_src/shared_modules/gaze_producer/worker/validate_gaze.py b/pupil_src/shared_modules/gaze_producer/worker/validate_gaze.py index 1bb693bafe..6115e12562 100644 --- a/pupil_src/shared_modules/gaze_producer/worker/validate_gaze.py +++ b/pupil_src/shared_modules/gaze_producer/worker/validate_gaze.py @@ -72,7 +72,7 @@ def validate( for ref in refs_in_validation_range ] - accuracy_result, precision_result, _ = Accuracy_Visualizer.calc_acc_prec_errlines( + result = Accuracy_Visualizer.calc_acc_prec_errlines( g_pool=g_pool, gazer_class=gazer_class, gazer_params=gazer_params, @@ -81,7 +81,7 @@ def validate( intrinsics=g_pool.capture.intrinsics, outlier_threshold=gaze_mapper.validation_outlier_threshold_deg, ) - return accuracy_result, precision_result + return result.accuracy, result.precision def _create_ref_dict(ref, frame_size): From 3edf5e31d1fe153c03ea6b9fcba87371cf02cd9e Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 9 Mar 2021 19:57:33 +0100 Subject: [PATCH 32/32] Enable accuracy visualisation and calculation for hmd data --- pupil_src/shared_modules/accuracy_visualizer.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pupil_src/shared_modules/accuracy_visualizer.py b/pupil_src/shared_modules/accuracy_visualizer.py index 17e1aafce1..4e5d7b1930 100644 --- a/pupil_src/shared_modules/accuracy_visualizer.py +++ b/pupil_src/shared_modules/accuracy_visualizer.py @@ -145,10 +145,6 @@ def update( @staticmethod def __gazer_class_from_name(gazer_class_name: str) -> T.Optional[T.Any]: - if "HMD" in gazer_class_name: - logger.info("Accuracy visualization is disabled for HMD calibration") - return None - gazers_by_name = gazer_classes_by_class_name(registered_gazer_classes()) try: