diff --git a/.travis.yml b/.travis.yml index febbdccef4..7f5c55ced0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,13 +22,11 @@ jobs: - name: black formatting check language: python + before_install: pip install -U pip # Travis automatically runs `pip install -r requirements.txt` if such a file is present. # Source: https://docs.travis-ci.com/user/languages/python/#dependency-management - # Since we do not need the requirements to be installed, we overwrite - install: ~ - before_script: - - pip install -U pip - - pip install black + # Since we only need black to be installed, we overwrite with: + install: pip install black script: - > black . --check --exclude pupil_src/tests || ( diff --git a/docs/dependencies-macos.md b/docs/dependencies-macos.md index f53866159d..14ae3a2385 100644 --- a/docs/dependencies-macos.md +++ b/docs/dependencies-macos.md @@ -62,32 +62,12 @@ make && make install ### Install Python Libraries -We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. +We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. To install all Python dependencies, you can use the [`requirements.txt`](https://github.com/pupil-labs/pupil/blob/master/requirements.txt) file from the root of the `pupil` repository. ```sh # Upgrade pip to latest version. This is necessary for some dependencies. -python -m pip install --upgrade pip - -pip install cysignals -pip install cython -pip install msgpack==0.5.6 -pip install numexpr -pip install packaging -pip install psutil -pip install pyaudio -pip install pyopengl -pip install pyzmq -pip install scikit-learn -pip install scipy -pip install glfw -pip install git+https://github.com/zeromq/pyre - -pip install pupil-apriltags -pip install pupil-detectors -pip install git+https://github.com/pupil-labs/PyAV -pip install git+https://github.com/pupil-labs/pyuvc -pip install git+https://github.com/pupil-labs/pyndsi -pip install git+https://github.com/pupil-labs/pyglui +python -m pip install --upgrade pip wheel +pip install -r requirements.txt ``` **NOTE:** Installing **pyglui** might fail on newer versions of **macOS** due to missing OpenGL headers. In this case, you need to install Xcode which comes with the required header files. diff --git a/docs/dependencies-ubuntu17.md b/docs/dependencies-ubuntu17.md index d0c3bb5b3c..2375c9b66a 100644 --- a/docs/dependencies-ubuntu17.md +++ b/docs/dependencies-ubuntu17.md @@ -152,32 +152,12 @@ sudo ldconfig ### Install Python Libraries -We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. +We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. To install all Python dependencies, you can use the [`requirements.txt`](https://github.com/pupil-labs/pupil/blob/master/requirements.txt) file from the root of the `pupil` repository. ```sh # Upgrade pip to latest version. This is necessary for some dependencies. -python -m pip install --upgrade pip - -pip install cysignals -pip install cython -pip install msgpack==0.5.6 -pip install numexpr -pip install packaging -pip install psutil -pip install pyaudio -pip install pyopengl -pip install pyzmq -pip install scikit-learn -pip install scipy -pip install glfw -pip install git+https://github.com/zeromq/pyre - -pip install pupil-apriltags -pip install pupil-detectors -pip install git+https://github.com/pupil-labs/PyAV -pip install git+https://github.com/pupil-labs/pyuvc -pip install git+https://github.com/pupil-labs/pyndsi -pip install git+https://github.com/pupil-labs/pyglui +python -m pip install --upgrade pip wheel +pip install -r requirements.txt ``` **NOTE**: If you get the error `ImportError: No module named 'cv2'` when trying to run Pupil, please refer to the section [OpenCV Troubleshooting](#opencv-troubleshooting) above. diff --git a/docs/dependencies-ubuntu18.md b/docs/dependencies-ubuntu18.md index bebc9f722e..00e2fc79e6 100644 --- a/docs/dependencies-ubuntu18.md +++ b/docs/dependencies-ubuntu18.md @@ -63,32 +63,12 @@ sudo udevadm trigger ### Install Python Libraries -We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. +We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. To install all Python dependencies, you can use the [`requirements.txt`](https://github.com/pupil-labs/pupil/blob/master/requirements.txt) file from the root of the `pupil` repository. ```sh # Upgrade pip to latest version. This is necessary for some dependencies. -python -m pip install --upgrade pip - -pip install cysignals -pip install cython -pip install msgpack==0.5.6 -pip install numexpr -pip install packaging -pip install psutil -pip install pyaudio -pip install pyopengl -pip install pyzmq -pip install scikit-learn -pip install scipy -pip install glfw -pip install git+https://github.com/zeromq/pyre - -pip install pupil-apriltags -pip install pupil-detectors -pip install git+https://github.com/pupil-labs/PyAV -pip install git+https://github.com/pupil-labs/pyuvc -pip install git+https://github.com/pupil-labs/pyndsi -pip install git+https://github.com/pupil-labs/pyglui +python -m pip install --upgrade pip wheel +pip install -r requirements.txt ``` ### OpenCV Troubleshooting diff --git a/docs/dependencies-windows.md b/docs/dependencies-windows.md index 8d0214c4e4..ae29917705 100644 --- a/docs/dependencies-windows.md +++ b/docs/dependencies-windows.md @@ -48,51 +48,17 @@ If you downloaded the linked installer: - Check the box `Add Python to PATH`. This will add Python to your System PATH Environment Variable. - Check the box `Install for all users`. **Note:** By default this will install Python to `C:\Program Files\Python36`. Some build scripts may fail to start Python due to spaces in the path name. So, you may want to consider installing Python to `C:\Python36` instead. - ## Install Python Libraries -We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. +We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. To install all Python dependencies, you can use the [`requirements.txt`](https://github.com/pupil-labs/pupil/blob/master/requirements.txt) file from the root of the `pupil` repository. ```sh # Upgrade pip to latest version. This is necessary for some dependencies. -python -m pip install --upgrade pip - -pip install cython -pip install msgpack==0.5.6 -pip install numexpr -pip install opencv-python==3.* -pip install packaging -pip install psutil -pip install pyaudio -pip install pyopengl -pip install pyzmq -pip install scikit-learn -pip install scipy -pip install glfw -pip install win_inet_pton -pip install git+https://github.com/zeromq/pyre - -pip install pupil-apriltags -pip install pupil-detectors +python -m pip install --upgrade pip wheel +pip install -r requirements.txt ``` -## Pupil Labs Python Wheels - -In addition to these libraries, you will need to install some Pupil-Labs support libraries. Since building them for Windows is also not automated yet, we provide some prebuilt wheels that you can use. If you want to build the support libraries yourself as well, you will have to look for install instructions on the respective GitHub repositories. - -Download the following Python wheels from Pupil Labs github repos: - -- [pyglui](https://github.com/pupil-labs/pyglui/releases/latest) -- [pyav](https://github.com/pupil-labs/pyav/releases/latest) -- [pyndsi](https://github.com/pupil-labs/pyndsi/releases/latest) -- [pyuvc](https://github.com/pupil-labs/pyuvc/releases/latest) - -`pyuvc` requires that you download Microsoft Visual C++ 2010 Redistributable from [microsoft](https://www.microsoft.com/en-us/download/details.aspx?id=14632). The `pthreadVC2` lib, which is used by libuvc, depends on `msvcr100.dll`. - -Open your command prompt and `Run as administrator` in the directory where the wheels are downloaded. - -- Install all wheels with `pip install X` (where X is the name of the `.whl` file) -- You can check that libs are installed with `python import X` statements in the command prompt where `X` is the name of the lib. +**NOTE:** `pyuvc` requires that you download Microsoft Visual C++ 2010 Redistributable from [microsoft](https://www.microsoft.com/en-us/download/details.aspx?id=14632). The `pthreadVC2` lib, which is used by libuvc, depends on `msvcr100.dll`. ## Modifying Pupil to Work with Windows diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index 52ae824da7..2d0b5360a5 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -204,8 +204,30 @@ def get_timestamp(): g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic + def load_runtime_pupil_detection_plugins(): + from plugin import import_runtime_plugins + from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin + + plugins_path = os.path.join(g_pool.user_dir, "plugins") + + for plugin in import_runtime_plugins(plugins_path): + if not isinstance(plugin, type): + continue + if not issubclass(plugin, PupilDetectorPlugin): + continue + if plugin is PupilDetectorPlugin: + continue + yield plugin + default_2d, default_3d, available_detectors = available_detector_plugins() - plugins = manager_classes + source_classes + available_detectors + [Roi] + runtime_detectors = list(load_runtime_pupil_detection_plugins()) + plugins = ( + manager_classes + + source_classes + + available_detectors + + runtime_detectors + + [Roi] + ) g_pool.plugin_by_name = {p.__name__: p for p in plugins} preferred_names = [ @@ -513,6 +535,14 @@ def set_window_size(): # with incorrect settings that were loaded from session settings. plugins_to_load.append(overwrite_cap_settings) + # Add runtime plugins to the list of plugins to load with default arguments, + # if not already restored from session settings + plugins_to_load_names = set(name for name, _ in plugins_to_load) + for runtime_detector in runtime_detectors: + runtime_name = runtime_detector.__name__ + if runtime_name not in plugins_to_load_names: + plugins_to_load.append((runtime_name, {})) + g_pool.plugins = Plugin_List(g_pool, plugins_to_load) if not g_pool.capture: @@ -590,7 +620,8 @@ def window_should_update(): glfw.swap_interval(0) # Event loop - while not glfw.window_should_close(main_window): + window_should_close = False + while not window_should_close: if notify_sub.new_data: t, notification = notify_sub.recv() @@ -657,6 +688,17 @@ def window_should_update(): ) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") + elif ( + subject.startswith("stop_eye_plugin") + and notification["target"] == g_pool.process + ): + try: + plugin_to_stop = g_pool.plugin_by_name[notification["name"]] + except KeyError as err: + logger.error(f"Attempt to load unknown plugin: {err}") + else: + plugin_to_stop.alive = False + g_pool.plugins.clean() for plugin in g_pool.plugins: plugin.on_notify(notification) @@ -724,13 +766,13 @@ def window_should_update(): for result in event.get(EVENT_KEY, ()): pupil_socket.send(result) - cpu_graph.update() - # GL drawing if window_should_update(): + cpu_graph.update() if is_window_visible(main_window): consume_events_and_render_buffer() glfw.poll_events() + window_should_close = glfw.window_should_close(main_window) # END while running diff --git a/pupil_src/launchables/player.py b/pupil_src/launchables/player.py index 7fd6cb2cd3..1ba134164a 100644 --- a/pupil_src/launchables/player.py +++ b/pupil_src/launchables/player.py @@ -117,6 +117,7 @@ def player( from gaze_producer.gaze_from_offline_calibration import ( GazeFromOfflineCalibration, ) + from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection @@ -152,6 +153,9 @@ def interrupt_handler(sig, frame): signal.signal(signal.SIGINT, interrupt_handler) runtime_plugins = import_runtime_plugins(os.path.join(user_dir, "plugins")) + runtime_plugins = [ + p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin) + ] system_plugins = [ Log_Display, Seek_Control, diff --git a/pupil_src/launchables/world.py b/pupil_src/launchables/world.py index 0e9b86faa5..a037b580db 100644 --- a/pupil_src/launchables/world.py +++ b/pupil_src/launchables/world.py @@ -161,6 +161,7 @@ def detection_enabled_setter(is_on: bool): from gaze_mapping import registered_gazer_classes from gaze_mapping.gazer_base import GazerBase + from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze @@ -241,6 +242,9 @@ def get_timestamp(): runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins") ) + runtime_plugins = [ + p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin) + ] user_plugins = [ Pupil_Groups, NetworkApiPlugin, diff --git a/pupil_src/shared_modules/circle_detector.py b/pupil_src/shared_modules/circle_detector.py index 8ca553a27b..16fbd2b654 100644 --- a/pupil_src/shared_modules/circle_detector.py +++ b/pupil_src/shared_modules/circle_detector.py @@ -702,7 +702,7 @@ def bench(): sts, img = cap.read() # img = cv2.imread('/Users/mkassner/Desktop/manual_calibration_marker-01.png') gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) - print(len(find_concetric_circles(gray, visual_debug=img))) + print(len(find_concentric_circles(gray, visual_debug=img))) cv2.imshow("img", img) cv2.waitKey(1) # return diff --git a/pupil_src/shared_modules/gaze_mapping/gazer_3d/gazer_headset.py b/pupil_src/shared_modules/gaze_mapping/gazer_3d/gazer_headset.py index 9514fe2a86..d68a1b8dd7 100644 --- a/pupil_src/shared_modules/gaze_mapping/gazer_3d/gazer_headset.py +++ b/pupil_src/shared_modules/gaze_mapping/gazer_3d/gazer_headset.py @@ -157,6 +157,10 @@ def _predict_single(self, x): gaze_3d = self._toWorld(gaze_point) normal_3d = np.dot(self.rotation_matrix, pupil_normal) + # Check if gaze is in front of camera. If it is not, flip direction. + if gaze_3d[-1] < 0: + gaze_3d *= -1.0 + g = { "eye_center_3d": eye_center.tolist(), "gaze_normal_3d": normal_3d.tolist(), @@ -284,6 +288,10 @@ def _predict_single(self, x): if nearest_intersection_point is None: return None + # Check if gaze is in front of camera. If it is not, flip direction. + if nearest_intersection_point[-1] < 0: + nearest_intersection_point *= -1.0 + g = { "eye_centers_3d": {0: s0_center.tolist(), 1: s1_center.tolist()}, "gaze_normals_3d": {0: s0_normal.tolist(), 1: s1_normal.tolist()}, diff --git a/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py b/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py index 32f8ca7de9..ca9919fd63 100644 --- a/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py +++ b/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py @@ -10,6 +10,7 @@ """ import logging +import pye3d from pye3d.detector_3d import Detector3D, CameraModel from pyglui import ui @@ -19,6 +20,16 @@ logger = logging.getLogger(__name__) +version_installed = getattr(pye3d, "__version__", "0.0.1") +version_supported = "0.0.1" + +if version_installed != version_supported: + logger.info( + f"Requires pye3d version {version_supported} " + f"(Installed: {version_installed})" + ) + raise ImportError + class Pye3DPlugin(PupilDetectorPlugin): uniqueness = "by_class" diff --git a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d.py b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/__init__.py similarity index 99% rename from pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d.py rename to pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/__init__.py index 542a89096f..1b57804c63 100644 --- a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d.py +++ b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/__init__.py @@ -37,6 +37,8 @@ from pyglui.cygl.utils import RGBA from visualizer import Visualizer +from .eye import LeGrandEye + class Eye_Visualizer(Visualizer): def __init__(self, g_pool, focal_length): diff --git a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/eye.py b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/eye.py new file mode 100644 index 0000000000..d80d9cb9cd --- /dev/null +++ b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/eye.py @@ -0,0 +1,300 @@ +""" +(*)~--------------------------------------------------------------------------- +Pupil - eye tracking platform +Copyright (C) 2012-2019 Pupil Labs + +Distributed under the terms of the GNU +Lesser General Public License (LGPL v3.0). +See COPYING and COPYING.LESSER for license details. +---------------------------------------------------------------------------~(*) +""" +import cv2 +import numpy as np +from OpenGL.GL import * + +from .pose import PosedObject +from .utilities import ( + normalize, + rotate_v1_on_v2, + sph2cart, + transform_as_homogeneous_point, + transform_as_homogeneous_vector, +) + + +class BasicEye(PosedObject): + def __init__(self): + super(BasicEye, self).__init__(pose=np.eye(4), extrinsics=None, children=()) + + self._gaze_vector = PosedObject() + self.eyeball_center = [0.0, 0.0, 0.0] + + def update_from_gaze_point(self, gaze_point): + new_gaze_vector = transform_as_homogeneous_vector( + normalize(gaze_point - self.eyeball_center), self.extrinsics + ) + self.update_from_gaze_vector(new_gaze_vector) + + def update_from_spherical(self, phi, theta): + new_gaze_vector = sph2cart(phi, theta) + self.update_from_gaze_vector(new_gaze_vector) + + def update_from_gaze_vector(self, new_gaze_vector): + rotation = rotate_v1_on_v2([0.0, 0.0, 1.0], new_gaze_vector) + self._gaze_vector.rmat = rotation + + def move_to_point(self, point): + self.translate(point - self.eyeball_center) + + @property + def eyeball_center(self): + return self.tvec + + @eyeball_center.setter + def eyeball_center(self, point): + self.tvec = np.asarray(point) + + @property + def gaze_vector(self): + return (self._gaze_vector.pose @ self.pose)[:3, 2] + + def __str__(self): + return "\n".join("{}:{}".format(k, v) for k, v in self.__dict__.items()) + + +class LeGrandEye(BasicEye): + def __init__( + self, + eyeball_radius=12.0, + cornea_radius=7.8, + iris_radius=6.0, + n_refraction=1.3375, + camera=None, + ): + + super(LeGrandEye, self).__init__() + + self.model_type = "LeGrand" + + # PUPIL + distance_eyeball_pupil = np.sqrt(eyeball_radius ** 2 - iris_radius ** 2) + self.__pupil_center = [0.0, 0.0, distance_eyeball_pupil] + self.pupil_radius = 2.0 + self.pupil_normal = np.asarray([0.0, 0.0, 1.0]) + + # IRIS + self.__iris_center = self.__pupil_center + self.iris_radius = iris_radius + self.iris_normal = np.asarray([0.0, 0.0, 1.0]) + self.iris_color = [46 / 255.0, 220 / 255.0, 255.0 / 255.0] + + # CORNEA + h = np.sqrt(cornea_radius ** 2 - iris_radius ** 2) + distance_eyeball_cornea = distance_eyeball_pupil - h + self.__cornea_center = np.asarray([0, 0, distance_eyeball_cornea]) + self.cornea_radius = cornea_radius + + # EYEBALL + self.eyeball_radius = eyeball_radius + + # self.translate(np.asarray([0., 0., 35.])) + # self.update_from_gaze_vector(np.asarray([0., 0., -1])) + + # PHYSICAL CONSTANTS + self.n_refraction = n_refraction + + # CAMERA POINTED AT EYE + self.camera = camera + + # GL SETUP + self.eyeball_alpha = np.arccos(distance_eyeball_pupil / self.eyeball_radius) + self.cornea_alpha = np.arccos(4.0 / self.cornea_radius) / 1.0 + self.set_up_gl_vertices() + + @property + def cornea_center(self): + cornea_center = transform_as_homogeneous_point( + self.__cornea_center, self.pose @ self._gaze_vector.pose + ) + return cornea_center + + @property + def iris_center(self): + iris_center = transform_as_homogeneous_point( + self.__iris_center, self.pose @ self._gaze_vector.pose + ) + return iris_center + + @property + def pupil_center(self): + pupil_center = transform_as_homogeneous_point( + self.__pupil_center, self.pose @ self._gaze_vector.pose + ) + return pupil_center + + def set_up_gl_vertices(self): + + # EYEBALL + self.central_ring_eyeball = [ + [self.eyeball_radius * np.sin(phi), 0, self.eyeball_radius * np.cos(phi)] + for phi in np.linspace( + self.eyeball_alpha, 2 * np.pi - self.eyeball_alpha, 30 + ) + ] + self.rings_eyeball = [self.central_ring_eyeball] + for phi in np.linspace(0, np.pi, 20): + central_ring_rotated = [ + cv2.Rodrigues(np.asarray([0.0, 0.0, phi]))[0] @ v + for v in self.central_ring_eyeball + ] + self.rings_eyeball.append(central_ring_rotated) + + # IRIS + angles = [phi for phi in np.linspace(0, 2 * np.pi, 40)] + self.iris_quads = [] + for i in range(len(angles) - 1): + self.iris_quads.append( + [ + np.array((np.cos(angles[i]), np.sin(angles[i]), 0)), + np.array((np.cos(angles[i + 1]), np.sin(angles[i + 1]), 0)), + np.array((np.cos(angles[i + 1]), np.sin(angles[i + 1]), 0)), + np.array((np.cos(angles[i]), np.sin(angles[i]), 0)), + ] + ) + + # CORNEA + self.central_ring_cornea = [ + [self.cornea_radius * np.sin(phi), 0, self.cornea_radius * np.cos(phi)] + for phi in np.linspace(-self.cornea_alpha, self.cornea_alpha, 20) + ] + + self.rings_cornea = [self.central_ring_cornea] + for phi in np.linspace(0, np.pi, 10): + central_ring_rotated = [ + cv2.Rodrigues(np.asarray([0.0, 0.0, phi]))[0] @ v + for v in self.central_ring_cornea + ] + self.rings_cornea.append(central_ring_rotated) + + def draw_gl( + self, + draw_eyeball=True, + draw_iris=True, + draw_cornea=True, + draw_gaze=True, + alpha=1.0, + ): + + glPushMatrix() + + glLoadIdentity() + # if self.camera is not None: + # glMultMatrixf(self.camera.pose.T) + glMultMatrixf(self.pose.T) + glMultMatrixf(self._gaze_vector.pose.T) + + glPushMatrix() + + glPolygonMode(GL_FRONT_AND_BACK, GL_LINE) + + if draw_gaze: + glLineWidth(2.0) + glColor4f(1, 1, 1, 1.0 * alpha) + glBegin(GL_LINES) + glVertex3f(*[0, 0, 0]) + glVertex3f(*[0, 0, 600]) + glEnd() + + # DRAW EYEBALL + if draw_eyeball: + + # glPolygonMode(GL_FRONT_AND_BACK, GL_FILL) + glColor4f(0.6, 0.6, 1.0, 1.0 * alpha) + glLineWidth(1.0) + + glPushMatrix() + for i in range(len(self.rings_eyeball) - 1): + for j in range(len(self.rings_eyeball[i]) - 1): + glBegin(GL_QUADS) + glVertex3f( + self.rings_eyeball[i][j][0], + self.rings_eyeball[i][j][1], + self.rings_eyeball[i][j][2], + ) + glVertex3f( + self.rings_eyeball[i][j + 1][0], + self.rings_eyeball[i][j + 1][1], + self.rings_eyeball[i][j + 1][2], + ) + glVertex3f( + self.rings_eyeball[i + 1][j + 1][0], + self.rings_eyeball[i + 1][j + 1][1], + self.rings_eyeball[i + 1][j + 1][2], + ) + glVertex3f( + self.rings_eyeball[i + 1][j][0], + self.rings_eyeball[i + 1][j][1], + self.rings_eyeball[i + 1][j][2], + ) + glEnd() + glPopMatrix() + + # DRAW IRIS + if draw_iris: + + glPolygonMode(GL_FRONT_AND_BACK, GL_FILL) + glColor4f( + self.iris_color[0], self.iris_color[1], self.iris_color[2], 0.4 * alpha + ) + + glPushMatrix() + glTranslate(0, 0, self.__pupil_center[2]) + for quad in self.iris_quads: + glBegin(GL_QUADS) + glVertex3f(*(quad[0] * self.pupil_radius)) + glVertex3f(*(quad[1] * self.pupil_radius)) + glVertex3f(*(quad[2] * self.iris_radius)) + glVertex3f(*(quad[3] * self.iris_radius)) + glEnd() + glPopMatrix() + + # DRAW CORNEA + if draw_cornea: + + glPolygonMode(GL_FRONT_AND_BACK, GL_LINE) + glColor4f(1, 1, 1, 0.3 * alpha) + glLineWidth(1.0) + + glPushMatrix() + glTranslate(0, 0, self.__cornea_center[2]) + for i in range(len(self.rings_cornea) - 1): + for j in range(len(self.rings_cornea[i]) - 1): + glBegin(GL_QUADS) + glVertex3f( + self.rings_cornea[i][j][0], + self.rings_cornea[i][j][1], + self.rings_cornea[i][j][2], + ) + glVertex3f( + self.rings_cornea[i][j + 1][0], + self.rings_cornea[i][j + 1][1], + self.rings_cornea[i][j + 1][2], + ) + glVertex3f( + self.rings_cornea[i + 1][j + 1][0], + self.rings_cornea[i + 1][j + 1][1], + self.rings_cornea[i + 1][j + 1][2], + ) + glVertex3f( + self.rings_cornea[i + 1][j][0], + self.rings_cornea[i + 1][j][1], + self.rings_cornea[i + 1][j][2], + ) + glEnd() + glPopMatrix() + + glPopMatrix() + + glPopMatrix() + + glPolygonMode(GL_FRONT_AND_BACK, GL_FILL) diff --git a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/pose.py b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/pose.py new file mode 100644 index 0000000000..2587a4abd9 --- /dev/null +++ b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/pose.py @@ -0,0 +1,101 @@ +""" +(*)~--------------------------------------------------------------------------- +Pupil - eye tracking platform +Copyright (C) 2012-2019 Pupil Labs + +Distributed under the terms of the GNU +Lesser General Public License (LGPL v3.0). +See COPYING and COPYING.LESSER for license details. +---------------------------------------------------------------------------~(*) +""" +import cv2 +import numpy as np + + +class PosedObject(object): + def __init__(self, pose=np.eye(4), extrinsics=None, children=(), parents=()): + + self.parents = parents + self.children = children + self._pose = np.eye( + 4 + ) # Needed during initialization for first call of pose.setter + + if type(pose) in [np.ndarray, list]: + self.pose = np.array(pose) + else: + self.pose = self.pose_from_extrinsics(extrinsics) + + @property + def pose(self): + return self._pose.copy() + + @pose.setter + def pose(self, new_pose): + for child in self.children: + child.pose = new_pose @ np.linalg.inv(self.pose) @ child.pose + self._pose = new_pose + + def translate(self, translation): + pose = self.pose.copy() + pose[:3, 3] += translation + self.pose = pose + + def rotate(self, rotation): + pose = self.pose.copy() + pose[:3, :3] = rotation @ pose[:3, :3] + self.pose = pose + + @staticmethod + def extrinsics_from_pose(pose): + rot = pose[:3, :3] + trans = pose[:3, 3] + extrinsics = np.eye(4) + extrinsics[:3, :3] = rot.T + extrinsics[:3, 3] = -rot.T @ trans + return extrinsics + + @staticmethod + def pose_from_extrinsics(extrinsics): + rot = extrinsics[:3, :3] + trans = extrinsics[:3, 3] + pose = np.eye(4) + pose[:3, :3] = rot.T + pose[:3, 3] = -rot.T @ trans + return pose + + @property + def tvec(self): + return self._pose[:3, 3] + + @tvec.setter + def tvec(self, new_tvec): + pose = self.pose + pose[:3, 3] = new_tvec + self.pose = pose + + @property + def rmat(self): + return self._pose[:3, :3] + + @rmat.setter + def rmat(self, new_rmat): + pose = self.pose + pose[:3, :3] = new_rmat + self.pose = pose + + @property + def rvec(self): + return cv2.Rodrigues(self.rmat)[0] + + @rvec.setter + def rvec(self, new_rvec): + self._pose[:3, :3] = cv2.Rodrigues(new_rvec)[0] + + @property + def extrinsics(self): + return self.extrinsics_from_pose(self.pose) + + @extrinsics.setter + def extrinsics(self, new_extrinsics): + self.pose = self.pose_from_extrinsics(new_extrinsics) diff --git a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/utilities.py b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/utilities.py new file mode 100644 index 0000000000..2223fef1cf --- /dev/null +++ b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/utilities.py @@ -0,0 +1,92 @@ +""" +(*)~--------------------------------------------------------------------------- +Pupil - eye tracking platform +Copyright (C) 2012-2019 Pupil Labs + +Distributed under the terms of the GNU +Lesser General Public License (LGPL v3.0). +See COPYING and COPYING.LESSER for license details. +---------------------------------------------------------------------------~(*) +""" +import numpy as np + + +def cart2sph(x): + + phi = np.arctan2(x[2], x[0]) + theta = np.arccos(x[1] / np.linalg.norm(x)) + + return phi, theta # Todo: This seems to be opposite to the pupil code + + +def sph2cart(phi, theta): + + result = np.empty(3) + + result[0] = np.sin(theta) * np.cos(phi) + result[1] = np.cos(theta) + result[2] = np.sin(theta) * np.sin(phi) + + return result + + +def normalize(v, axis=-1): + + return v / np.linalg.norm(v, axis=axis) + + +def enclosed_angle(v1, v2, unit="deg", axis=-1): + + v1 = normalize(v1, axis=axis) + v2 = normalize(v2, axis=axis) + + alpha = np.arccos(np.clip(np.dot(v1.T, v2), -1, 1)) + + if unit == "deg": + return 180.0 / np.pi * alpha + else: + return alpha + + +def make_homogeneous_vector(v): + + return np.hstack((v, [0.0])) + + +def make_homogeneous_point(p): + return np.hstack((p, [1.0])) + + +def transform_as_homogeneous_point(p, trafo): + p = make_homogeneous_point(p) + return (trafo @ p)[:3] + + +def transform_as_homogeneous_vector(v, trafo): + v = make_homogeneous_vector(v) + return (trafo @ v)[:3] + + +def rotate_v1_on_v2(v1, v2): + + v1 = normalize(v1) + v2 = normalize(v2) + cos_angle = np.dot(v1, v2) + + if not np.allclose(np.abs(cos_angle), 1): + u = np.cross(v1, v2) + s = np.linalg.norm(u) + c = np.dot(v1, v2) + + I = np.eye(3) + ux = np.asarray([[0, -u[2], u[1]], [u[2], 0, -u[0]], [-u[1], u[0], 0]]) + + R = I + ux + np.dot(ux, ux) * (1 - c) / s ** 2 + + elif np.allclose(cos_angle, 1): + R = np.eye(3) + + elif np.allclose(cos_angle, -1): + R = -np.eye(3) + + return R diff --git a/pupil_src/shared_modules/pupil_producers.py b/pupil_src/shared_modules/pupil_producers.py index 89f7a9a129..3e2800fb9a 100644 --- a/pupil_src/shared_modules/pupil_producers.py +++ b/pupil_src/shared_modules/pupil_producers.py @@ -328,7 +328,7 @@ def __init__(self, g_pool): self.data_sub = zmq_tools.Msg_Receiver( zmq_ctx, g_pool.ipc_sub_url, - topics=("pupil", "notify.file_source.video_finished"), + topics=("pupil", "notify.file_source"), hwm=100_000, ) @@ -354,6 +354,7 @@ def __init__(self, g_pool): # Start offline pupil detection if not complete yet: self.eye_video_loc = [None, None] self.eye_frame_num = [0, 0] + self.eye_frame_idx = [-1, -1] # start processes for eye_id in range(2): @@ -381,6 +382,7 @@ def start_eye_process(self, eye_id): video_loc = existing_locs[0] n_valid_frames = np.count_nonzero(self.videoset.lookup.container_idx > -1) self.eye_frame_num[eye_id] = n_valid_frames + self.eye_frame_idx = [-1, -1] capure_settings = "File_Source", {"source_path": video_loc, "timing": None} self.notify_all( @@ -395,17 +397,21 @@ def start_eye_process(self, eye_id): @property def detection_progress(self) -> float: - total = sum(self.eye_frame_num) - # TODO: Figure out the number of frames independent of 3d detection - detected = self._pupil_data_store.count_collected(detector_tag="3d") - if total: - return min( - detected / total, - 1.0, - ) - else: + + if not sum(self.eye_frame_num): return 0.0 + progress_by_eye = [0.0, 0.0] + + for eye_id in (0, 1): + total_frames = self.eye_frame_num[eye_id] + current_index = self.eye_frame_idx[eye_id] + progress = (current_index + 1) / total_frames + progress = max(0.0, min(progress, 1.0)) + progress_by_eye[eye_id] = progress + + return min(progress_by_eye) + def stop_eye_process(self, eye_id): self.notify_all({"subject": "eye_process.should_stop", "eye_id": eye_id}) self.eye_video_loc[eye_id] = None @@ -425,15 +431,20 @@ def recent_events(self, events): else: payload = self.data_sub.deserialize_payload(*remaining_frames) if payload["subject"] == "file_source.video_finished": - for eyeid in (0, 1): - if self.eye_video_loc[eyeid] == payload["source_path"]: - logger.debug("eye {} process complete".format(eyeid)) - self.detection_status[eyeid] = "complete" - self.stop_eye_process(eyeid) + for eye_id in (0, 1): + if self.eye_video_loc[eye_id] == payload["source_path"]: + logger.debug("eye {} process complete".format(eye_id)) + self.eye_frame_idx[eye_id] = self.eye_frame_num[eye_id] + self.detection_status[eye_id] = "complete" + self.stop_eye_process(eye_id) break if self.eye_video_loc == [None, None]: data = self._pupil_data_store.as_pupil_data_bisector() self.publish_new(pupil_data_bisector=data) + if payload["subject"] == "file_source.current_frame_index": + for eye_id in (0, 1): + if self.eye_video_loc[eye_id] == payload["source_path"]: + self.eye_frame_idx[eye_id] = payload["index"] self.menu_icon.indicator_stop = self.detection_progress diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index 342e9e4f1d..61a28b1ff7 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -16,7 +16,7 @@ import typing as T from abc import ABC, abstractmethod from multiprocessing import cpu_count -from time import sleep +from time import sleep, monotonic import av import numpy as np @@ -490,6 +490,36 @@ def recent_events_own_timing(self, events): self._recent_frame = frame events["frame"] = frame + if self.timing is None: + self._notify_current_index(min_time_diff_sec=1.0) + + def _notify_current_index(self, min_time_diff_sec: float): + timestamp_now = monotonic() + current_index = self.get_frame_index() + + try: + timestamp_last = self.__last_current_index_notification_timestamp + last_index = self.__last_current_index_notification_index + except AttributeError: + should_send_notification = True + else: + should_send_notification = True + should_send_notification &= ( + timestamp_now - timestamp_last + ) >= min_time_diff_sec + should_send_notification &= current_index != last_index + + if should_send_notification: + self.notify_all( + { + "subject": "file_source.current_frame_index", + "index": current_index, + "source_path": self.source_path, + } + ) + self.__last_current_index_notification_timestamp = timestamp_now + self.__last_current_index_notification_index = current_index + def seek_to_frame(self, seek_pos): try: target_entry = self.videoset.lookup[seek_pos] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..b85eb31504 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,42 @@ +### +### Third-party +### +cython +msgpack==0.5.6 +numexpr +packaging>=20.0 +psutil +pyaudio +pyopengl +pyzmq +scikit-learn +scipy +glfw +pyre @ git+https://github.com/zeromq/pyre + +cysignals ; platform_system != "Windows" + +win_inet_pton ; platform_system == "Windows" +opencv-python==3.* ; platform_system == "Windows" + +### +### Pupil-Labs +### +pupil-apriltags==1.0.4 +pupil-detectors==1.1.1 + +# pupil-labs/PyAV 0.4.6 +av @ git+https://github.com/pupil-labs/PyAV@v0.4.6 ; platform_system != "Windows" +av @ https://github.com/pupil-labs/PyAV/releases/download/v0.4.6/av-0.4.6-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" + +# pupil-labs/pyuvc 0.14 +uvc @ git+https://github.com/pupil-labs/pyuvc@v0.14.1 ; platform_system != "Windows" # Minor patch fixes build issues in pyproject.toml +uvc @ https://github.com/pupil-labs/pyuvc/releases/download/v0.14/uvc-0.14-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" + +# pupil-labs/pyglui 1.28 +pyglui @ git+https://github.com/pupil-labs/pyglui@v1.28 ; platform_system != "Windows" +pyglui @ https://github.com/pupil-labs/pyglui/releases/download/v1.28/pyglui-1.28-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" + +# pupil-labs/pyndsi 1.4 +ndsi @ git+https://github.com/pupil-labs/pyndsi@v1.4 ; platform_system != "Windows" +ndsi @ https://github.com/pupil-labs/pyndsi/releases/download/v1.4/ndsi-1.4-cp36-cp36m-win_amd64.whl ; platform_system == "Windows"