diff --git a/.gitignore b/.gitignore index bf285359f6..e16d4b09be 100644 --- a/.gitignore +++ b/.gitignore @@ -51,3 +51,8 @@ mprofile_*.dat .ipynb_checkpoints/ *.dmg *.zip +deployment/pupil_v* +*.pyd +*.dll +win_drv +.vs \ No newline at end of file diff --git a/deployment/bundle.bat b/deployment/bundle.bat index 81f2e0301c..acf5f485ac 100644 --- a/deployment/bundle.bat +++ b/deployment/bundle.bat @@ -32,9 +32,12 @@ if not exist %release_dir% ( mkdir %release_dir% ) -call python ..\pupil_src\shared_modules\pupil_detectors\build.py -call python ..\pupil_src\shared_modules\cython_methods\build.py -call python ..\pupil_src\shared_modules\calibration_routines\optimization_calibration\build.py +set PATH=%PATH%;C:\Python36\Lib\site-packages\scipy\.libs +set PATH=%PATH%;C:\Python36\Lib\site-packages\zmq + +python ..\pupil_src\shared_modules\pupil_detectors\build.py +python ..\pupil_src\shared_modules\cython_methods\build.py +python ..\pupil_src\shared_modules\calibration_routines\optimization_calibration\build.py call :Bundle capture %current_tag% call :Bundle service %current_tag% @@ -43,7 +46,7 @@ call :Bundle player %current_tag% cd %release_dir% for /d %%d in (*) do ( echo Adding %%d - 7z a -t7z ..\%release_dir%.7z %%d + 7z a -tzip ..\%release_dir%.zip %%d ) cd .. diff --git a/deployment/deploy_capture/bundle.spec b/deployment/deploy_capture/bundle.spec index 7ec7f72240..c9705dd5f5 100644 --- a/deployment/deploy_capture/bundle.spec +++ b/deployment/deploy_capture/bundle.spec @@ -104,10 +104,6 @@ if platform.system() == "Darwin": [("pyglui/pupil_icons.ttf", ui.get_pupil_icons_font_path(), "DATA")], [("pyrealsense/lrs_parsed_classes", pyrealsense_path, "DATA")], apriltag_libs, - Tree( - "../../pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/", - prefix="weights", - ), strip=None, upx=True, name="Pupil Capture", @@ -174,10 +170,6 @@ elif platform.system() == "Linux": [("pyglui/Roboto-Regular.ttf", ui.get_roboto_font_path(), "DATA")], [("pyglui/pupil_icons.ttf", ui.get_pupil_icons_font_path(), "DATA")], apriltag_libs, - Tree( - "../../pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/", - prefix="weights", - ), strip=True, upx=True, name="pupil_capture", @@ -211,9 +203,11 @@ elif platform.system() == "Windows": "scipy.special._ufuncs_cxx", ] + external_libs_path = pathlib.Path("../../pupil_external") + a = Analysis( ["../../pupil_src/main.py"], - pathex=["../../pupil_src/shared_modules/", "../../pupil_external"], + pathex=["../../pupil_src/shared_modules/", str(external_libs_path)], binaries=None, datas=None, hiddenimports=pyglui_hidden_imports @@ -246,6 +240,11 @@ elif platform.system() == "Windows": for lib in apriltag_lib_path.rglob("*.dll") ] + vc_redist_path = external_libs_path / "vc_redist" + vc_redist_libs = [ + (lib.name, str(lib), "BINARY") for lib in vc_redist_path.glob("*.dll") + ] + coll = COLLECT( exe, a.binaries, @@ -257,10 +256,7 @@ elif platform.system() == "Windows": [("pyglui/Roboto-Regular.ttf", ui.get_roboto_font_path(), "DATA")], [("pyglui/pupil_icons.ttf", ui.get_pupil_icons_font_path(), "DATA")], apriltag_libs, - Tree( - "../../pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/", - prefix="weights", - ), + vc_redist_libs, np_dll_list, strip=False, upx=True, diff --git a/deployment/deploy_player/bundle.spec b/deployment/deploy_player/bundle.spec index 97e5cd3f2b..3db861f62a 100644 --- a/deployment/deploy_player/bundle.spec +++ b/deployment/deploy_player/bundle.spec @@ -101,10 +101,6 @@ if platform.system() == "Darwin": [("pyglui/Roboto-Regular.ttf", ui.get_roboto_font_path(), "DATA")], [("pyglui/pupil_icons.ttf", ui.get_pupil_icons_font_path(), "DATA")], apriltag_libs, - Tree( - "../../pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/", - prefix="weights", - ), strip=None, upx=True, name="Pupil Player", @@ -173,10 +169,6 @@ elif platform.system() == "Linux": [("pyglui/Roboto-Regular.ttf", ui.get_roboto_font_path(), "DATA")], [("pyglui/pupil_icons.ttf", ui.get_pupil_icons_font_path(), "DATA")], apriltag_libs, - Tree( - "../../pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/", - prefix="weights", - ), strip=True, upx=True, name="pupil_player", @@ -210,9 +202,11 @@ elif platform.system() == "Windows": "scipy.special._ufuncs_cxx", ] + external_libs_path = pathlib.Path("../../pupil_external") + a = Analysis( ["../../pupil_src/main.py"], - pathex=["../../pupil_src/shared_modules/", "../../pupil_external"], + pathex=["../../pupil_src/shared_modules/", str(external_libs_path)], hiddenimports=["pyglui.cygl.shader"] + scipy_imports + av_hidden_imports @@ -242,6 +236,11 @@ elif platform.system() == "Windows": for lib in apriltag_lib_path.rglob("*.dll") ] + vc_redist_path = external_libs_path / "vc_redist" + vc_redist_libs = [ + (lib.name, str(lib), "BINARY") for lib in vc_redist_path.glob("*.dll") + ] + coll = COLLECT( exe, a.binaries, @@ -252,10 +251,7 @@ elif platform.system() == "Windows": [("pyglui/Roboto-Regular.ttf", ui.get_roboto_font_path(), "DATA")], [("pyglui/pupil_icons.ttf", ui.get_pupil_icons_font_path(), "DATA")], apriltag_libs, - Tree( - "../../pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/", - prefix="weights", - ), + vc_redist_libs, np_dll_list, strip=None, upx=True, diff --git a/deployment/deploy_service/bundle.spec b/deployment/deploy_service/bundle.spec index 9e01abf4c0..5f01431bb7 100644 --- a/deployment/deploy_service/bundle.spec +++ b/deployment/deploy_service/bundle.spec @@ -1,7 +1,7 @@ # -*- mode: python -*- -import platform, sys, os, os.path, numpy, glob +import platform, sys, os, os.path, numpy, glob, pathlib av_hidden_imports = [ "av.format", @@ -78,10 +78,6 @@ if platform.system() == "Darwin": [("pyglui/OpenSans-Regular.ttf", ui.get_opensans_font_path(), "DATA")], [("pyglui/Roboto-Regular.ttf", ui.get_roboto_font_path(), "DATA")], [("pyglui/pupil_icons.ttf", ui.get_pupil_icons_font_path(), "DATA")], - Tree( - "../../pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/", - prefix="weights", - ), strip=None, upx=True, name="Pupil Service", @@ -140,10 +136,6 @@ elif platform.system() == "Linux": [("pyglui/OpenSans-Regular.ttf", ui.get_opensans_font_path(), "DATA")], [("pyglui/Roboto-Regular.ttf", ui.get_roboto_font_path(), "DATA")], [("pyglui/pupil_icons.ttf", ui.get_pupil_icons_font_path(), "DATA")], - Tree( - "../../pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/", - prefix="weights", - ), strip=True, upx=True, name="pupil_service", @@ -177,9 +169,11 @@ elif platform.system() == "Windows": "scipy.special._ufuncs_cxx", ] + external_libs_path = pathlib.Path("../../pupil_external") + a = Analysis( ["../../pupil_src/main.py"], - pathex=["../../pupil_src/shared_modules/", "../../pupil_external"], + pathex=["../../pupil_src/shared_modules/", str(external_libs_path)], binaries=None, datas=None, hiddenimports=pyglui_hidden_imports + scipy_imports + av_hidden_imports, @@ -203,6 +197,13 @@ elif platform.system() == "Windows": console=False, resources=["pupil-service.ico,ICON"], ) + + vc_redist_path = external_libs_path / "vc_redist" + vc_redist_libs = [ + (lib.name, str(lib), "BINARY") + for lib in vc_redist_path.glob("*.dll") + ] + coll = COLLECT( exe, a.binaries, @@ -214,6 +215,7 @@ elif platform.system() == "Windows": [("pyglui/Roboto-Regular.ttf", ui.get_roboto_font_path(), "DATA")], [("pyglui/pupil_icons.ttf", ui.get_pupil_icons_font_path(), "DATA")], np_dll_list, + vc_redist_libs, strip=False, upx=True, name="Pupil Service", diff --git a/docs/dependencies-macos.md b/docs/dependencies-macos.md index 178715236b..be2267a9be 100644 --- a/docs/dependencies-macos.md +++ b/docs/dependencies-macos.md @@ -76,7 +76,6 @@ pip install pyaudio pip install pyopengl pip install pyzmq pip install scipy -pip install torch torchvision pip install git+https://github.com/zeromq/pyre pip install pupil_apriltags diff --git a/docs/dependencies-ubuntu17.md b/docs/dependencies-ubuntu17.md index 9c514e38dd..69f726b06b 100644 --- a/docs/dependencies-ubuntu17.md +++ b/docs/dependencies-ubuntu17.md @@ -143,7 +143,6 @@ pip install pyaudio pip install pyopengl pip install pyzmq pip install scipy -pip install torch torchvision pip install git+https://github.com/zeromq/pyre pip install pupil_apriltags diff --git a/docs/dependencies-ubuntu18.md b/docs/dependencies-ubuntu18.md index 6b9f1e5409..05f18e9855 100644 --- a/docs/dependencies-ubuntu18.md +++ b/docs/dependencies-ubuntu18.md @@ -62,7 +62,6 @@ pip install pyaudio pip install pyopengl pip install pyzmq pip install scipy -pip install torch torchvision pip install git+https://github.com/zeromq/pyre pip install pupil_apriltags diff --git a/docs/dependencies-windows.md b/docs/dependencies-windows.md index f2b3154504..30de96741d 100644 --- a/docs/dependencies-windows.md +++ b/docs/dependencies-windows.md @@ -101,11 +101,6 @@ pip install git+https://github.com/pupil-labs/nslr pip install git+https://github.com/pupil-labs/nslr-hmm ``` -Now install pytorch: -- Open the pytorch website for local installation: https://pytorch.org/get-started/locally/ -- Select options: Stable, Windows, Pip, Python 3.6, CUDA 9.0. -- You will be provided with two commands. Run them in the order given to install the wheels. - ## Pupil Labs Python Wheels In addition to these libraries, you will need to install some Pupil-Labs support libraries. Since building them for Windows is also not automated yet, we provide some prebuilt wheels that you can use. If you want to build the support libraries yourself as well, you will have to look for install instructions on the respective GitHub repositories. diff --git a/pupil_external/vc_redist/README.txt b/pupil_external/vc_redist/README.txt new file mode 100644 index 0000000000..8ea5a0564d --- /dev/null +++ b/pupil_external/vc_redist/README.txt @@ -0,0 +1,5 @@ +This folder includes dlls that have been extracted from vc_redist.x64.exe to fix https://github.com/pupil-labs/pupil/issues/1661 + +Specifically, one needs: +- concrt140.dll +- vcomp140.dll \ No newline at end of file diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index 8091f31ba6..df333d623f 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -878,6 +878,7 @@ def eye_profiled( version, eye_id, overwrite_cap_settings=None, + hide_ui=False, ): import cProfile import subprocess @@ -885,7 +886,7 @@ def eye_profiled( from .eye import eye cProfile.runctx( - "eye(timebase, is_alive_flag,ipc_pub_url,ipc_sub_url,ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings)", + "eye(timebase, is_alive_flag,ipc_pub_url,ipc_sub_url,ipc_push_url, user_dir, version, eye_id, overwrite_cap_settings, hide_ui)", { "timebase": timebase, "is_alive_flag": is_alive_flag, @@ -896,6 +897,7 @@ def eye_profiled( "version": version, "eye_id": eye_id, "overwrite_cap_settings": overwrite_cap_settings, + "hide_ui": hide_ui, }, locals(), "eye{}.pstats".format(eye_id), diff --git a/pupil_src/launchables/marker_detectors.py b/pupil_src/launchables/marker_detectors.py index 2b2320114e..b45f2c9c6f 100644 --- a/pupil_src/launchables/marker_detectors.py +++ b/pupil_src/launchables/marker_detectors.py @@ -38,7 +38,11 @@ def circle_detector(ipc_push_url, pair_url, source_path, batch_size=20): from circle_detector import CircleTracker try: - src = File_Source(SimpleNamespace(), source_path, timing=None) + # TODO: we need fill_gaps=True for correct frame indices to paint the circle + # markers on the world stream. But actually we don't want to process gap frames + # with the marker detector. We should make an option to only receive non-gap + # frames, but with gap-like indices? + src = File_Source(SimpleNamespace(), source_path, timing=None, fill_gaps=True) frame = src.get_frame() logger.info("Starting calibration marker detection...") diff --git a/pupil_src/launchables/service.py b/pupil_src/launchables/service.py index ea2b228bf7..8d320ab7f2 100644 --- a/pupil_src/launchables/service.py +++ b/pupil_src/launchables/service.py @@ -334,12 +334,13 @@ def service_profiled( user_dir, version, preferred_remote_port, + hide_ui, ): import cProfile, subprocess, os from .service import service cProfile.runctx( - "service(timebase,eye_procs_alive,ipc_pub_url,ipc_sub_url,ipc_push_url,user_dir,version)", + "service(timebase,eye_procs_alive,ipc_pub_url,ipc_sub_url,ipc_push_url,user_dir,version,preferred_remote_port,hide_ui)", { "timebase": timebase, "eye_procs_alive": eye_procs_alive, @@ -349,6 +350,7 @@ def service_profiled( "user_dir": user_dir, "version": version, "preferred_remote_port": preferred_remote_port, + "hide_ui": hide_ui, }, locals(), "service.pstats", diff --git a/pupil_src/launchables/world.py b/pupil_src/launchables/world.py index 3650577e1b..a98dd08f86 100644 --- a/pupil_src/launchables/world.py +++ b/pupil_src/launchables/world.py @@ -756,6 +756,7 @@ def world_profiled( user_dir, version, preferred_remote_port, + hide_ui, ): import cProfile import subprocess @@ -763,7 +764,7 @@ def world_profiled( from .world import world cProfile.runctx( - "world(timebase, eye_procs_alive, ipc_pub_url,ipc_sub_url,ipc_push_url,user_dir,version,preferred_remote_port)", + "world(timebase, eye_procs_alive, ipc_pub_url,ipc_sub_url,ipc_push_url,user_dir,version,preferred_remote_port, hide_ui)", { "timebase": timebase, "eye_procs_alive": eye_procs_alive, @@ -773,6 +774,7 @@ def world_profiled( "user_dir": user_dir, "version": version, "preferred_remote_port": preferred_remote_port, + "hide_ui": hide_ui, }, locals(), "world.pstats", diff --git a/pupil_src/shared_modules/calibration_routines/__init__.py b/pupil_src/shared_modules/calibration_routines/__init__.py index f63028f067..f1d20ceb97 100644 --- a/pupil_src/shared_modules/calibration_routines/__init__.py +++ b/pupil_src/shared_modules/calibration_routines/__init__.py @@ -12,7 +12,6 @@ # import detector classes from sibling files from .screen_marker_calibration import Screen_Marker_Calibration from .manual_marker_calibration import Manual_Marker_Calibration -from .fingertip_calibration import Fingertip_Calibration from .single_marker_calibration import Single_Marker_Calibration from .natural_features_calibration import Natural_Features_Calibration from .hmd_calibration import HMD_Calibration, HMD_Calibration_3D @@ -30,7 +29,6 @@ calibration_plugins = [ Screen_Marker_Calibration, Manual_Marker_Calibration, - Fingertip_Calibration, Single_Marker_Calibration, Natural_Features_Calibration, HMD_Calibration, diff --git a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/__init__.py b/pupil_src/shared_modules/calibration_routines/fingertip_calibration/__init__.py deleted file mode 100644 index 930e0bd548..0000000000 --- a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2019 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" - -from calibration_routines.fingertip_calibration.fingertip_calibration import ( - Fingertip_Calibration, -) diff --git a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/fingertip_calibration.py b/pupil_src/shared_modules/calibration_routines/fingertip_calibration/fingertip_calibration.py deleted file mode 100644 index c44272aba4..0000000000 --- a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/fingertip_calibration.py +++ /dev/null @@ -1,305 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2019 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" - - -import torch -import os, sys -import cv2 -import numpy as np -import logging - -logger = logging.getLogger(__name__) - -from pyglui import ui -from pyglui.cygl import utils as cygl_utils - -import audio -from calibration_routines import calibration_plugin_base, finish_calibration -from calibration_routines.fingertip_calibration.models import ssd_lite, unet - -if getattr(sys, "frozen", False): - weights_path = os.path.join(sys._MEIPASS, "weights") -else: - weights_path = os.path.join(os.path.split(__file__)[0], "weights") - - -class Fingertip_Calibration(calibration_plugin_base.Calibration_Plugin): - """Calibrate gaze parameters using your fingertip. - Move your head for example horizontally and vertically while gazing at your fingertip - to quickly sample a wide range gaze angles. - """ - - def __init__(self, g_pool, visualize=True): - super().__init__(g_pool) - self.menu = None - - # Initialize CNN pipeline - self.device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") - - # Hand Detector - self.hand_detector_cfg = { - "input_size": 225, - "confidence_thresh": 0.9, - "max_num_detection": 1, - "nms_thresh": 0.45, - "weights_path": os.path.join(weights_path, "hand_detector_model.pkl"), - } - self.hand_transform = BaseTransform( - self.hand_detector_cfg["input_size"], - (117.77, 115.42, 107.29), - (72.03, 69.83, 71.43), - ) - self.hand_detector = ssd_lite.build_ssd_lite(self.hand_detector_cfg) - self.hand_detector.load_state_dict( - torch.load( - self.hand_detector_cfg["weights_path"], - map_location=lambda storage, loc: storage, - ) - ) - self.hand_detector.eval().to(self.device) - - # Fingertip Detector - self.fingertip_detector_cfg = { - "confidence_thresh": 0.6, - "weights_path": os.path.join(weights_path, "fingertip_detector_model.pkl"), - } - self.fingertip_transform = BaseTransform( - 64, (121.97, 119.65, 111.42), (67.58, 65.17, 67.72) - ) - self.fingertip_detector = unet.UNet( - num_classes=10, in_channels=3, depth=4, start_filts=32, up_mode="transpose" - ) - self.fingertip_detector.load_state_dict( - torch.load( - self.fingertip_detector_cfg["weights_path"], - map_location=lambda storage, loc: storage, - ) - ) - self.fingertip_detector.eval().to(self.device) - - self.collect_tips = False - self.visualize = visualize - self.hand_viz = [] - self.finger_viz = [] - - def get_init_dict(self): - return {"visualize": self.visualize} - - def init_ui(self): - super().init_ui() - self.menu.label = "Fingertip Calibration" - self.menu.append( - ui.Info_Text("Calibrate gaze parameters using your fingertip!") - ) - self.menu.append( - ui.Info_Text( - "Hold your index finger still at the center of the field of view of the world camera. " - "Move your head horizontally and then vertically while gazing at your fingertip." - "Then show five fingers to finish the calibration." - ) - ) - if self.device == torch.device("cpu"): - self.menu.append( - ui.Info_Text( - "* No GPU utilized for fingertip detection network. " - "Note that the frame rate will drop during fingertip detection." - ) - ) - else: - self.menu.append( - ui.Info_Text("* GPUs utilized for fingertip detection network") - ) - - self.vis_toggle = ui.Thumb("visualize", self, label="V", hotkey="v") - self.g_pool.quickbar.append(self.vis_toggle) - - def start(self): - if not self.g_pool.capture.online: - logger.error("This calibration requires world capture video input.") - return - super().start() - audio.say("Starting Fingertip Calibration") - logger.info("Starting Fingertip Calibration") - - self.active = True - self.ref_list = [] - self.pupil_list = [] - - def stop(self): - # TODO: redundancy between all gaze mappers -> might be moved to parent class - audio.say("Stopping Fingertip Calibration") - logger.info("Stopping Fingertip Calibration") - self.active = False - self.button.status_text = "" - if self.mode == "calibration": - finish_calibration.finish_calibration( - self.g_pool, self.pupil_list, self.ref_list - ) - elif self.mode == "accuracy_test": - self.finish_accuracy_test(self.pupil_list, self.ref_list) - super().stop() - - def recent_events(self, events): - frame = events.get("frame") - if (self.visualize or self.active) and frame: - orig_img = frame.img - img_width, img_height = frame.width, frame.height - orig_img = cv2.cvtColor(orig_img, cv2.COLOR_BGR2RGB) - - # Hand Detection - x = orig_img.copy() - x = self.hand_transform(x) - x = x.to(self.device) - hand_detections = self.hand_detector(x).detach().numpy()[0][1] - - self.hand_viz = [] - self.finger_viz = [] - for hand_detection in hand_detections: - confidence, x1, y1, x2, y2 = hand_detection - if confidence == 0: - break - - x1 *= img_width - x2 *= img_width - y1 *= img_height - y2 *= img_height - self.hand_viz.append((x1, y1, x2, y2)) - - tl = np.array((x1, y1)) - br = np.array((x2, y2)) - W, H = br - tl - crop_len = np.clip(max(W, H) * 1.25, 1, min(img_width, img_height)) - crop_center = (br + tl) / 2 - crop_center = np.clip( - crop_center, crop_len / 2, (img_width, img_height) - crop_len / 2 - ) - crop_tl = (crop_center - crop_len / 2).astype(np.int) - crop_br = (crop_tl + crop_len).astype(np.int) - - # Fingertip detection - y = orig_img[crop_tl[1] : crop_br[1], crop_tl[0] : crop_br[0]].copy() - y = self.fingertip_transform(y) - y = y.to(self.device) - fingertip_detections = ( - self.fingertip_detector(y).cpu().detach().numpy()[0][:5] - ) - - self.finger_viz.append([]) - detected_fingers = 0 - ref = None - for fingertip_detection in fingertip_detections: - p = np.unravel_index( - fingertip_detection.argmax(), fingertip_detection.shape - ) - if ( - fingertip_detection[p] - >= self.fingertip_detector_cfg["confidence_thresh"] - ): - p = np.array(p) / (fingertip_detection.shape) * ( - crop_br[1] - crop_tl[1], - crop_br[0] - crop_tl[0], - ) + (crop_tl[1], crop_tl[0]) - self.finger_viz[-1].append(p) - detected_fingers += 1 - ref = p - else: - self.finger_viz[-1].append(None) - - if detected_fingers == 1 and self.active: - y, x = ref - ref = { - "screen_pos": (x, y), - "norm_pos": (x / img_width, 1 - (y / img_height)), - "timestamp": frame.timestamp, - } - self.ref_list.append(ref) - elif detected_fingers == 5 and self.active: - if self.collect_tips and len(self.ref_list) > 5: - self.collect_tips = False - self.stop() - elif not self.collect_tips: - self.collect_tips = True - elif detected_fingers == 0: - # hand detections without fingertips are false positives - del self.hand_viz[-1] - del self.finger_viz[-1] - - if self.active: - # always save pupil positions - self.pupil_list.extend(events["pupil"]) - - def gl_display(self): - """ - use gl calls to render - at least: - the published position of the reference - better: - show the detected postion even if not published - """ - if self.active or self.visualize: - # Draw hand detection results - for (x1, y1, x2, y2), fingertips in zip(self.hand_viz, self.finger_viz): - pts = np.array( - [[x1, y1], [x1, y2], [x2, y2], [x2, y1], [x1, y1]], np.int32 - ) - cygl_utils.draw_polyline( - pts, - thickness=3 * self.g_pool.gui_user_scale, - color=cygl_utils.RGBA(0.0, 1.0, 0.0, 1.0), - ) - for tip in fingertips: - if tip is not None: - y, x = tip - cygl_utils.draw_progress( - (x, y), - 0.0, - 1.0, - inner_radius=25 * self.g_pool.gui_user_scale, - outer_radius=35 * self.g_pool.gui_user_scale, - color=cygl_utils.RGBA(1.0, 1.0, 1.0, 1.0), - sharpness=0.9, - ) - - cygl_utils.draw_points( - [(x, y)], - size=10 * self.g_pool.gui_user_scale, - color=cygl_utils.RGBA(1.0, 1.0, 1.0, 1.0), - sharpness=0.9, - ) - - def deinit_ui(self): - """gets called when the plugin get terminated. - either voluntarily or forced. - """ - if self.active: - self.stop() - self.g_pool.quickbar.remove(self.vis_toggle) - self.vis_toggle = None - super().deinit_ui() - - -class BaseTransform: - def __init__(self, size=None, mean=None, std=None): - self.size = size - self.mean = mean - self.std = std - - def __call__(self, image): - x = image.astype(np.float32) - if self.size is not None: - x = cv2.resize(x, dsize=(self.size, self.size)) - if self.mean is not None: - x -= self.mean - if self.std is not None: - x /= self.std - x = torch.from_numpy(x).permute(2, 0, 1) - x = x.unsqueeze(0) - return x diff --git a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/layers/box_utils.py b/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/layers/box_utils.py deleted file mode 100644 index 7dac096399..0000000000 --- a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/layers/box_utils.py +++ /dev/null @@ -1,111 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2019 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" -# Adapted from https://github.com/amdegroot/ssd.pytorch/blob/master/layers/box_utils.py - - -import torch - - -# Adapted from https://github.com/Hakuyume/chainer-ssd -def decode(loc, priors, variances): - """Decode locations from predictions using priors to undo - the encoding we did for offset regression at train time. - Args: - loc (tensor): location predictions for loc layers, - Shape: [num_priors,4] - priors (tensor): Prior boxes in center-offset form. - Shape: [num_priors,4]. - variances: (list[float]) Variances of priorboxes - Return: - decoded bounding box predictions - """ - - boxes = torch.cat( - ( - priors[:, :2] + loc[:, :2] * variances[0] * priors[:, 2:], - priors[:, 2:] * torch.exp(loc[:, 2:] * variances[1]), - ), - 1, - ) - boxes[:, :2] -= boxes[:, 2:] / 2 - boxes[:, 2:] += boxes[:, :2] - return boxes - - -# Original author: Francisco Massa: -# https://github.com/fmassa/object-detection.torch -# Ported to PyTorch by Max deGroot (02/01/2017) -def nms(boxes, scores, overlap=0.5, top_k=10): - """Apply non-maximum suppression at test time to avoid detecting too many - overlapping bounding boxes for a given object. - - Args: - boxes: (tensor) The location preds for the img, Shape: [num_priors,4]. - scores: (tensor) The class predscores for the img, Shape:[num_priors]. - overlap: (float) The overlap thresh for suppressing unnecessary boxes. - top_k: (int) The Maximum number of box preds to consider. - Return: - The indices of the kept boxes with respect to num_priors. - """ - - if boxes.numel() == 0: - return - keep = scores.new(scores.size(0)).zero_().long() - x1 = boxes[:, 0] - y1 = boxes[:, 1] - x2 = boxes[:, 2] - y2 = boxes[:, 3] - area = torch.mul(x2 - x1, y2 - y1) - v, idx = scores.sort(0) # sort in ascending order - # I = I[v >= 0.01] - idx = idx[-top_k:] # indices of the top-k largest vals - xx1 = boxes.new() - yy1 = boxes.new() - xx2 = boxes.new() - yy2 = boxes.new() - w = boxes.new() - h = boxes.new() - - # keep = torch.Tensor() - count = 0 - while idx.numel() > 0: - i = idx[-1] # index of current largest val - # keep.append(i) - keep[count] = i - count += 1 - if idx.size(0) == 1: - break - idx = idx[:-1] # remove kept element from view - # load bboxes of next highest vals - torch.index_select(x1, 0, idx, out=xx1) - torch.index_select(y1, 0, idx, out=yy1) - torch.index_select(x2, 0, idx, out=xx2) - torch.index_select(y2, 0, idx, out=yy2) - # store element-wise max with next highest score - xx1 = torch.clamp(xx1, min=x1[i]) - yy1 = torch.clamp(yy1, min=y1[i]) - xx2 = torch.clamp(xx2, max=x2[i]) - yy2 = torch.clamp(yy2, max=y2[i]) - w.resize_as_(xx2) - h.resize_as_(yy2) - w = xx2 - xx1 - h = yy2 - yy1 - # check sizes of xx1 and xx2.. after each iteration - w = torch.clamp(w, min=0.0) - h = torch.clamp(h, min=0.0) - inter = w * h - # IoU = i / (area(a) + area(b) - i) - rem_areas = torch.index_select(area, 0, idx) # load remaining areas) - union = (rem_areas - inter) + area[i] - IoU = inter / union # store result in iou - # keep only elements with an IoU <= overlap - idx = idx[IoU.le(overlap)] - return keep, count diff --git a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/layers/functions/detection.py b/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/layers/functions/detection.py deleted file mode 100644 index 93135646d0..0000000000 --- a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/layers/functions/detection.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2019 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" -# Adapted from https://github.com/amdegroot/ssd.pytorch/blob/master/layers/functions/detection.py - - -import torch -from torch.autograd import Function -from calibration_routines.fingertip_calibration.models.layers import box_utils - - -class Detect(Function): - """At test time, Detect is the final layer of SSD. Decode location preds, - apply non-maximum suppression to location predictions based on conf - scores and threshold to a top_k number of output predictions for both - confidence score and locations. - """ - - def __init__(self, cfg): - super(Detect, self).__init__() - self.num_classes = cfg["num_classes"] - self.top_k = cfg["max_num_detection"] - self.nms_thresh = cfg["nms_thresh"] - self.confidence_thresh = cfg["confidence_thresh"] - self.bkg_label = 0 - self.variance = [0.1, 0.2] - - def forward(self, loc_data, conf_data, prior_data): - """ - Args: - loc_data: (tensor) Loc preds from loc layers - Shape: [batch,num_priors*4] - conf_data: (tensor) Shape: Conf preds from conf layers - Shape: [batch*num_priors,num_classes] - prior_data: (tensor) Prior boxes and variances from priorbox layers - Shape: [1,num_priors,4] - """ - num = loc_data.size(0) # batch size - num_priors = prior_data.size(0) - output = torch.zeros(num, self.num_classes, self.top_k, 5) - conf_preds = conf_data.view(num, num_priors, self.num_classes).transpose(2, 1) - - # Decode predictions into bboxes. - for i in range(num): - decoded_boxes = box_utils.decode(loc_data[i], prior_data, self.variance) - # For each class, perform nms - conf_scores = conf_preds[i].clone() - - for cl in range(self.num_classes): - if cl == self.bkg_label: - continue - c_mask = conf_scores[cl].gt(self.confidence_thresh) - scores = conf_scores[cl][c_mask] - if scores.dim() == 0: - continue - l_mask = c_mask.unsqueeze(1).expand_as(decoded_boxes) - boxes = decoded_boxes[l_mask].view(-1, 4) - # idx of highest scoring and non-overlapping boxes per class - res = box_utils.nms(boxes, scores, self.nms_thresh, self.top_k) - if res is not None: - ids, count = res - output[i, cl, :count] = torch.cat( - (scores[ids[:count]].unsqueeze(1), boxes[ids[:count]]), 1 - ) - - flt = output.contiguous().view(num, -1, 5) - _, idx = flt[:, :, 0].sort(1, descending=True) - _, rank = idx.sort(1) - flt[(rank < self.top_k).unsqueeze(-1).expand_as(flt)].fill_(0) - return output diff --git a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/layers/functions/prior_box.py b/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/layers/functions/prior_box.py deleted file mode 100644 index 5b67b31bfc..0000000000 --- a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/layers/functions/prior_box.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2019 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" -# Adapted from https://github.com/amdegroot/ssd.pytorch/blob/master/layers/functions/prior_box.py - - -import torch -import math -import itertools - - -class PriorBox(object): - """Compute priorbox coordinates in center-offset form for each source feature map. - """ - - def __init__(self, cfg): - super(PriorBox, self).__init__() - self.img_size = cfg["input_size"] - self.num_priors = len(cfg["aspect_ratios"]) - self.aspect_ratios = cfg["aspect_ratios"] - self.feature_maps = cfg["feature_maps_reso"] - self.steps = [int(self.img_size / f + 1) for f in cfg["feature_maps_reso"]] - self.min_sizes = [self.img_size * f for f in [0.15, 0.3, 0.45, 0.6, 0.75, 0.9]] - self.max_sizes = [self.img_size * f for f in [0.3, 0.45, 0.6, 0.75, 0.9, 1.05]] - self.clip = True - - def forward(self): - mean = [] - for k, f in enumerate(self.feature_maps): - for i, j in itertools.product(range(f), repeat=2): - f_k = self.img_size / self.steps[k] - # unit center x,y - cx = (j + 0.5) / f_k - cy = (i + 0.5) / f_k - - # aspect_ratio: 1; rel size: min_size - s_k = self.min_sizes[k] / self.img_size - mean += [cx, cy, s_k, s_k] - - # aspect_ratio: 1; rel size: sqrt(s_k * s_(k+1)) - s_k_prime = math.sqrt(s_k * (self.max_sizes[k] / self.img_size)) - mean += [cx, cy, s_k_prime, s_k_prime] - - # rest of aspect ratios - for ar in self.aspect_ratios[k]: - mean += [cx, cy, s_k * math.sqrt(ar), s_k / math.sqrt(ar)] - output = torch.Tensor(mean).view(-1, 4) - if self.clip: - output.clamp_(max=1, min=0) - return output diff --git a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/mobilenet.py b/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/mobilenet.py deleted file mode 100644 index 6b0321747d..0000000000 --- a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/mobilenet.py +++ /dev/null @@ -1,90 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2019 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" - - -import torch.nn as nn - - -def conv(inp, oup, stride): - return nn.Sequential( - nn.Conv2d(inp, oup, kernel_size=3, stride=stride, padding=1, bias=False), - nn.BatchNorm2d(oup), - nn.ReLU(inplace=True), - ) - - -def depth_sep_conv(inp, oup, stride): - return nn.Sequential( - # dw - nn.Conv2d( - inp, inp, kernel_size=3, stride=stride, padding=1, groups=inp, bias=False - ), - nn.BatchNorm2d(inp), - nn.ReLU(inplace=True), - # pw - nn.Conv2d(inp, oup, kernel_size=1, stride=1, padding=0, bias=False), - nn.BatchNorm2d(oup), - nn.ReLU(inplace=True), - ) - - -def mobilenet(width_multiplier=1.0): - """MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications - See: https://arxiv.org/abs/1704.04861 - for more details. - - Args: - width_multiplier: the parameter to thin a network uniformly at each layer - - """ - layers = [] - layers += [conv(3, int(32 * width_multiplier), 2)] - layers += [ - depth_sep_conv(int(32 * width_multiplier), int(64 * width_multiplier), 1) - ] - layers += [ - depth_sep_conv(int(64 * width_multiplier), int(128 * width_multiplier), 2) - ] - layers += [ - depth_sep_conv(int(128 * width_multiplier), int(128 * width_multiplier), 1) - ] - layers += [ - depth_sep_conv(int(128 * width_multiplier), int(256 * width_multiplier), 2) - ] - layers += [ - depth_sep_conv(int(256 * width_multiplier), int(256 * width_multiplier), 1) - ] - layers += [ - depth_sep_conv(int(256 * width_multiplier), int(512 * width_multiplier), 2) - ] - layers += [ - depth_sep_conv(int(512 * width_multiplier), int(512 * width_multiplier), 1) - ] - layers += [ - depth_sep_conv(int(512 * width_multiplier), int(512 * width_multiplier), 1) - ] - layers += [ - depth_sep_conv(int(512 * width_multiplier), int(512 * width_multiplier), 1) - ] - layers += [ - depth_sep_conv(int(512 * width_multiplier), int(512 * width_multiplier), 1) - ] - layers += [ - depth_sep_conv(int(512 * width_multiplier), int(512 * width_multiplier), 1) - ] - layers += [ - depth_sep_conv(int(512 * width_multiplier), int(1024 * width_multiplier), 2) - ] - layers += [ - depth_sep_conv(int(1024 * width_multiplier), int(1024 * width_multiplier), 1) - ] - - return layers diff --git a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/ssd_lite.py b/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/ssd_lite.py deleted file mode 100644 index 27b0726acc..0000000000 --- a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/ssd_lite.py +++ /dev/null @@ -1,147 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2019 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" -# Adapted from https://github.com/ShuangXieIrene/ssds.pytorch/blob/master/lib/modeling/ssds/ssd_lite.py & -# https://github.com/amdegroot/ssd.pytorch/blob/master/ssd.py - - -import torch -import torch.nn as nn - -from calibration_routines.fingertip_calibration.models.layers.functions import ( - prior_box, - detection, -) -from calibration_routines.fingertip_calibration.models import mobilenet - - -class SSDLite(nn.Module): - """SSD: Single Shot MultiBox Detector - See: https://arxiv.org/pdf/1512.02325.pdf & - https://arxiv.org/pdf/1611.10012.pdf & - https://arxiv.org/pdf/1801.04381.pdf - for more details. - - Args: - cfg: configurations of the network - base: base layers for input - extras: extra layers that feed to multibox loc and conf layers - head: "multibox head" consists of loc and conf conv layers - """ - - def __init__(self, cfg, base, extras, head): - super(SSDLite, self).__init__() - self.cfg = cfg - self.mobilenet = nn.ModuleList(base) - self.extras = nn.ModuleList(extras) - self.loc = nn.ModuleList(head[0]) - self.conf = nn.ModuleList(head[1]) - self.softmax = nn.Softmax(dim=-1) - self.detect = detection.Detect(cfg) - - with torch.no_grad(): - self.priors = prior_box.PriorBox(self.cfg).forward() - - def forward(self, x): - """Applies network layers on input image(s) x. - - Args: - x: batch of images. Shape: [batch, 3, img_size, img_size]. - - Return: - tensor of output predictions for confidence score and corresponding locations. - Shape: [batch, num_classes, topk, 5] - """ - sources, loc, conf = [list() for _ in range(3)] - - # apply bases layers and cache source layer outputs - for k in range(len(self.mobilenet)): - x = self.mobilenet[k](x) - if k in self.cfg["feature_maps_layer"]: - sources.append(x) - - # apply extra layers and cache source layer outputs - for v in self.extras: - x = v(x) - sources.append(x) - - # apply multibox head to source layers - for (x, l, c) in zip(sources, self.loc, self.conf): - loc.append(l(x).permute(0, 2, 3, 1).contiguous()) - conf.append(c(x).permute(0, 2, 3, 1).contiguous()) - loc = torch.cat([o.view(o.size(0), -1) for o in loc], 1) - conf = torch.cat([o.view(o.size(0), -1) for o in conf], 1) - - # get output predictions for confidence score and locations. - output = self.detect( - loc.view(loc.size(0), -1, 4), # loc preds - self.softmax( - conf.view(conf.size(0), -1, self.cfg["num_classes"]) - ), # conf preds - self.priors.type((x.detach()).type()), # default boxes - ) - return output - - -def add_extras(cfg): - extra_layers = [] - in_channels = None - for layer, out_channels in zip( - cfg["feature_maps_layer"], cfg["feature_maps_channel"] - ): - if layer == "S": - extra_layers += [ - mobilenet.depth_sep_conv(in_channels, out_channels, stride=2) - ] - in_channels = out_channels - - return extra_layers - - -def add_head(cfg): - mbox = [len(ar) + 2 for ar in cfg["aspect_ratios"]] - loc_layers = [] - conf_layers = [] - for in_channels, num_box in zip(cfg["feature_maps_channel"], mbox): - loc_layers += [nn.Conv2d(in_channels, num_box * 4, kernel_size=3, padding=1)] - conf_layers += [ - nn.Conv2d( - in_channels, num_box * cfg["num_classes"], kernel_size=3, padding=1 - ) - ] - - return loc_layers, conf_layers - - -def build_ssd_lite(cfg): - width_multiplier = 0.25 - cfg["num_classes"] = 2 - cfg["aspect_ratios"] = [ - [1 / 2, 1 / 3], - [1 / 2, 1 / 3], - [1 / 2, 1 / 3], - [1 / 2, 1 / 3], - [1 / 2, 1 / 3], - [1 / 2, 1 / 3], - ] - cfg["feature_maps_reso"] = [ - int(cfg["input_size"] / d + 1) for d in [16, 32, 64, 128, 256, 512] - ] - cfg["feature_maps_channel"] = [ - int(d * width_multiplier) for d in [512, 1024, 512, 256, 256, 128] - ] - cfg["feature_maps_layer"] = [11, 13, "S", "S", "S", "S"] - - return SSDLite( - cfg=cfg, - base=mobilenet.mobilenet(width_multiplier), - extras=add_extras(cfg), - head=add_head(cfg), - ) diff --git a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/unet.py b/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/unet.py deleted file mode 100644 index 849ac9456e..0000000000 --- a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/models/unet.py +++ /dev/null @@ -1,249 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2019 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" -# Adapted from https://github.com/jaxony/unet-pytorch/blob/master/model.py - - -import torch -import torch.nn as nn - - -def conv3x3(in_channels, out_channels, stride=1, padding=1, bias=True, groups=1): - return nn.Conv2d( - in_channels, - out_channels, - kernel_size=3, - stride=stride, - padding=padding, - bias=bias, - groups=groups, - ) - - -def upconv2x2(in_channels, out_channels, mode="transpose"): - if mode == "transpose": - return nn.ConvTranspose2d(in_channels, out_channels, kernel_size=2, stride=2) - else: - # out_channels is always going to be the same - # as in_channels - return nn.Sequential( - nn.Upsample(scale_factor=2, mode="bilinear", align_corners=True), - conv1x1(in_channels, out_channels), - ) - - -def conv1x1(in_channels, out_channels, groups=1): - return nn.Conv2d(in_channels, out_channels, kernel_size=1, groups=groups, stride=1) - - -class DownConv(nn.Module): - """ - A helper Module that performs 2 convolutions and 1 MaxPool. - A ReLU activation follows each convolution. - """ - - def __init__(self, in_channels, out_channels, pooling=True): - super(DownConv, self).__init__() - - self.in_channels = in_channels - self.out_channels = out_channels - self.pooling = pooling - - self.conv1 = conv3x3(self.in_channels, self.out_channels) - self.conv2 = conv3x3(self.out_channels, self.out_channels) - - if self.pooling: - self.pool = nn.MaxPool2d(kernel_size=2, stride=2) - - def forward(self, x): - x = torch.relu(self.conv1(x)) - x = torch.relu(self.conv2(x)) - before_pool = x - if self.pooling: - x = self.pool(x) - return x, before_pool - - -class UpConv(nn.Module): - """ - A helper Module that performs 2 convolutions and 1 UpConvolution. - A ReLU activation follows each convolution. - """ - - def __init__( - self, in_channels, out_channels, merge_mode="concat", up_mode="transpose" - ): - super(UpConv, self).__init__() - - self.in_channels = in_channels - self.out_channels = out_channels - self.merge_mode = merge_mode - self.up_mode = up_mode - - self.upconv = upconv2x2(self.in_channels, self.out_channels, mode=self.up_mode) - - if self.merge_mode == "concat": - self.conv1 = conv3x3(2 * self.out_channels, self.out_channels) - else: - # num of input channels to conv2 is same - self.conv1 = conv3x3(self.out_channels, self.out_channels) - self.conv2 = conv3x3(self.out_channels, self.out_channels) - - def forward(self, from_down, from_up): - """ Forward pass - Arguments: - from_down: tensor from the encoder pathway - from_up: upconv'd tensor from the decoder pathway - """ - from_up = self.upconv(from_up) - if self.merge_mode == "concat": - x = torch.cat((from_up, from_down), 1) - else: - x = from_up + from_down - x = torch.relu(self.conv1(x)) - x = torch.relu(self.conv2(x)) - return x - - -class UNet(nn.Module): - """ `UNet` class is based on https://arxiv.org/abs/1505.04597 - - The U-Net is a convolutional encoder-decoder neural network. - Contextual spatial information (from the decoding, - expansive pathway) about an input tensor is merged with - information representing the localization of details - (from the encoding, compressive pathway). - - Modifications to the original paper: - (1) padding is used in 3x3 convolutions to prevent loss - of border pixels - (2) merging outputs does not require cropping due to (1) - (3) residual connections can be used by specifying - UNet(merge_mode='add') - (4) if non-parametric upsampling is used in the decoder - pathway (specified by upmode='upsample'), then an - additional 1x1 2d convolution occurs after upsampling - to reduce channel dimensionality by a factor of 2. - This channel halving happens with the convolution in - the tranpose convolution (specified by upmode='transpose') - """ - - def __init__( - self, - num_classes, - in_channels=3, - depth=5, - start_filts=64, - up_mode="transpose", - merge_mode="concat", - ): - """ - Arguments: - in_channels: int, number of channels in the input tensor. - Default is 3 for RGB images. - depth: int, number of MaxPools in the U-Net. - start_filts: int, number of convolutional filters for the - first conv. - up_mode: string, type of upconvolution. Choices: 'transpose' - for transpose convolution or 'upsample' for nearest neighbour - upsampling. - """ - super(UNet, self).__init__() - - if up_mode in ("transpose", "upsample"): - self.up_mode = up_mode - else: - raise ValueError( - '"{}" is not a valid mode for ' - 'upsampling. Only "transpose" and ' - '"upsample" are allowed.'.format(up_mode) - ) - - if merge_mode in ("concat", "add"): - self.merge_mode = merge_mode - else: - raise ValueError( - '"{}" is not a valid mode for' - "merging up and down paths. " - 'Only "concat" and ' - '"add" are allowed.'.format(up_mode) - ) - - # NOTE: up_mode 'upsample' is incompatible with merge_mode 'add' - if self.up_mode == "upsample" and self.merge_mode == "add": - raise ValueError( - 'up_mode "upsample" is incompatible ' - 'with merge_mode "add" at the moment ' - "because it doesn't make sense to use " - "nearest neighbour to reduce " - "depth channels (by half)." - ) - - self.num_classes = num_classes - self.in_channels = in_channels - self.start_filts = start_filts - self.depth = depth - - self.down_convs = [] - self.up_convs = [] - - # create the encoder pathway and add to a list - for i in range(depth): - ins = self.in_channels if i == 0 else outs - outs = self.start_filts * (2 ** i) - pooling = True if i < depth - 1 else False - - down_conv = DownConv(ins, outs, pooling=pooling) - self.down_convs.append(down_conv) - - # create the decoder pathway and add to a list - # - careful! decoding only requires depth-1 blocks - for i in range(depth - 1): - ins = outs - outs = ins // 2 - up_conv = UpConv(ins, outs, up_mode=up_mode, merge_mode=merge_mode) - self.up_convs.append(up_conv) - - self.conv_final = conv1x1(outs, self.num_classes) - - # add the list of modules to current module - self.down_convs = nn.ModuleList(self.down_convs) - self.up_convs = nn.ModuleList(self.up_convs) - - self.reset_params() - - @staticmethod - def weight_init(m): - if isinstance(m, nn.Conv2d): - nn.init.xavier_normal_(m.weight) - nn.init.constant_(m.bias, 0) - - def reset_params(self): - for i, m in enumerate(self.modules()): - self.weight_init(m) - - def forward(self, x): - encoder_outs = [] - - # encoder pathway, save outputs for merging - for i, module in enumerate(self.down_convs): - x, before_pool = module(x) - encoder_outs.append(before_pool) - - for i, module in enumerate(self.up_convs): - before_pool = encoder_outs[-(i + 2)] - x = module(before_pool, x) - - # No softmax is used. This means you need to use - # nn.CrossEntropyLoss is your training script, - # as this module includes a softmax already. - x = self.conv_final(x) - x = torch.tanh(x) - return x diff --git a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/fingertip_detector_model.pkl b/pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/fingertip_detector_model.pkl deleted file mode 100755 index 16b729100e..0000000000 Binary files a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/fingertip_detector_model.pkl and /dev/null differ diff --git a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/hand_detector_model.pkl b/pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/hand_detector_model.pkl deleted file mode 100755 index a1b0f206c6..0000000000 Binary files a/pupil_src/shared_modules/calibration_routines/fingertip_calibration/weights/hand_detector_model.pkl and /dev/null differ diff --git a/pupil_src/shared_modules/fingertip_detector.py b/pupil_src/shared_modules/fingertip_detector.py deleted file mode 100644 index 1a6c05bbf3..0000000000 --- a/pupil_src/shared_modules/fingertip_detector.py +++ /dev/null @@ -1,420 +0,0 @@ -""" -(*)~--------------------------------------------------------------------------- -Pupil - eye tracking platform -Copyright (C) 2012-2019 Pupil Labs - -Distributed under the terms of the GNU -Lesser General Public License (LGPL v3.0). -See COPYING and COPYING.LESSER for license details. ----------------------------------------------------------------------------~(*) -""" - -import cv2 -import numpy as np -from methods import normalize - - -class FingertipTracker(object): - def __init__(self, wait_interval=30, roi_wait_interval=120): - self.wait_interval = wait_interval - self.roi_wait_interval = roi_wait_interval - self.train_done = 0 - self._ROIpts_1 = [] - self._ROIpts_2 = [] - self._defineROIpts() - self.ROIpts = self._ROIpts_1 - self.method = HSV_Bound() - - self._previous_finger_dict = None - self._predict_motion = None - self._wait_count = 0 - self._roi_wait_count = 0 - self._flag_check = False - self._flag_check_roi = False - - self._contourwidthThres = None - self._contourheightThres = None - self._epsilon = None - self._margin = None - self._kernel_OPEN = None - self._kernel_CLOSE = None - self._finger_area_ratios = [] - self._finger_rects = [] - - def _initParam(self, img_size): - self._contourwidthThres = img_size[0] // 128 - self._contourheightThres = img_size[1] // 24 - self._epsilon = img_size[0] / 256.0 - self._margin = img_size[0] // 128 - kernel_size = img_size[0] // 256 - self._kernel_OPEN = cv2.getStructuringElement( - cv2.MORPH_ELLIPSE, (kernel_size, kernel_size) - ) - kernel_size = img_size[0] // 36 - self._kernel_CLOSE = cv2.getStructuringElement( - cv2.MORPH_ELLIPSE, (kernel_size, kernel_size) - ) - - self._previous_finger_dict = None - self._predict_motion = None - self._wait_count = 0 - self._roi_wait_count = 0 - self.train_done = 0 - self.method.hsv_median = [] - self.ROIpts = self._ROIpts_1 - - def update(self, img, press_key): - finger_dict = None - if press_key == -1 or press_key == 2: - img_size = img.shape[1], img.shape[0] - self._initParam(img_size) - - if self.train_done == 0: - if press_key == 1: - self._trainSkinColorDetector(img) - self.train_done = 1 - self.ROIpts = self._ROIpts_2 - elif self.train_done == 1: - if press_key == 1: - self._trainSkinColorDetector(img) - self.train_done = 2 - self.ROIpts = None - elif self.train_done == 2: - if press_key == 1: - self.train_done = 3 - elif self.train_done == 3: - self.method.train() - self.train_done = 4 - elif self.train_done == 4: - if self._wait_count <= 0 or self._roi_wait_count <= 0: - self._flag_check = True - self._flag_check_roi = False - self._wait_count = self.wait_interval - self._roi_wait_count = self.roi_wait_interval - - if self._flag_check: - finger_dict = self._checkFrame(img) - self._predict_motion = None - if finger_dict is not None: - self._flag_check = True - self._flag_check_roi = True - self._roi_wait_count -= 1 - if self._previous_finger_dict is not None: - self._predict_motion = np.array( - finger_dict["screen_pos"] - ) - np.array(self._previous_finger_dict["screen_pos"]) - else: - if self._flag_check_roi: - self._flag_check = True - self._flag_check_roi = False - else: - self._flag_check = False - self._flag_check_roi = False - - self._wait_count -= 1 - self._previous_finger_dict = finger_dict - - return finger_dict - - def _checkFrame(self, img): - img_size = img.shape[1], img.shape[0] - - # Check whole frame - if not self._flag_check_roi: - b0, b1, b2, b3 = 0, img_size[0], 0, img_size[1] - - # Check roi - else: - previous_fingertip_center = self._previous_finger_dict["screen_pos"] - # Set up the boundary of the roi - temp = img_size[0] / 16 - if self._predict_motion is not None: - predict_center = ( - previous_fingertip_center[0] + self._predict_motion[0], - previous_fingertip_center[1] + self._predict_motion[1], - ) - b0 = predict_center[0] - temp * 0.5 - abs(self._predict_motion[0]) * 2 - b1 = predict_center[0] + temp * 0.5 + abs(self._predict_motion[0]) * 2 - b2 = predict_center[1] - temp * 0.8 - abs(self._predict_motion[1]) * 2 - b3 = predict_center[1] + temp * 2.0 + abs(self._predict_motion[1]) * 2 - else: - predict_center = previous_fingertip_center - b0 = predict_center[0] - temp * 0.5 - b1 = predict_center[0] + temp * 0.5 - b2 = predict_center[1] - temp * 0.8 - b3 = predict_center[1] + temp * 2.0 - - b0 = 0 if b0 < 0 else int(b0) - b1 = img_size[0] - 1 if b1 > img_size[0] - 1 else int(b1) - b2 = 0 if b2 < 0 else int(b2) - b3 = img_size[1] - 1 if b3 > img_size[1] - 1 else int(b3) - col_slice = b0, b1 - row_slice = b2, b3 - img = img[slice(*row_slice), slice(*col_slice)] - - handmask = self.method.generateMask(img) - handmask_smooth = self._smoothmask(handmask) - f_dict = self._findFingertip(handmask_smooth, img_size, b0, b2) - - if f_dict is not None: - norm_pos = normalize(f_dict["fingertip_center"], img_size, flip_y=True) - norm_rect_points = [ - normalize(p, img_size, flip_y=True) for p in f_dict["rect_points"] - ] - return { - "screen_pos": f_dict["fingertip_center"], - "norm_pos": norm_pos, - "norm_rect_points": norm_rect_points, - } - else: - return None - - def _findFingertip(self, handmask_smooth, img_size, b0, b2): - *_, contours, _ = cv2.findContours( - handmask_smooth, mode=cv2.RETR_EXTERNAL, method=cv2.CHAIN_APPROX_TC89_KCOS - ) - if len(contours) == 0: - return None - - self._finger_area_ratios = [] - self._finger_rects = [] - for contour in contours: - x0, y0, w0, h0 = cv2.boundingRect(contour) - if w0 < self._contourwidthThres or h0 < self._contourheightThres: - continue - - width_sum = np.sum(handmask_smooth[y0 : y0 + h0, x0 : x0 + w0], axis=1) - if np.std(width_sum) > 12 * 255: - x1, y1, w1, h1 = x0, y0, w0, np.argmax(width_sum) - if h0 < self._contourheightThres: - continue - - width_sum = width_sum[:h1] - hist, bin_edges = np.histogram(width_sum, bins=5) - local_max = np.where( - np.r_[True, hist[1:] > hist[:-1]] - & np.r_[hist[:-1] > hist[1:], True] - == True - )[0] - if len(local_max) == 0: - continue - - x2, y2, w2, h2 = ( - x1, - y1, - w1, - np.max(np.where(width_sum < bin_edges[np.min(local_max) + 1])), - ) - if h2 < self._contourheightThres: - continue - - new_mask = handmask_smooth[y2 : y2 + h2, x2 : x2 + w2] - if np.sum(new_mask) == 0: - continue - - length_sum = np.sum(new_mask, axis=0) // 255 - length_sum_max_index = np.argmax(length_sum) - - x3, y3, w3, h3 = x2, y2, w2, h2 - small_length_sum_index = np.where(length_sum < np.max(length_sum) / 3)[ - 0 - ] - temp_min = np.where(small_length_sum_index > length_sum_max_index)[0] - if len(temp_min) > 0: - local_min_index = small_length_sum_index[temp_min[0]] - w3 = local_min_index - temp_min = np.where(small_length_sum_index < length_sum_max_index)[0] - if len(temp_min) > 0: - local_min_index = small_length_sum_index[temp_min[-1]] - w3 = w3 - local_min_index - x3 = x3 + local_min_index - if w3 < self._contourwidthThres: - continue - - new_mask = handmask_smooth[y3 : y3 + h3, x3 : x3 + w3] - *_, new_contours, _ = cv2.findContours( - new_mask, mode=cv2.RETR_EXTERNAL, method=cv2.CHAIN_APPROX_TC89_KCOS - ) - if len(new_contours) == 0: - continue - - contour = new_contours[ - np.argmax(np.array([len(c) for c in new_contours])) - ] - contour += np.array((x3, y3)) - - contour_poly = cv2.approxPolyDP(contour, epsilon=self._epsilon, closed=True) - if not 3 <= len(contour_poly) <= 20: - continue - - rect = cv2.minAreaRect(contour_poly) - _, rect_size, _ = rect - rect_width, rect_height = min(rect_size), max(rect_size) - if ( - rect_width < self._contourwidthThres - or rect_height < self._contourheightThres - or rect_height < rect_width * 1.2 - ): - continue - - finger_area_ratio = cv2.contourArea(contour_poly) / ( - rect_size[0] * rect_size[1] - ) - if finger_area_ratio < 1 / 3: - continue - - self._finger_area_ratios.append(finger_area_ratio) - self._finger_rects.append(rect) - - if len(self._finger_area_ratios) == 0: - return None - - best_rect = self._finger_rects[int(np.argmax(self._finger_area_ratios))] - (rect_x, rect_y), (rect_size_1, rect_size_2), rect_angle = best_rect - rect_angle = -rect_angle - rect_angle_cos = np.cos(rect_angle * np.pi / 180.0) - rect_angle_sin = np.sin(rect_angle * np.pi / 180.0) - if rect_size_2 >= rect_size_1 and rect_angle <= 45: - rect_x = rect_x - (rect_size_2 - rect_size_1) / 2 * rect_angle_sin - rect_y = rect_y - (rect_size_2 - rect_size_1) / 2 * rect_angle_cos - rect_size_2 = rect_size_1 - rect_size_1 += self._epsilon * 2 - elif rect_size_2 <= rect_size_1 and rect_angle >= 45: - rect_x = rect_x + (rect_size_1 - rect_size_2) / 2 * rect_angle_cos - rect_y = rect_y - (rect_size_1 - rect_size_2) / 2 * rect_angle_sin - rect_size_1 = rect_size_2 - rect_size_2 += self._epsilon * 2 - elif rect_size_2 <= rect_size_1 and rect_angle <= 45: - rect_size_2 += self._epsilon * 2 - elif rect_size_2 >= rect_size_1 and rect_angle >= 45: - rect_size_1 += self._epsilon * 2 - - if ( - rect_x + b0 > img_size[0] - self._margin - or rect_x + b0 < self._margin - or rect_y + b2 < self._margin - ): - return None - - new_rect = (rect_x, rect_y), (rect_size_1, rect_size_2), -rect_angle - new_mask = np.zeros_like(handmask_smooth) - cv2.drawContours( - new_mask, [np.int0(cv2.boxPoints(new_rect))], 0, 255, thickness=-1 - ) - *_, new_contours, _ = cv2.findContours( - np.bitwise_and(new_mask, handmask_smooth), - mode=cv2.RETR_EXTERNAL, - method=cv2.CHAIN_APPROX_TC89_KCOS, - ) - if len(new_contours) == 0: - return None - - new_contour = new_contours[np.argmax(np.array([len(c) for c in new_contours]))] - (rect_x, rect_y), (rect_size_1, rect_size_2), rect_angle = cv2.minAreaRect( - new_contour - ) - rect_angle = -rect_angle - rect_angle_cos = np.cos(rect_angle * np.pi / 180.0) - rect_angle_sin = np.sin(rect_angle * np.pi / 180.0) - if rect_size_2 >= rect_size_1 and rect_angle <= 45: - rect_x = rect_x - (rect_size_2 - rect_size_1) / 2 * rect_angle_sin - rect_y = rect_y - (rect_size_2 - rect_size_1) / 2 * rect_angle_cos - elif rect_size_2 <= rect_size_1 and rect_angle >= 45: - rect_x = rect_x + (rect_size_1 - rect_size_2) / 2 * rect_angle_cos - rect_y = rect_y - (rect_size_1 - rect_size_2) / 2 * rect_angle_sin - - fingertip_center = float(rect_x + b0), float(rect_y + b2) - if ( - fingertip_center[0] > img_size[0] - self._margin - or min(fingertip_center) < self._margin - ): - return None - - rect_points = cv2.boxPoints(best_rect) + np.array( - (b0, b2) - ) # For visualization of the fingertip detector - return {"fingertip_center": fingertip_center, "rect_points": rect_points} - - def _defineROIpts(self): - roi_len_h = 1 / 20 - roi_len_w = 1 / 100 - self._ROIpts_1 = [] - self._ROIpts_1.append((1 / 3 + 1 / 10 * 0, 1 / 6, roi_len_h, roi_len_w)) - self._ROIpts_1.append((1 / 3 + 1 / 10 * 1, 1 / 6, roi_len_h, roi_len_w)) - self._ROIpts_1.append((1 / 3 + 1 / 10 * 2, 1 / 6, roi_len_h, roi_len_w)) - self._ROIpts_1.append((1 / 3 + 1 / 10 * 3, 1 / 6, roi_len_h, roi_len_w)) - - self._ROIpts_2 = [] - self._ROIpts_2.append((1 / 3 + 1 / 10 * 0, 5 / 6, roi_len_h, roi_len_w)) - self._ROIpts_2.append((1 / 3 + 1 / 10 * 1, 5 / 6, roi_len_h, roi_len_w)) - self._ROIpts_2.append((1 / 3 + 1 / 10 * 2, 5 / 6, roi_len_h, roi_len_w)) - self._ROIpts_2.append((1 / 3 + 1 / 10 * 3, 5 / 6, roi_len_h, roi_len_w)) - - def _trainSkinColorDetector(self, img): - h, w = img.shape[0], img.shape[1] - ROIpts = np.array(self.ROIpts * np.array((h, w, h, w)), dtype=np.int) - ROIimg = [img[p[0] : p[0] + p[2], p[1] : p[1] + p[3]] for p in ROIpts] - self.method.findSkinColorMedian(ROIimg) - - def _smoothmask(self, handmask): - handmask_smooth = cv2.morphologyEx(handmask, cv2.MORPH_OPEN, self._kernel_OPEN) - handmask_smooth = cv2.morphologyEx( - handmask_smooth, cv2.MORPH_CLOSE, self._kernel_CLOSE - ) - return handmask_smooth - - -class HSV_Bound(object): - def __init__(self): - self.c_lower = np.array((2, 5, 25), dtype=np.float) - self.c_upper = np.array((2, 5, 25), dtype=np.float) - self.lowerBound = [] - self.upperBound = [] - self.hsv_median = [] - - def findSkinColorMedian(self, ROIimg): - for r in ROIimg: - ROIimgHSV = cv2.cvtColor(r, cv2.COLOR_BGR2HSV) - ROIimgH = (ROIimgHSV[:, :, 0] + 75) % 180 - ROIimgS = ROIimgHSV[:, :, 1] - ROIimgV = ROIimgHSV[:, :, 2] - ROIimgH_median = np.median(ROIimgH) - index = np.where(ROIimgH <= ROIimgH_median) - self.hsv_median.append( - ( - np.median(ROIimgH[index]), - np.median(ROIimgS[index]), - np.median(ROIimgV[index]), - ) - ) - index = np.where(ROIimgH >= ROIimgH_median) - self.hsv_median.append( - ( - np.median(ROIimgH[index]), - np.median(ROIimgS[index]), - np.median(ROIimgV[index]), - ) - ) - - def train(self): - if len(self.hsv_median): - hsv_median = np.array(self.hsv_median, dtype=np.float) - else: - hsv_median = np.array((80, 80, 128), dtype=np.float) - self.lowerBound = np.clip(hsv_median - self.c_lower, 0, 255) - self.upperBound = np.clip(hsv_median + self.c_upper, 0, 255) - - def generateMask(self, img_test): - frameHSV = cv2.cvtColor(img_test, cv2.COLOR_BGR2HSV) - frameHSV[:, :, 0] = (frameHSV[:, :, 0] + 75) % 180 - frameHSV = cv2.blur(frameHSV, (3, 3)) - - bwList = [ - cv2.inRange(frameHSV, l, p) - for l, p in zip(self.lowerBound, self.upperBound) - ] - HandMask = np.zeros_like(bwList[0]) - for li in bwList: - HandMask = np.bitwise_or(HandMask, li) - - return HandMask diff --git a/pupil_src/shared_modules/gl_utils/utils.py b/pupil_src/shared_modules/gl_utils/utils.py index e8042a1295..91b8ad20e2 100644 --- a/pupil_src/shared_modules/gl_utils/utils.py +++ b/pupil_src/shared_modules/gl_utils/utils.py @@ -8,17 +8,21 @@ See COPYING and COPYING.LESSER for license details. ---------------------------------------------------------------------------~(*) """ +import logging +import math +import numpy as np import OpenGL +import OpenGL.error +from OpenGL.GL import * +from OpenGL.GLU import gluPerspective, gluErrorString + +import glfw # OpenGL.FULL_LOGGING = True OpenGL.ERROR_LOGGING = False -from OpenGL.GL import * -from OpenGL.GLU import gluPerspective -import numpy as np -import math -import glfw +logger = logging.getLogger(__name__) __all__ = [ "make_coord_system_norm_based", @@ -33,6 +37,59 @@ ] +######################################################################################## +# START OPENGL DEBUGGING + +# We are injecting a custom error handling function into PyOpenGL to better investigate +# GLErrors that we are potentially producing in pyglui or cygl and that we don't catch +# appropriately. You can produce OpenGL errors with the following snippets for testing: + +# import ctypes +# gl = ctypes.windll.LoadLibrary("opengl32") # NOTE: this is for windows, modify if needed + +# # This produces `error 1281 (GL_INVALID_VALUE)` +# gl.glViewport(0, 0, -100, 1) + +# # This produces `error 1280 (GL_INVALID_ENUM)` for some reason (!?) +# gl.glBegin() +# gl.glViewport(0, 0, -100, 1) +# gl.glEnd() + +# # Check errors: (will consume flag) +# logger.debug(gl.glGetError()) + + +_original_gl_error_check = OpenGL.error._ErrorChecker.glCheckError + + +def custom_gl_error_handling( + error_checker, result, baseOperation=None, cArguments=None, *args +): + try: + return _original_gl_error_check( + error_checker, result, baseOperation, cArguments, *args + ) + except Exception as e: + logging.error(f"Encountered PyOpenGL error: {e}") + found_more_errors = False + while True: + err = glGetError() + if err == GL_NO_ERROR: + break + if not found_more_errors: + found_more_errors = True + logging.debug("Emptying OpenGL error queue:") + logging.debug(f" glError: {err} -> {gluErrorString(err)}") + if not found_more_errors: + logging.debug("No more errors found in OpenGL error queue!") + + +OpenGL.error._ErrorChecker.glCheckError = custom_gl_error_handling + +# END OPENGL DEBUGGING +######################################################################################## + + def is_window_visible(window): visible = glfw.glfwGetWindowAttrib(window, glfw.GLFW_VISIBLE) iconified = glfw.glfwGetWindowAttrib(window, glfw.GLFW_ICONIFIED) diff --git a/pupil_src/shared_modules/video_capture/utils.py b/pupil_src/shared_modules/video_capture/utils.py index d6d8311938..5a10aabc77 100644 --- a/pupil_src/shared_modules/video_capture/utils.py +++ b/pupil_src/shared_modules/video_capture/utils.py @@ -183,8 +183,13 @@ def is_valid(self): def check_valid(self): try: cont = av.open(self.path) - n = cont.decode(video=0) - _ = next(n) + # Three failure scenarios: + # 1. Broken video -> AVError + # 2. decode() does not yield anything + # 3. decode() yields None + first_frame = next(cont.decode(video=0), None) + if first_frame is None: + raise av.AVError("Video does not contain any frames") except av.AVError: return False else: