From 71662cd4a4af71169e1256120086d709ba3ce2d0 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Thu, 15 Oct 2020 21:53:48 +0200 Subject: [PATCH 01/27] Extract Python dependencies from docs into requirements.txt --- docs/dependencies-macos.md | 22 +----------------- docs/dependencies-ubuntu17.md | 22 +----------------- docs/dependencies-ubuntu18.md | 22 +----------------- docs/dependencies-windows.md | 36 +----------------------------- requirements.txt | 42 +++++++++++++++++++++++++++++++++++ 5 files changed, 46 insertions(+), 98 deletions(-) create mode 100644 requirements.txt diff --git a/docs/dependencies-macos.md b/docs/dependencies-macos.md index f53866159d..0b68cb63b4 100644 --- a/docs/dependencies-macos.md +++ b/docs/dependencies-macos.md @@ -67,27 +67,7 @@ We recommend using a [virtual environment](https://docs.python.org/3/tutorial/ve ```sh # Upgrade pip to latest version. This is necessary for some dependencies. python -m pip install --upgrade pip - -pip install cysignals -pip install cython -pip install msgpack==0.5.6 -pip install numexpr -pip install packaging -pip install psutil -pip install pyaudio -pip install pyopengl -pip install pyzmq -pip install scikit-learn -pip install scipy -pip install glfw -pip install git+https://github.com/zeromq/pyre - -pip install pupil-apriltags -pip install pupil-detectors -pip install git+https://github.com/pupil-labs/PyAV -pip install git+https://github.com/pupil-labs/pyuvc -pip install git+https://github.com/pupil-labs/pyndsi -pip install git+https://github.com/pupil-labs/pyglui +pip install -r requirements.txt ``` **NOTE:** Installing **pyglui** might fail on newer versions of **macOS** due to missing OpenGL headers. In this case, you need to install Xcode which comes with the required header files. diff --git a/docs/dependencies-ubuntu17.md b/docs/dependencies-ubuntu17.md index d0c3bb5b3c..c46b55f5bc 100644 --- a/docs/dependencies-ubuntu17.md +++ b/docs/dependencies-ubuntu17.md @@ -157,27 +157,7 @@ We recommend using a [virtual environment](https://docs.python.org/3/tutorial/ve ```sh # Upgrade pip to latest version. This is necessary for some dependencies. python -m pip install --upgrade pip - -pip install cysignals -pip install cython -pip install msgpack==0.5.6 -pip install numexpr -pip install packaging -pip install psutil -pip install pyaudio -pip install pyopengl -pip install pyzmq -pip install scikit-learn -pip install scipy -pip install glfw -pip install git+https://github.com/zeromq/pyre - -pip install pupil-apriltags -pip install pupil-detectors -pip install git+https://github.com/pupil-labs/PyAV -pip install git+https://github.com/pupil-labs/pyuvc -pip install git+https://github.com/pupil-labs/pyndsi -pip install git+https://github.com/pupil-labs/pyglui +pip install -r requirements.txt ``` **NOTE**: If you get the error `ImportError: No module named 'cv2'` when trying to run Pupil, please refer to the section [OpenCV Troubleshooting](#opencv-troubleshooting) above. diff --git a/docs/dependencies-ubuntu18.md b/docs/dependencies-ubuntu18.md index bebc9f722e..81c5a47074 100644 --- a/docs/dependencies-ubuntu18.md +++ b/docs/dependencies-ubuntu18.md @@ -68,27 +68,7 @@ We recommend using a [virtual environment](https://docs.python.org/3/tutorial/ve ```sh # Upgrade pip to latest version. This is necessary for some dependencies. python -m pip install --upgrade pip - -pip install cysignals -pip install cython -pip install msgpack==0.5.6 -pip install numexpr -pip install packaging -pip install psutil -pip install pyaudio -pip install pyopengl -pip install pyzmq -pip install scikit-learn -pip install scipy -pip install glfw -pip install git+https://github.com/zeromq/pyre - -pip install pupil-apriltags -pip install pupil-detectors -pip install git+https://github.com/pupil-labs/PyAV -pip install git+https://github.com/pupil-labs/pyuvc -pip install git+https://github.com/pupil-labs/pyndsi -pip install git+https://github.com/pupil-labs/pyglui +pip install -r requirements.txt ``` ### OpenCV Troubleshooting diff --git a/docs/dependencies-windows.md b/docs/dependencies-windows.md index 4a12899454..5516459f93 100644 --- a/docs/dependencies-windows.md +++ b/docs/dependencies-windows.md @@ -56,43 +56,9 @@ We recommend using a [virtual environment](https://docs.python.org/3/tutorial/ve ```sh # Upgrade pip to latest version. This is necessary for some dependencies. python -m pip install --upgrade pip - -pip install cython -pip install msgpack==0.5.6 -pip install numexpr -pip install opencv-python==3.* -pip install packaging -pip install psutil -pip install pyaudio -pip install pyopengl -pip install pyzmq -pip install scikit-learn -pip install scipy -pip install glfw -pip install win_inet_pton -pip install git+https://github.com/zeromq/pyre - -pip install pupil-apriltags -pip install pupil-detectors +pip install -r requirements.txt ``` -## Pupil Labs Python Wheels - -In addition to these libraries, you will need to install some Pupil-Labs support libraries. Since building them for Windows is also not automated yet, we provide some prebuilt wheels that you can use. If you want to build the support libraries yourself as well, you will have to look for install instructions on the respective GitHub repositories. - -Download the following Python wheels from Pupil Labs github repos: - -- [pyglui](https://github.com/pupil-labs/pyglui/releases/latest) -- [pyav](https://github.com/pupil-labs/pyav/releases/latest) -- [pyndsi](https://github.com/pupil-labs/pyndsi/releases/latest) -- [pyuvc](https://github.com/pupil-labs/pyuvc/releases/latest) - -`pyuvc` requires that you download Microsoft Visual C++ 2010 Redistributable from [microsoft](https://www.microsoft.com/en-us/download/details.aspx?id=14632). The `pthreadVC2` lib, which is used by libuvc, depends on `msvcr100.dll`. - -Open your command prompt and `Run as administrator` in the directory where the wheels are downloaded. - -- Install all wheels with `pip install X` (where X is the name of the `.whl` file) -- You can check that libs are installed with `python import X` statements in the command prompt where `X` is the name of the lib. ## Modifying Pupil to Work with Windows diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..e916bd3aa8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,42 @@ +### +### Third-party +### +cython +msgpack==0.5.6 +numexpr +packaging +psutil +pyaudio +pyopengl +pyzmq +scikit-learn +scipy +glfw +git+https://github.com/zeromq/pyre + +cysignals ; platform_system != "Windows" + +win_inet_pton ; platform_system == "Windows" +opencv-python==3.* ; platform_system == "Windows" + +### +### Pupil-Labs +### +pupil-apriltags==1.0.3 +pupil-detectors==1.1.1 + +# pupil-labs/PyAV 0.4.5 +av @ https://github.com/pupil-labs/PyAV/archive/v0.4.5.zip ; platform_system != "Windows" +av @ https://github.com/pupil-labs/PyAV/releases/download/v0.4.5/av-0.4.5-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" + +# pupil-labs/pyuvc +uvc @ https://github.com/pupil-labs/pyuvc/archive/v0.13.zip ; platform_system != "Windows" +uvc @ https://github.com/pupil-labs/pyuvc/releases/download/v0.13/uvc-0.13-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" + +# pupil-labs/pyglui 1.28 +pyglui @ https://github.com/pupil-labs/pyglui/archive/v1.28.zip ; platform_system != "Windows" +pyglui @ https://github.com/pupil-labs/pyglui/releases/download/v1.28/pyglui-1.28-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" + +# pupil-labs/pyndsi 1.3 +ndsi @ https://github.com/pupil-labs/pyndsi/archive/v1.3.zip ; platform_system != "Windows" +ndsi @ https://github.com/pupil-labs/pyndsi/releases/download/v1.3/ndsi-1.3-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" From ea3102a50b4c07f88fbe747bf545faaf2970f9a2 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Thu, 15 Oct 2020 21:54:42 +0200 Subject: [PATCH 02/27] Add instruction about using the requirements.txt --- docs/dependencies-macos.md | 2 +- docs/dependencies-ubuntu17.md | 2 +- docs/dependencies-ubuntu18.md | 2 +- docs/dependencies-windows.md | 3 +-- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/docs/dependencies-macos.md b/docs/dependencies-macos.md index 0b68cb63b4..fc4c215863 100644 --- a/docs/dependencies-macos.md +++ b/docs/dependencies-macos.md @@ -62,7 +62,7 @@ make && make install ### Install Python Libraries -We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. +We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. To install all Python dependencies, you can use the [`requirements.txt`](https://github.com/pupil-labs/pupil/blob/master/requirements.txt) file from the root of the `pupil` repository. ```sh # Upgrade pip to latest version. This is necessary for some dependencies. diff --git a/docs/dependencies-ubuntu17.md b/docs/dependencies-ubuntu17.md index c46b55f5bc..5f10ffaae8 100644 --- a/docs/dependencies-ubuntu17.md +++ b/docs/dependencies-ubuntu17.md @@ -152,7 +152,7 @@ sudo ldconfig ### Install Python Libraries -We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. +We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. To install all Python dependencies, you can use the [`requirements.txt`](https://github.com/pupil-labs/pupil/blob/master/requirements.txt) file from the root of the `pupil` repository. ```sh # Upgrade pip to latest version. This is necessary for some dependencies. diff --git a/docs/dependencies-ubuntu18.md b/docs/dependencies-ubuntu18.md index 81c5a47074..6fa00a5799 100644 --- a/docs/dependencies-ubuntu18.md +++ b/docs/dependencies-ubuntu18.md @@ -63,7 +63,7 @@ sudo udevadm trigger ### Install Python Libraries -We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. +We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. To install all Python dependencies, you can use the [`requirements.txt`](https://github.com/pupil-labs/pupil/blob/master/requirements.txt) file from the root of the `pupil` repository. ```sh # Upgrade pip to latest version. This is necessary for some dependencies. diff --git a/docs/dependencies-windows.md b/docs/dependencies-windows.md index 5516459f93..a1300204e3 100644 --- a/docs/dependencies-windows.md +++ b/docs/dependencies-windows.md @@ -48,10 +48,9 @@ If you downloaded the linked installer: - Check the box `Add Python to PATH`. This will add Python to your System PATH Environment Variable. - Check the box `Install for all users`. **Note:** By default this will install Python to `C:\Program Files\Python36`. Some build scripts may fail to start Python due to spaces in the path name. So, you may want to consider installing Python to `C:\Python36` instead. - ## Install Python Libraries -We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. +We recommend using a [virtual environment](https://docs.python.org/3/tutorial/venv.html) for running Pupil. To install all Python dependencies, you can use the [`requirements.txt`](https://github.com/pupil-labs/pupil/blob/master/requirements.txt) file from the root of the `pupil` repository. ```sh # Upgrade pip to latest version. This is necessary for some dependencies. From 8e302edec1174ebf0b8d40208446f09dd7c33c3f Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Thu, 15 Oct 2020 21:55:32 +0200 Subject: [PATCH 03/27] Add Windows-specific note about pyuvc requiring msvcr100.dll --- docs/dependencies-windows.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dependencies-windows.md b/docs/dependencies-windows.md index a1300204e3..c4b06cc57c 100644 --- a/docs/dependencies-windows.md +++ b/docs/dependencies-windows.md @@ -58,6 +58,7 @@ python -m pip install --upgrade pip pip install -r requirements.txt ``` +**NOTE:** `pyuvc` requires that you download Microsoft Visual C++ 2010 Redistributable from [microsoft](https://www.microsoft.com/en-us/download/details.aspx?id=14632). The `pthreadVC2` lib, which is used by libuvc, depends on `msvcr100.dll`. ## Modifying Pupil to Work with Windows From 0bfa295e32dd3570c8155472875144392c9b816a Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Thu, 15 Oct 2020 23:33:31 +0200 Subject: [PATCH 04/27] Update Pupil-Labs Python dependency URLs Temporarily point URLs to revisions that contain pyproject.toml files. --- requirements.txt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index e916bd3aa8..7e8a6361ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -26,17 +26,23 @@ pupil-apriltags==1.0.3 pupil-detectors==1.1.1 # pupil-labs/PyAV 0.4.5 -av @ https://github.com/pupil-labs/PyAV/archive/v0.4.5.zip ; platform_system != "Windows" +# TODO: Replace the line below with the following: +# av @ git+https://github.com/pupil-labs/PyAV@v0.4.5 ; platform_system != "Windows" +# when the branch `add-pyproject` is merged and there is a new release +av @ git+https://github.com/pupil-labs/PyAV@add-pyproject ; platform_system != "Windows" av @ https://github.com/pupil-labs/PyAV/releases/download/v0.4.5/av-0.4.5-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" # pupil-labs/pyuvc -uvc @ https://github.com/pupil-labs/pyuvc/archive/v0.13.zip ; platform_system != "Windows" +# TODO: Replace the line below with the following: +# uvc @ git+https://github.com/pupil-labs/pyuvc@v0.13 ; platform_system != "Windows" +# when there is a new release +uvc @ git+https://github.com/pupil-labs/pyuvc@master ; platform_system != "Windows" uvc @ https://github.com/pupil-labs/pyuvc/releases/download/v0.13/uvc-0.13-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" # pupil-labs/pyglui 1.28 -pyglui @ https://github.com/pupil-labs/pyglui/archive/v1.28.zip ; platform_system != "Windows" +pyglui @ git+https://github.com/pupil-labs/pyglui@v1.28 ; platform_system != "Windows" pyglui @ https://github.com/pupil-labs/pyglui/releases/download/v1.28/pyglui-1.28-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" # pupil-labs/pyndsi 1.3 -ndsi @ https://github.com/pupil-labs/pyndsi/archive/v1.3.zip ; platform_system != "Windows" +ndsi @ git+https://github.com/pupil-labs/pyndsi@v1.3 ; platform_system != "Windows" ndsi @ https://github.com/pupil-labs/pyndsi/releases/download/v1.3/ndsi-1.3-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" From 3f9469bea5478cef053916872310895b5665624b Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 21 Oct 2020 13:38:18 +0200 Subject: [PATCH 05/27] Update requirements.txt wi packaging minimal requirement Co-authored-by: Pablo Prietz --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7e8a6361ab..33b01e06f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ cython msgpack==0.5.6 numexpr -packaging +packaging>=20.0 psutil pyaudio pyopengl From 56e70fb435ce93e5c228df4d14e1fa476acbc568 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 28 Oct 2020 11:12:08 +0100 Subject: [PATCH 06/27] Update pupil-labs dependencies to release tags --- requirements.txt | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/requirements.txt b/requirements.txt index 33b01e06f3..22571a14c2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,24 +25,18 @@ opencv-python==3.* ; platform_system == "Windows" pupil-apriltags==1.0.3 pupil-detectors==1.1.1 -# pupil-labs/PyAV 0.4.5 -# TODO: Replace the line below with the following: -# av @ git+https://github.com/pupil-labs/PyAV@v0.4.5 ; platform_system != "Windows" -# when the branch `add-pyproject` is merged and there is a new release -av @ git+https://github.com/pupil-labs/PyAV@add-pyproject ; platform_system != "Windows" -av @ https://github.com/pupil-labs/PyAV/releases/download/v0.4.5/av-0.4.5-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" +# pupil-labs/PyAV 0.4.6 +av @ git+https://github.com/pupil-labs/PyAV@v0.4.6 ; platform_system != "Windows" +av @ https://github.com/pupil-labs/PyAV/releases/download/v0.4.6/av-0.4.6-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" -# pupil-labs/pyuvc -# TODO: Replace the line below with the following: -# uvc @ git+https://github.com/pupil-labs/pyuvc@v0.13 ; platform_system != "Windows" -# when there is a new release -uvc @ git+https://github.com/pupil-labs/pyuvc@master ; platform_system != "Windows" -uvc @ https://github.com/pupil-labs/pyuvc/releases/download/v0.13/uvc-0.13-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" +# pupil-labs/pyuvc 0.14 +uvc @ git+https://github.com/pupil-labs/pyuvc@v0.14 ; platform_system != "Windows" +uvc @ https://github.com/pupil-labs/pyuvc/releases/download/v0.14/uvc-0.14-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" # pupil-labs/pyglui 1.28 pyglui @ git+https://github.com/pupil-labs/pyglui@v1.28 ; platform_system != "Windows" pyglui @ https://github.com/pupil-labs/pyglui/releases/download/v1.28/pyglui-1.28-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" -# pupil-labs/pyndsi 1.3 -ndsi @ git+https://github.com/pupil-labs/pyndsi@v1.3 ; platform_system != "Windows" -ndsi @ https://github.com/pupil-labs/pyndsi/releases/download/v1.3/ndsi-1.3-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" +# pupil-labs/pyndsi 1.4 +ndsi @ git+https://github.com/pupil-labs/pyndsi@v1.4 ; platform_system != "Windows" +ndsi @ https://github.com/pupil-labs/pyndsi/releases/download/v1.4/ndsi-1.4-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" From 78477ebf57377703c07bc2d70c086d1b62f319a6 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Wed, 28 Oct 2020 11:42:18 +0100 Subject: [PATCH 07/27] requirements: Name pyre module name --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 22571a14c2..f84b3100b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ pyzmq scikit-learn scipy glfw -git+https://github.com/zeromq/pyre +pyre @ git+https://github.com/zeromq/pyre cysignals ; platform_system != "Windows" From abe183902e252f30b8e6b49ca6695f4220f08a38 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 28 Oct 2020 14:24:01 +0100 Subject: [PATCH 08/27] Update pyuvc version tag to include build fixes --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f84b3100b2..1f73da9d61 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ av @ git+https://github.com/pupil-labs/PyAV@v0.4.6 ; platform_system != "Windows av @ https://github.com/pupil-labs/PyAV/releases/download/v0.4.6/av-0.4.6-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" # pupil-labs/pyuvc 0.14 -uvc @ git+https://github.com/pupil-labs/pyuvc@v0.14 ; platform_system != "Windows" +uvc @ git+https://github.com/pupil-labs/pyuvc@v0.14.1 ; platform_system != "Windows" # Minor patch fixes build issues in pyproject.toml uvc @ https://github.com/pupil-labs/pyuvc/releases/download/v0.14/uvc-0.14-cp36-cp36m-win_amd64.whl ; platform_system == "Windows" # pupil-labs/pyglui 1.28 From d84cc238df8b8e582a7cb8493a1329ba8b4e21a5 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 28 Oct 2020 14:26:27 +0100 Subject: [PATCH 09/27] Install wheel in order to avoid legacy python setup.py installbuilds --- docs/dependencies-macos.md | 2 +- docs/dependencies-ubuntu17.md | 2 +- docs/dependencies-ubuntu18.md | 2 +- docs/dependencies-windows.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/dependencies-macos.md b/docs/dependencies-macos.md index fc4c215863..14ae3a2385 100644 --- a/docs/dependencies-macos.md +++ b/docs/dependencies-macos.md @@ -66,7 +66,7 @@ We recommend using a [virtual environment](https://docs.python.org/3/tutorial/ve ```sh # Upgrade pip to latest version. This is necessary for some dependencies. -python -m pip install --upgrade pip +python -m pip install --upgrade pip wheel pip install -r requirements.txt ``` diff --git a/docs/dependencies-ubuntu17.md b/docs/dependencies-ubuntu17.md index 5f10ffaae8..2375c9b66a 100644 --- a/docs/dependencies-ubuntu17.md +++ b/docs/dependencies-ubuntu17.md @@ -156,7 +156,7 @@ We recommend using a [virtual environment](https://docs.python.org/3/tutorial/ve ```sh # Upgrade pip to latest version. This is necessary for some dependencies. -python -m pip install --upgrade pip +python -m pip install --upgrade pip wheel pip install -r requirements.txt ``` diff --git a/docs/dependencies-ubuntu18.md b/docs/dependencies-ubuntu18.md index 6fa00a5799..00e2fc79e6 100644 --- a/docs/dependencies-ubuntu18.md +++ b/docs/dependencies-ubuntu18.md @@ -67,7 +67,7 @@ We recommend using a [virtual environment](https://docs.python.org/3/tutorial/ve ```sh # Upgrade pip to latest version. This is necessary for some dependencies. -python -m pip install --upgrade pip +python -m pip install --upgrade pip wheel pip install -r requirements.txt ``` diff --git a/docs/dependencies-windows.md b/docs/dependencies-windows.md index c4b06cc57c..c49e54fc78 100644 --- a/docs/dependencies-windows.md +++ b/docs/dependencies-windows.md @@ -54,7 +54,7 @@ We recommend using a [virtual environment](https://docs.python.org/3/tutorial/ve ```sh # Upgrade pip to latest version. This is necessary for some dependencies. -python -m pip install --upgrade pip +python -m pip install --upgrade pip wheel pip install -r requirements.txt ``` From ce03bdbf62a752a36cc017a25e87ee85b57ea9b5 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Thu, 29 Oct 2020 11:58:38 +0100 Subject: [PATCH 10/27] Update apriltag requirements --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1f73da9d61..b85eb31504 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,7 @@ opencv-python==3.* ; platform_system == "Windows" ### ### Pupil-Labs ### -pupil-apriltags==1.0.3 +pupil-apriltags==1.0.4 pupil-detectors==1.1.1 # pupil-labs/PyAV 0.4.6 From 48ced330e63a695438429e291c8eacbeb3169e51 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Fri, 30 Oct 2020 10:01:09 +0100 Subject: [PATCH 11/27] Load runtime pupil detection plugins and start them automatically in eye process --- pupil_src/launchables/eye.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index fb5254068e..92fb493333 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -202,8 +202,24 @@ def get_timestamp(): g_pool.get_timestamp = get_timestamp g_pool.get_now = get_time_monotonic + def load_runtime_pupil_detection_plugins(): + from plugin import import_runtime_plugins + from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin + + plugins_path = os.path.join(g_pool.user_dir, "plugins") + + for plugin in import_runtime_plugins(plugins_path): + if not isinstance(plugin, type(PupilDetectorPlugin)): + continue + if not issubclass(plugin, PupilDetectorPlugin): + continue + if plugin is PupilDetectorPlugin: + continue + yield plugin + default_2d, default_3d, available_detectors = available_detector_plugins() - plugins = manager_classes + source_classes + available_detectors + [Roi] + runtime_detectors = list(load_runtime_pupil_detection_plugins()) + plugins = manager_classes + source_classes + available_detectors + runtime_detectors + [Roi] g_pool.plugin_by_name = {p.__name__: p for p in plugins} preferred_names = [ @@ -511,6 +527,14 @@ def set_window_size(): # with incorrect settings that were loaded from session settings. plugins_to_load.append(overwrite_cap_settings) + # Add runtime plugins to the list of plugins to load with default arguments, + # if not already restored from session settings + plugins_to_load_names = set(name for name, _ in plugins_to_load) + for runtime_detector in runtime_detectors: + runtime_name = runtime_detector.__name__ + if runtime_name not in plugins_to_load_names: + plugins_to_load.append((runtime_name, {})) + g_pool.plugins = Plugin_List(g_pool, plugins_to_load) if not g_pool.capture: From 2e1a293342bcbab347af3456edce47b709235125 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Fri, 30 Oct 2020 10:03:11 +0100 Subject: [PATCH 12/27] Add stop_eye_plugin notification handling in eye process --- pupil_src/launchables/eye.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index 92fb493333..1959d25dd5 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -679,6 +679,17 @@ def window_should_update(): ) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") + elif ( + subject.startswith("stop_eye_plugin") + and notification["target"] == g_pool.process + ): + try: + plugin_to_stop = g_pool.plugin_by_name[notification["name"]] + except KeyError as err: + logger.error(f"Attempt to load unknown plugin: {err}") + else: + plugin_to_stop.alive = False + g_pool.plugins.clean() for plugin in g_pool.plugins: plugin.on_notify(notification) From b844db1044cbbf85de53ab836cdda2f5c3a1ed6a Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Fri, 30 Oct 2020 10:03:57 +0100 Subject: [PATCH 13/27] Hide pupil detection plugins from plugin manager in Capture --- pupil_src/launchables/world.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pupil_src/launchables/world.py b/pupil_src/launchables/world.py index c9139bff85..f771e26015 100644 --- a/pupil_src/launchables/world.py +++ b/pupil_src/launchables/world.py @@ -159,6 +159,7 @@ def detection_enabled_setter(is_on: bool): from gaze_mapping import registered_gazer_classes from gaze_mapping.gazer_base import GazerBase + from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin from fixation_detector import Fixation_Detector from recorder import Recorder from display_recent_gaze import Display_Recent_Gaze @@ -239,6 +240,9 @@ def get_timestamp(): runtime_plugins = import_runtime_plugins( os.path.join(g_pool.user_dir, "plugins") ) + runtime_plugins = [ + p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin) + ] user_plugins = [ Pupil_Groups, NetworkApiPlugin, From 774fd6605c2df33dd8fa6cebe17657092c9c4b8b Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Fri, 30 Oct 2020 10:22:51 +0100 Subject: [PATCH 14/27] Apply black to eye.py --- pupil_src/launchables/eye.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index 1959d25dd5..85e1bd3b95 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -219,7 +219,13 @@ def load_runtime_pupil_detection_plugins(): default_2d, default_3d, available_detectors = available_detector_plugins() runtime_detectors = list(load_runtime_pupil_detection_plugins()) - plugins = manager_classes + source_classes + available_detectors + runtime_detectors + [Roi] + plugins = ( + manager_classes + + source_classes + + available_detectors + + runtime_detectors + + [Roi] + ) g_pool.plugin_by_name = {p.__name__: p for p in plugins} preferred_names = [ From d398534fd85248414ff8e8ccc0bec9d1d151973e Mon Sep 17 00:00:00 2001 From: Ryan Prendergast Date: Tue, 3 Nov 2020 15:59:58 -0600 Subject: [PATCH 15/27] Update circle_detector.py Fixed typo in circle detector bench. --- pupil_src/shared_modules/circle_detector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pupil_src/shared_modules/circle_detector.py b/pupil_src/shared_modules/circle_detector.py index 8ca553a27b..16fbd2b654 100644 --- a/pupil_src/shared_modules/circle_detector.py +++ b/pupil_src/shared_modules/circle_detector.py @@ -702,7 +702,7 @@ def bench(): sts, img = cap.read() # img = cv2.imread('/Users/mkassner/Desktop/manual_calibration_marker-01.png') gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) - print(len(find_concetric_circles(gray, visual_debug=img))) + print(len(find_concentric_circles(gray, visual_debug=img))) cv2.imshow("img", img) cv2.waitKey(1) # return From 4b9f6f545db2070a80b0852a470e5f44aac60c60 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Wed, 4 Nov 2020 10:18:08 +0100 Subject: [PATCH 16/27] Update .travis.yml --- .travis.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index febbdccef4..7f5c55ced0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,13 +22,11 @@ jobs: - name: black formatting check language: python + before_install: pip install -U pip # Travis automatically runs `pip install -r requirements.txt` if such a file is present. # Source: https://docs.travis-ci.com/user/languages/python/#dependency-management - # Since we do not need the requirements to be installed, we overwrite - install: ~ - before_script: - - pip install -U pip - - pip install black + # Since we only need black to be installed, we overwrite with: + install: pip install black script: - > black . --check --exclude pupil_src/tests || ( From cbb65ff2160c32b3b3871adaab6bea91fc0b9ac8 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 4 Nov 2020 19:21:26 +0100 Subject: [PATCH 17/27] Implement pupil producers progress reporting based on current file source index --- pupil_src/shared_modules/pupil_producers.py | 41 ++++++++++++------- .../video_capture/file_backend.py | 30 +++++++++++++- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/pupil_src/shared_modules/pupil_producers.py b/pupil_src/shared_modules/pupil_producers.py index 89f7a9a129..dbc03a7f65 100644 --- a/pupil_src/shared_modules/pupil_producers.py +++ b/pupil_src/shared_modules/pupil_producers.py @@ -328,7 +328,7 @@ def __init__(self, g_pool): self.data_sub = zmq_tools.Msg_Receiver( zmq_ctx, g_pool.ipc_sub_url, - topics=("pupil", "notify.file_source.video_finished"), + topics=("pupil", "notify.file_source"), hwm=100_000, ) @@ -354,6 +354,7 @@ def __init__(self, g_pool): # Start offline pupil detection if not complete yet: self.eye_video_loc = [None, None] self.eye_frame_num = [0, 0] + self.eye_frame_idx = [-1, -1] # start processes for eye_id in range(2): @@ -381,6 +382,7 @@ def start_eye_process(self, eye_id): video_loc = existing_locs[0] n_valid_frames = np.count_nonzero(self.videoset.lookup.container_idx > -1) self.eye_frame_num[eye_id] = n_valid_frames + self.eye_frame_idx = [-1, -1] capure_settings = "File_Source", {"source_path": video_loc, "timing": None} self.notify_all( @@ -395,17 +397,21 @@ def start_eye_process(self, eye_id): @property def detection_progress(self) -> float: - total = sum(self.eye_frame_num) - # TODO: Figure out the number of frames independent of 3d detection - detected = self._pupil_data_store.count_collected(detector_tag="3d") - if total: - return min( - detected / total, - 1.0, - ) - else: + + if not sum(self.eye_frame_num): return 0.0 + progress_by_eye = [0.0, 0.0] + + for eye_id in (0, 1): + total_frames = self.eye_frame_num[eye_id] + current_index = self.eye_frame_idx[eye_id] + progress = (current_index + 1) / total_frames + progress = max(0.0, min(progress, 1.0)) + progress_by_eye[eye_id] = progress + + return min(progress_by_eye) + def stop_eye_process(self, eye_id): self.notify_all({"subject": "eye_process.should_stop", "eye_id": eye_id}) self.eye_video_loc[eye_id] = None @@ -425,15 +431,20 @@ def recent_events(self, events): else: payload = self.data_sub.deserialize_payload(*remaining_frames) if payload["subject"] == "file_source.video_finished": - for eyeid in (0, 1): - if self.eye_video_loc[eyeid] == payload["source_path"]: - logger.debug("eye {} process complete".format(eyeid)) - self.detection_status[eyeid] = "complete" - self.stop_eye_process(eyeid) + for eye_id in (0, 1): + if self.eye_video_loc[eye_id] == payload["source_path"]: + logger.debug("eye {} process complete".format(eye_id)) + self.eye_frame_idx[eye_id] = self.eye_frame_num[eye_id] + self.detection_status[eye_id] = "complete" + self.stop_eye_process(eye_id) break if self.eye_video_loc == [None, None]: data = self._pupil_data_store.as_pupil_data_bisector() self.publish_new(pupil_data_bisector=data) + if payload["subject"] == "file_source.current_frame_index": + for eye_id in (0, 1): + if self.eye_video_loc[eye_id] == payload["source_path"]: + self.eye_frame_idx[eye_id] = payload["index"] self.menu_icon.indicator_stop = self.detection_progress diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index 342e9e4f1d..42f7b681fc 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -16,7 +16,7 @@ import typing as T from abc import ABC, abstractmethod from multiprocessing import cpu_count -from time import sleep +from time import sleep, monotonic import av import numpy as np @@ -490,6 +490,34 @@ def recent_events_own_timing(self, events): self._recent_frame = frame events["frame"] = frame + if self.timing is None: + self._notify_current_index(min_time_diff_sec=1.0) + + def _notify_current_index(self, min_time_diff_sec: float): + timestamp_now = monotonic() + current_index = self.get_frame_index() + + try: + timestamp_last = self.__last_current_index_notification_timestamp + last_index = self.__last_current_index_notification_index + except AttributeError: + should_send_notification = True + else: + should_send_notification = True + should_send_notification &= (timestamp_now - timestamp_last) >= min_time_diff_sec + should_send_notification &= current_index != last_index + + if should_send_notification: + self.notify_all( + { + "subject": "file_source.current_frame_index", + "index": current_index, + "source_path": self.source_path, + } + ) + self.__last_current_index_notification_timestamp = timestamp_now + self.__last_current_index_notification_index = current_index + def seek_to_frame(self, seek_pos): try: target_entry = self.videoset.lookup[seek_pos] From e6017d23c426becc407c02aa4c460a106e6d00c7 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 4 Nov 2020 19:23:06 +0100 Subject: [PATCH 18/27] Fix checking for instance of class "type" Co-authored-by: Pablo Prietz --- pupil_src/launchables/eye.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index 85e1bd3b95..33bf245033 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -209,7 +209,7 @@ def load_runtime_pupil_detection_plugins(): plugins_path = os.path.join(g_pool.user_dir, "plugins") for plugin in import_runtime_plugins(plugins_path): - if not isinstance(plugin, type(PupilDetectorPlugin)): + if not isinstance(plugin, type): continue if not issubclass(plugin, PupilDetectorPlugin): continue From 3ab7892b08b5e14ae08eb582786aaae6b665f853 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 4 Nov 2020 19:31:07 +0100 Subject: [PATCH 19/27] Apply black formatting --- pupil_src/shared_modules/pupil_producers.py | 2 +- pupil_src/shared_modules/video_capture/file_backend.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pupil_src/shared_modules/pupil_producers.py b/pupil_src/shared_modules/pupil_producers.py index dbc03a7f65..3e2800fb9a 100644 --- a/pupil_src/shared_modules/pupil_producers.py +++ b/pupil_src/shared_modules/pupil_producers.py @@ -409,7 +409,7 @@ def detection_progress(self) -> float: progress = (current_index + 1) / total_frames progress = max(0.0, min(progress, 1.0)) progress_by_eye[eye_id] = progress - + return min(progress_by_eye) def stop_eye_process(self, eye_id): diff --git a/pupil_src/shared_modules/video_capture/file_backend.py b/pupil_src/shared_modules/video_capture/file_backend.py index 42f7b681fc..61a28b1ff7 100644 --- a/pupil_src/shared_modules/video_capture/file_backend.py +++ b/pupil_src/shared_modules/video_capture/file_backend.py @@ -504,7 +504,9 @@ def _notify_current_index(self, min_time_diff_sec: float): should_send_notification = True else: should_send_notification = True - should_send_notification &= (timestamp_now - timestamp_last) >= min_time_diff_sec + should_send_notification &= ( + timestamp_now - timestamp_last + ) >= min_time_diff_sec should_send_notification &= current_index != last_index if should_send_notification: From a3fcba6bbeeca74e7b5019787584bf249dbe2e05 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Wed, 4 Nov 2020 20:54:47 +0100 Subject: [PATCH 20/27] Support stopping eye plugins via notifications --- pupil_src/launchables/eye.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index fb5254068e..ed054555ca 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -655,6 +655,17 @@ def window_should_update(): ) except KeyError as err: logger.error(f"Attempt to load unknown plugin: {err}") + elif ( + subject.startswith("stop_eye_plugin") + and notification["target"] == g_pool.process + ): + try: + plugin_to_stop = g_pool.plugin_by_name[notification["name"]] + except KeyError as err: + logger.error(f"Attempt to stop unknown plugin: {err}") + else: + plugin_to_stop.alive = False + g_pool.plugins.clean() for plugin in g_pool.plugins: plugin.on_notify(notification) From 30fb38ca1cd466285e20a97b8bca3b9fb950770e Mon Sep 17 00:00:00 2001 From: basile Date: Wed, 4 Nov 2020 15:38:11 -0500 Subject: [PATCH 21/27] fix cpu graph refresh, improve performance and UI look --- pupil_src/launchables/eye.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index ed054555ca..6db248f4a7 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -733,10 +733,9 @@ def window_should_update(): for result in event.get(EVENT_KEY, ()): pupil_socket.send(result) - cpu_graph.update() - # GL drawing if window_should_update(): + cpu_graph.update() if is_window_visible(main_window): consume_events_and_render_buffer() glfw.poll_events() From f0a1e75956aa5d9f43cc8b4a1551875a0c450a8b Mon Sep 17 00:00:00 2001 From: basile Date: Thu, 5 Nov 2020 13:01:28 -0500 Subject: [PATCH 22/27] reduce calls to glfw.window_should_close to screen fps for performance --- pupil_src/launchables/eye.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pupil_src/launchables/eye.py b/pupil_src/launchables/eye.py index ed054555ca..32498ee862 100644 --- a/pupil_src/launchables/eye.py +++ b/pupil_src/launchables/eye.py @@ -588,7 +588,8 @@ def window_should_update(): glfw.swap_interval(0) # Event loop - while not glfw.window_should_close(main_window): + window_should_close = False + while not window_should_close: if notify_sub.new_data: t, notification = notify_sub.recv() @@ -740,6 +741,7 @@ def window_should_update(): if is_window_visible(main_window): consume_events_and_render_buffer() glfw.poll_events() + window_should_close = glfw.window_should_close(main_window) # END while running From 87d38c30c7b6b2d861578e6f8752f48478580a71 Mon Sep 17 00:00:00 2001 From: Roman Roibu Date: Mon, 9 Nov 2020 10:10:13 +0100 Subject: [PATCH 23/27] Remove PupilDetectorPlugins from plugin list in player --- pupil_src/launchables/player.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pupil_src/launchables/player.py b/pupil_src/launchables/player.py index b6aa076c07..cdb8529772 100644 --- a/pupil_src/launchables/player.py +++ b/pupil_src/launchables/player.py @@ -115,6 +115,7 @@ def player( from gaze_producer.gaze_from_offline_calibration import ( GazeFromOfflineCalibration, ) + from pupil_detector_plugins.detector_base_plugin import PupilDetectorPlugin from system_graphs import System_Graphs from system_timelines import System_Timelines from blink_detection import Offline_Blink_Detection @@ -150,6 +151,9 @@ def interrupt_handler(sig, frame): signal.signal(signal.SIGINT, interrupt_handler) runtime_plugins = import_runtime_plugins(os.path.join(user_dir, "plugins")) + runtime_plugins = [ + p for p in runtime_plugins if not issubclass(p, PupilDetectorPlugin) + ] system_plugins = [ Log_Display, Seek_Control, From 001d3b76fa9cffaf87ab4d57c8594530ecbcb38d Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 10 Nov 2020 14:50:35 +0100 Subject: [PATCH 24/27] Check if 3d gaze is in front of camera. If it is not, flip direction Precedence: https://github.com/pupil-labs/hmd-eyes/blob/master/plugin/Scripts/GazeData.cs#L182-L187 --- .../shared_modules/gaze_mapping/gazer_3d/gazer_headset.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pupil_src/shared_modules/gaze_mapping/gazer_3d/gazer_headset.py b/pupil_src/shared_modules/gaze_mapping/gazer_3d/gazer_headset.py index 9514fe2a86..d68a1b8dd7 100644 --- a/pupil_src/shared_modules/gaze_mapping/gazer_3d/gazer_headset.py +++ b/pupil_src/shared_modules/gaze_mapping/gazer_3d/gazer_headset.py @@ -157,6 +157,10 @@ def _predict_single(self, x): gaze_3d = self._toWorld(gaze_point) normal_3d = np.dot(self.rotation_matrix, pupil_normal) + # Check if gaze is in front of camera. If it is not, flip direction. + if gaze_3d[-1] < 0: + gaze_3d *= -1.0 + g = { "eye_center_3d": eye_center.tolist(), "gaze_normal_3d": normal_3d.tolist(), @@ -284,6 +288,10 @@ def _predict_single(self, x): if nearest_intersection_point is None: return None + # Check if gaze is in front of camera. If it is not, flip direction. + if nearest_intersection_point[-1] < 0: + nearest_intersection_point *= -1.0 + g = { "eye_centers_3d": {0: s0_center.tolist(), 1: s1_center.tolist()}, "gaze_normals_3d": {0: s0_normal.tolist(), 1: s1_normal.tolist()}, From 4281f8f2f86cda7f4fa486beb66c41cbd97cf7e0 Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Tue, 10 Nov 2020 16:55:29 +0100 Subject: [PATCH 25/27] Add pupil-specific pye3d visualization code to Pupil --- .../__init__.py} | 2 +- .../visualizer_pye3d/eye.py | 300 ++++++++++++++++++ .../visualizer_pye3d/pose.py | 101 ++++++ .../visualizer_pye3d/utilities.py | 92 ++++++ 4 files changed, 494 insertions(+), 1 deletion(-) rename pupil_src/shared_modules/pupil_detector_plugins/{visualizer_pye3d.py => visualizer_pye3d/__init__.py} (99%) create mode 100644 pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/eye.py create mode 100644 pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/pose.py create mode 100644 pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/utilities.py diff --git a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d.py b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/__init__.py similarity index 99% rename from pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d.py rename to pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/__init__.py index 357834cf43..4f89a79ea0 100644 --- a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d.py +++ b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/__init__.py @@ -17,7 +17,7 @@ from pyglui.cygl.utils import RGBA from visualizer import Visualizer -from pye3d.geometry.eye import LeGrandEye +from .eye import LeGrandEye from collections import deque diff --git a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/eye.py b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/eye.py new file mode 100644 index 0000000000..d80d9cb9cd --- /dev/null +++ b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/eye.py @@ -0,0 +1,300 @@ +""" +(*)~--------------------------------------------------------------------------- +Pupil - eye tracking platform +Copyright (C) 2012-2019 Pupil Labs + +Distributed under the terms of the GNU +Lesser General Public License (LGPL v3.0). +See COPYING and COPYING.LESSER for license details. +---------------------------------------------------------------------------~(*) +""" +import cv2 +import numpy as np +from OpenGL.GL import * + +from .pose import PosedObject +from .utilities import ( + normalize, + rotate_v1_on_v2, + sph2cart, + transform_as_homogeneous_point, + transform_as_homogeneous_vector, +) + + +class BasicEye(PosedObject): + def __init__(self): + super(BasicEye, self).__init__(pose=np.eye(4), extrinsics=None, children=()) + + self._gaze_vector = PosedObject() + self.eyeball_center = [0.0, 0.0, 0.0] + + def update_from_gaze_point(self, gaze_point): + new_gaze_vector = transform_as_homogeneous_vector( + normalize(gaze_point - self.eyeball_center), self.extrinsics + ) + self.update_from_gaze_vector(new_gaze_vector) + + def update_from_spherical(self, phi, theta): + new_gaze_vector = sph2cart(phi, theta) + self.update_from_gaze_vector(new_gaze_vector) + + def update_from_gaze_vector(self, new_gaze_vector): + rotation = rotate_v1_on_v2([0.0, 0.0, 1.0], new_gaze_vector) + self._gaze_vector.rmat = rotation + + def move_to_point(self, point): + self.translate(point - self.eyeball_center) + + @property + def eyeball_center(self): + return self.tvec + + @eyeball_center.setter + def eyeball_center(self, point): + self.tvec = np.asarray(point) + + @property + def gaze_vector(self): + return (self._gaze_vector.pose @ self.pose)[:3, 2] + + def __str__(self): + return "\n".join("{}:{}".format(k, v) for k, v in self.__dict__.items()) + + +class LeGrandEye(BasicEye): + def __init__( + self, + eyeball_radius=12.0, + cornea_radius=7.8, + iris_radius=6.0, + n_refraction=1.3375, + camera=None, + ): + + super(LeGrandEye, self).__init__() + + self.model_type = "LeGrand" + + # PUPIL + distance_eyeball_pupil = np.sqrt(eyeball_radius ** 2 - iris_radius ** 2) + self.__pupil_center = [0.0, 0.0, distance_eyeball_pupil] + self.pupil_radius = 2.0 + self.pupil_normal = np.asarray([0.0, 0.0, 1.0]) + + # IRIS + self.__iris_center = self.__pupil_center + self.iris_radius = iris_radius + self.iris_normal = np.asarray([0.0, 0.0, 1.0]) + self.iris_color = [46 / 255.0, 220 / 255.0, 255.0 / 255.0] + + # CORNEA + h = np.sqrt(cornea_radius ** 2 - iris_radius ** 2) + distance_eyeball_cornea = distance_eyeball_pupil - h + self.__cornea_center = np.asarray([0, 0, distance_eyeball_cornea]) + self.cornea_radius = cornea_radius + + # EYEBALL + self.eyeball_radius = eyeball_radius + + # self.translate(np.asarray([0., 0., 35.])) + # self.update_from_gaze_vector(np.asarray([0., 0., -1])) + + # PHYSICAL CONSTANTS + self.n_refraction = n_refraction + + # CAMERA POINTED AT EYE + self.camera = camera + + # GL SETUP + self.eyeball_alpha = np.arccos(distance_eyeball_pupil / self.eyeball_radius) + self.cornea_alpha = np.arccos(4.0 / self.cornea_radius) / 1.0 + self.set_up_gl_vertices() + + @property + def cornea_center(self): + cornea_center = transform_as_homogeneous_point( + self.__cornea_center, self.pose @ self._gaze_vector.pose + ) + return cornea_center + + @property + def iris_center(self): + iris_center = transform_as_homogeneous_point( + self.__iris_center, self.pose @ self._gaze_vector.pose + ) + return iris_center + + @property + def pupil_center(self): + pupil_center = transform_as_homogeneous_point( + self.__pupil_center, self.pose @ self._gaze_vector.pose + ) + return pupil_center + + def set_up_gl_vertices(self): + + # EYEBALL + self.central_ring_eyeball = [ + [self.eyeball_radius * np.sin(phi), 0, self.eyeball_radius * np.cos(phi)] + for phi in np.linspace( + self.eyeball_alpha, 2 * np.pi - self.eyeball_alpha, 30 + ) + ] + self.rings_eyeball = [self.central_ring_eyeball] + for phi in np.linspace(0, np.pi, 20): + central_ring_rotated = [ + cv2.Rodrigues(np.asarray([0.0, 0.0, phi]))[0] @ v + for v in self.central_ring_eyeball + ] + self.rings_eyeball.append(central_ring_rotated) + + # IRIS + angles = [phi for phi in np.linspace(0, 2 * np.pi, 40)] + self.iris_quads = [] + for i in range(len(angles) - 1): + self.iris_quads.append( + [ + np.array((np.cos(angles[i]), np.sin(angles[i]), 0)), + np.array((np.cos(angles[i + 1]), np.sin(angles[i + 1]), 0)), + np.array((np.cos(angles[i + 1]), np.sin(angles[i + 1]), 0)), + np.array((np.cos(angles[i]), np.sin(angles[i]), 0)), + ] + ) + + # CORNEA + self.central_ring_cornea = [ + [self.cornea_radius * np.sin(phi), 0, self.cornea_radius * np.cos(phi)] + for phi in np.linspace(-self.cornea_alpha, self.cornea_alpha, 20) + ] + + self.rings_cornea = [self.central_ring_cornea] + for phi in np.linspace(0, np.pi, 10): + central_ring_rotated = [ + cv2.Rodrigues(np.asarray([0.0, 0.0, phi]))[0] @ v + for v in self.central_ring_cornea + ] + self.rings_cornea.append(central_ring_rotated) + + def draw_gl( + self, + draw_eyeball=True, + draw_iris=True, + draw_cornea=True, + draw_gaze=True, + alpha=1.0, + ): + + glPushMatrix() + + glLoadIdentity() + # if self.camera is not None: + # glMultMatrixf(self.camera.pose.T) + glMultMatrixf(self.pose.T) + glMultMatrixf(self._gaze_vector.pose.T) + + glPushMatrix() + + glPolygonMode(GL_FRONT_AND_BACK, GL_LINE) + + if draw_gaze: + glLineWidth(2.0) + glColor4f(1, 1, 1, 1.0 * alpha) + glBegin(GL_LINES) + glVertex3f(*[0, 0, 0]) + glVertex3f(*[0, 0, 600]) + glEnd() + + # DRAW EYEBALL + if draw_eyeball: + + # glPolygonMode(GL_FRONT_AND_BACK, GL_FILL) + glColor4f(0.6, 0.6, 1.0, 1.0 * alpha) + glLineWidth(1.0) + + glPushMatrix() + for i in range(len(self.rings_eyeball) - 1): + for j in range(len(self.rings_eyeball[i]) - 1): + glBegin(GL_QUADS) + glVertex3f( + self.rings_eyeball[i][j][0], + self.rings_eyeball[i][j][1], + self.rings_eyeball[i][j][2], + ) + glVertex3f( + self.rings_eyeball[i][j + 1][0], + self.rings_eyeball[i][j + 1][1], + self.rings_eyeball[i][j + 1][2], + ) + glVertex3f( + self.rings_eyeball[i + 1][j + 1][0], + self.rings_eyeball[i + 1][j + 1][1], + self.rings_eyeball[i + 1][j + 1][2], + ) + glVertex3f( + self.rings_eyeball[i + 1][j][0], + self.rings_eyeball[i + 1][j][1], + self.rings_eyeball[i + 1][j][2], + ) + glEnd() + glPopMatrix() + + # DRAW IRIS + if draw_iris: + + glPolygonMode(GL_FRONT_AND_BACK, GL_FILL) + glColor4f( + self.iris_color[0], self.iris_color[1], self.iris_color[2], 0.4 * alpha + ) + + glPushMatrix() + glTranslate(0, 0, self.__pupil_center[2]) + for quad in self.iris_quads: + glBegin(GL_QUADS) + glVertex3f(*(quad[0] * self.pupil_radius)) + glVertex3f(*(quad[1] * self.pupil_radius)) + glVertex3f(*(quad[2] * self.iris_radius)) + glVertex3f(*(quad[3] * self.iris_radius)) + glEnd() + glPopMatrix() + + # DRAW CORNEA + if draw_cornea: + + glPolygonMode(GL_FRONT_AND_BACK, GL_LINE) + glColor4f(1, 1, 1, 0.3 * alpha) + glLineWidth(1.0) + + glPushMatrix() + glTranslate(0, 0, self.__cornea_center[2]) + for i in range(len(self.rings_cornea) - 1): + for j in range(len(self.rings_cornea[i]) - 1): + glBegin(GL_QUADS) + glVertex3f( + self.rings_cornea[i][j][0], + self.rings_cornea[i][j][1], + self.rings_cornea[i][j][2], + ) + glVertex3f( + self.rings_cornea[i][j + 1][0], + self.rings_cornea[i][j + 1][1], + self.rings_cornea[i][j + 1][2], + ) + glVertex3f( + self.rings_cornea[i + 1][j + 1][0], + self.rings_cornea[i + 1][j + 1][1], + self.rings_cornea[i + 1][j + 1][2], + ) + glVertex3f( + self.rings_cornea[i + 1][j][0], + self.rings_cornea[i + 1][j][1], + self.rings_cornea[i + 1][j][2], + ) + glEnd() + glPopMatrix() + + glPopMatrix() + + glPopMatrix() + + glPolygonMode(GL_FRONT_AND_BACK, GL_FILL) diff --git a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/pose.py b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/pose.py new file mode 100644 index 0000000000..2587a4abd9 --- /dev/null +++ b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/pose.py @@ -0,0 +1,101 @@ +""" +(*)~--------------------------------------------------------------------------- +Pupil - eye tracking platform +Copyright (C) 2012-2019 Pupil Labs + +Distributed under the terms of the GNU +Lesser General Public License (LGPL v3.0). +See COPYING and COPYING.LESSER for license details. +---------------------------------------------------------------------------~(*) +""" +import cv2 +import numpy as np + + +class PosedObject(object): + def __init__(self, pose=np.eye(4), extrinsics=None, children=(), parents=()): + + self.parents = parents + self.children = children + self._pose = np.eye( + 4 + ) # Needed during initialization for first call of pose.setter + + if type(pose) in [np.ndarray, list]: + self.pose = np.array(pose) + else: + self.pose = self.pose_from_extrinsics(extrinsics) + + @property + def pose(self): + return self._pose.copy() + + @pose.setter + def pose(self, new_pose): + for child in self.children: + child.pose = new_pose @ np.linalg.inv(self.pose) @ child.pose + self._pose = new_pose + + def translate(self, translation): + pose = self.pose.copy() + pose[:3, 3] += translation + self.pose = pose + + def rotate(self, rotation): + pose = self.pose.copy() + pose[:3, :3] = rotation @ pose[:3, :3] + self.pose = pose + + @staticmethod + def extrinsics_from_pose(pose): + rot = pose[:3, :3] + trans = pose[:3, 3] + extrinsics = np.eye(4) + extrinsics[:3, :3] = rot.T + extrinsics[:3, 3] = -rot.T @ trans + return extrinsics + + @staticmethod + def pose_from_extrinsics(extrinsics): + rot = extrinsics[:3, :3] + trans = extrinsics[:3, 3] + pose = np.eye(4) + pose[:3, :3] = rot.T + pose[:3, 3] = -rot.T @ trans + return pose + + @property + def tvec(self): + return self._pose[:3, 3] + + @tvec.setter + def tvec(self, new_tvec): + pose = self.pose + pose[:3, 3] = new_tvec + self.pose = pose + + @property + def rmat(self): + return self._pose[:3, :3] + + @rmat.setter + def rmat(self, new_rmat): + pose = self.pose + pose[:3, :3] = new_rmat + self.pose = pose + + @property + def rvec(self): + return cv2.Rodrigues(self.rmat)[0] + + @rvec.setter + def rvec(self, new_rvec): + self._pose[:3, :3] = cv2.Rodrigues(new_rvec)[0] + + @property + def extrinsics(self): + return self.extrinsics_from_pose(self.pose) + + @extrinsics.setter + def extrinsics(self, new_extrinsics): + self.pose = self.pose_from_extrinsics(new_extrinsics) diff --git a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/utilities.py b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/utilities.py new file mode 100644 index 0000000000..2223fef1cf --- /dev/null +++ b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/utilities.py @@ -0,0 +1,92 @@ +""" +(*)~--------------------------------------------------------------------------- +Pupil - eye tracking platform +Copyright (C) 2012-2019 Pupil Labs + +Distributed under the terms of the GNU +Lesser General Public License (LGPL v3.0). +See COPYING and COPYING.LESSER for license details. +---------------------------------------------------------------------------~(*) +""" +import numpy as np + + +def cart2sph(x): + + phi = np.arctan2(x[2], x[0]) + theta = np.arccos(x[1] / np.linalg.norm(x)) + + return phi, theta # Todo: This seems to be opposite to the pupil code + + +def sph2cart(phi, theta): + + result = np.empty(3) + + result[0] = np.sin(theta) * np.cos(phi) + result[1] = np.cos(theta) + result[2] = np.sin(theta) * np.sin(phi) + + return result + + +def normalize(v, axis=-1): + + return v / np.linalg.norm(v, axis=axis) + + +def enclosed_angle(v1, v2, unit="deg", axis=-1): + + v1 = normalize(v1, axis=axis) + v2 = normalize(v2, axis=axis) + + alpha = np.arccos(np.clip(np.dot(v1.T, v2), -1, 1)) + + if unit == "deg": + return 180.0 / np.pi * alpha + else: + return alpha + + +def make_homogeneous_vector(v): + + return np.hstack((v, [0.0])) + + +def make_homogeneous_point(p): + return np.hstack((p, [1.0])) + + +def transform_as_homogeneous_point(p, trafo): + p = make_homogeneous_point(p) + return (trafo @ p)[:3] + + +def transform_as_homogeneous_vector(v, trafo): + v = make_homogeneous_vector(v) + return (trafo @ v)[:3] + + +def rotate_v1_on_v2(v1, v2): + + v1 = normalize(v1) + v2 = normalize(v2) + cos_angle = np.dot(v1, v2) + + if not np.allclose(np.abs(cos_angle), 1): + u = np.cross(v1, v2) + s = np.linalg.norm(u) + c = np.dot(v1, v2) + + I = np.eye(3) + ux = np.asarray([[0, -u[2], u[1]], [u[2], 0, -u[0]], [-u[1], u[0], 0]]) + + R = I + ux + np.dot(ux, ux) * (1 - c) / s ** 2 + + elif np.allclose(cos_angle, 1): + R = np.eye(3) + + elif np.allclose(cos_angle, -1): + R = -np.eye(3) + + return R From 3c1bffcceac2dafcd018d5d7db6ec12d85e9070c Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Wed, 11 Nov 2020 13:55:14 +0100 Subject: [PATCH 26/27] pye3d_plugin: Support specific pye3d version 0.0.1 only --- .../pupil_detector_plugins/pye3d_plugin.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py b/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py index 32f8ca7de9..ca9919fd63 100644 --- a/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py +++ b/pupil_src/shared_modules/pupil_detector_plugins/pye3d_plugin.py @@ -10,6 +10,7 @@ """ import logging +import pye3d from pye3d.detector_3d import Detector3D, CameraModel from pyglui import ui @@ -19,6 +20,16 @@ logger = logging.getLogger(__name__) +version_installed = getattr(pye3d, "__version__", "0.0.1") +version_supported = "0.0.1" + +if version_installed != version_supported: + logger.info( + f"Requires pye3d version {version_supported} " + f"(Installed: {version_installed})" + ) + raise ImportError + class Pye3DPlugin(PupilDetectorPlugin): uniqueness = "by_class" From 4c7dcec051a90e4d3fec67b610b581a56471402c Mon Sep 17 00:00:00 2001 From: Pablo Prietz Date: Wed, 11 Nov 2020 15:56:21 +0100 Subject: [PATCH 27/27] Apply black format --- .../pupil_detector_plugins/visualizer_pye3d/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/__init__.py b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/__init__.py index a511efde88..1b57804c63 100644 --- a/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/__init__.py +++ b/pupil_src/shared_modules/pupil_detector_plugins/visualizer_pye3d/__init__.py @@ -39,6 +39,7 @@ from .eye import LeGrandEye + class Eye_Visualizer(Visualizer): def __init__(self, g_pool, focal_length): super().__init__(g_pool, "Debug Visualizer", False)