From ae3bc8d3ad6dc5a5889846537eaf02c2c4008701 Mon Sep 17 00:00:00 2001 From: samypr100 <3933065+samypr100@users.noreply.github.com> Date: Fri, 20 Jan 2023 13:06:14 -0500 Subject: [PATCH] Support for Python Optional Dependencies Split out visualization and ml modules as optional dependencies --- .github/workflows/windows.yml | 3 +- 3rdparty/README_SYCL.md | 2 +- CHANGELOG.md | 1 + README.md | 2 +- cpp/pybind/make_install_pip_package.cmake | 2 +- cpp/pybind/make_python_package.cmake | 1 + docker/Dockerfile.openblas | 1 + docs/arm.rst | 34 +++++++++---------- docs/docker.in.rst | 2 +- docs/getting_started.in.rst | 13 +++++--- docs/tutorial/sensor/realsense.rst | 6 ++-- python/open3d/__init__.py | 21 +++++++++--- python/requirements.txt | 2 -- python/requirements_gui.txt | 2 ++ python/setup.py | 40 +++++++++++++++++++++-- util/ci_utils.sh | 4 ++- 16 files changed, 98 insertions(+), 38 deletions(-) create mode 100644 python/requirements_gui.txt diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 069adeeaae04..71658b2ff57c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -279,6 +279,7 @@ jobs: run: | $ErrorActionPreference = 'Stop' python -m pip install -r python/requirements.txt + python -m pip install -r python/requirements_gui.txt python -m pip install -r python/requirements_jupyter_build.txt - name: Config @@ -402,7 +403,7 @@ jobs: $PIP_PKG_NAME=(Get-ChildItem open3d*-$py_tag-*.whl).Name } echo "Installing Open3D wheel $PIP_PKG_NAME in virtual environment..." - python -m pip install "$PIP_PKG_NAME" + python -m pip install "$PIP_PKG_NAME[gui,ml]" python -c "import open3d; print('Imported:', open3d)" python -c "import open3d; print('CUDA enabled: ', open3d.core.cuda.is_available())" diff --git a/3rdparty/README_SYCL.md b/3rdparty/README_SYCL.md index 5a6120f753af..bfa95a1376c0 100644 --- a/3rdparty/README_SYCL.md +++ b/3rdparty/README_SYCL.md @@ -98,7 +98,7 @@ Open3D is designed to make use of the SYCL GPU devices. ## List of oneAPI Python packages -To make `pip install open3d` works out-of-the box on SYCL-enabled platforms, +To make `pip install open3d[gui,ml]` works out-of-the box on SYCL-enabled platforms, we can utilize runtime libraries released via PyPI. This feature needs to be implemented. diff --git a/CHANGELOG.md b/CHANGELOG.md index a0880065ba51..89cafd0c7bad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ * Fix raycasting scene: Allow setting of number of threads that are used for building a raycasting scene * Fix Python bindings for CUDA device synchronization, voxel grid saving (PR #5425) * Support msgpack versions without cmake +* Introduce new optional dependencies support, allowing you to do `pip install open3d` with any of `gui`, `ml`, `tf`, `torch`, `all` such as `pip install open3d[gui,ml]` ## 0.13 diff --git a/README.md b/README.md index 2033866f770f..e8bb8c43ecb0 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Pre-built pip packages support Ubuntu 18.04+, macOS 10.15+ and Windows 10+ ```bash # Install -pip install open3d +pip install open3d[gui,ml] # Verify installation python -c "import open3d as o3d; print(o3d.__version__)" diff --git a/cpp/pybind/make_install_pip_package.cmake b/cpp/pybind/make_install_pip_package.cmake index a5c9fcdd05ce..b65ff8cc0cdb 100644 --- a/cpp/pybind/make_install_pip_package.cmake +++ b/cpp/pybind/make_install_pip_package.cmake @@ -5,4 +5,4 @@ # it is guaranteed that there is only one wheel in ${PYTHON_PACKAGE_DST_DIR}/pip_package/*.whl file(GLOB WHEEL_FILE "${PYTHON_PACKAGE_DST_DIR}/pip_package/*.whl") execute_process(COMMAND ${Python3_EXECUTABLE} -m pip uninstall open3d --yes) -execute_process(COMMAND ${Python3_EXECUTABLE} -m pip install ${WHEEL_FILE} -U) +execute_process(COMMAND ${Python3_EXECUTABLE} -m pip install "${WHEEL_FILE}[gui,ml]" -U) diff --git a/cpp/pybind/make_python_package.cmake b/cpp/pybind/make_python_package.cmake index dee6e394d369..7ee237a069c7 100644 --- a/cpp/pybind/make_python_package.cmake +++ b/cpp/pybind/make_python_package.cmake @@ -113,6 +113,7 @@ if (BUILD_JUPYTER_EXTENSION) # These will be installed when `pip install open3d`. execute_process(COMMAND ${CMAKE_COMMAND} -E cat ${PYTHON_PACKAGE_SRC_DIR}/requirements.txt + ${PYTHON_PACKAGE_SRC_DIR}/requirements_gui.txt ${PYTHON_PACKAGE_SRC_DIR}/requirements_jupyter_install.txt OUTPUT_VARIABLE ALL_REQUIREMENTS ) diff --git a/docker/Dockerfile.openblas b/docker/Dockerfile.openblas index aba9297a5338..a7911a3e8a65 100644 --- a/docker/Dockerfile.openblas +++ b/docker/Dockerfile.openblas @@ -72,6 +72,7 @@ COPY ./python/requirements*.txt /root/Open3D/python/ RUN which python \ && python --version \ && python -m pip install -U -r /root/Open3D/python/requirements.txt \ + -r /root/Open3D/python/requirements_gui.txt \ -r /root/Open3D/python/requirements_build.txt \ -r /root/Open3D/python/requirements_test.txt diff --git a/docs/arm.rst b/docs/arm.rst index dddb894987d9..dd857ee9a8ad 100644 --- a/docs/arm.rst +++ b/docs/arm.rst @@ -9,7 +9,7 @@ we provide pre-compiled ARM64 wheels for Linux and macOS. Install the wheel by: .. code-block:: bash - pip install open3d + pip install open3d[gui,ml] python -c "import open3d; print(open3d.__version__)" # Test the legacy visualizer @@ -18,21 +18,21 @@ we provide pre-compiled ARM64 wheels for Linux and macOS. Install the wheel by: # Test the new GUI visualizer python -c "import open3d as o3d; c = o3d.geometry.TriangleMesh.create_box(); o3d.visualization.draw(c)" -+------------------------+----------------+---------------------+------------+----------------+ -| | Linux (OpenGL) | Linux (OpenGL ES) | macOS | Windows on ARM | -+========================+================+=====================+============+================+ -| ``pip install open3d`` | Yes | Yes | Yes | No | -+------------------------+----------------+---------------------+------------+----------------+ -| Compile from source | Yes | Yes | Yes | No | -+------------------------+----------------+---------------------+------------+----------------+ -| Visualizer and GUI | Yes | No | Yes | No | -+------------------------+----------------+---------------------+------------+----------------+ -| Non-GUI features | Yes | Yes | Yes | No | -+------------------------+----------------+---------------------+------------+----------------+ -| Special build flags | Not needed | ``-DBUILD_GUI=OFF`` | Not needed | N/A | -+------------------------+----------------+---------------------+------------+----------------+ -| Example device | Nvidia Jetson | Raspberry Pi 4 | M1 MacBook | Surface Pro X | -+------------------------+----------------+---------------------+------------+----------------+ ++--------------------------------+----------------+---------------------+------------+----------------+ +| | Linux (OpenGL) | Linux (OpenGL ES) | macOS | Windows on ARM | ++================================+================+=====================+============+================+ +| ``pip install open3d[gui,ml]`` | Yes | Yes | Yes | No | ++--------------------------------+----------------+---------------------+------------+----------------+ +| Compile from source | Yes | Yes | Yes | No | ++--------------------------------+----------------+---------------------+------------+----------------+ +| Visualizer and GUI | Yes | No | Yes | No | ++--------------------------------+----------------+---------------------+------------+----------------+ +| Non-GUI features | Yes | Yes | Yes | No | ++--------------------------------+----------------+---------------------+------------+----------------+ +| Special build flags | Not needed | ``-DBUILD_GUI=OFF`` | Not needed | N/A | ++--------------------------------+----------------+---------------------+------------+----------------+ +| Example device | Nvidia Jetson | Raspberry Pi 4 | M1 MacBook | Surface Pro X | ++--------------------------------+----------------+---------------------+------------+----------------+ Additional notes: @@ -107,7 +107,7 @@ TBB, Parallel STL, BLAS, LAPACK) may cause compatibility issues if they are not the same version as the one used by Open3D. If you only need the Python wheel, consider using the Docker build method or -install Open3D via ``pip install open3d`` directly. +install Open3D via ``pip install open3d[gui,ml]`` directly. Install dependencies ```````````````````` diff --git a/docs/docker.in.rst b/docs/docker.in.rst index f22c0a3b03d5..ff14062ef4aa 100644 --- a/docs/docker.in.rst +++ b/docs/docker.in.rst @@ -41,7 +41,7 @@ Python applications looks like this: # Install Open3D from the PyPI repositories RUN python3 -m pip install --no-cache-dir --upgrade pip && \ - python3 -m pip install --no-cache-dir --upgrade open3d + python3 -m pip install --no-cache-dir --upgrade open3d[gui,ml] If you have an NVIDIA GPU and want to use it for computation (``CUDA``) or visualization, follow these `directions. diff --git a/docs/getting_started.in.rst b/docs/getting_started.in.rst index e20c8da9735a..e8793f9f2df2 100644 --- a/docs/getting_started.in.rst +++ b/docs/getting_started.in.rst @@ -35,7 +35,12 @@ Pip (PyPI) .. code-block:: bash - pip install open3d + pip install open3d[gui,ml] + +.. note:: + Above will work for most purposes and aligns with behavior seen in `v0.16` and below. + Using anything under :mod:`open3d.visualization` will require ``[gui]`` where as + using anything under :mod:`open3d.ml` will require ``[ml]``. .. note:: Please upgrade your ``pip`` to a version >=20.3 to install Open3D in Linux, @@ -53,11 +58,11 @@ Pip (PyPI) .. code-block:: bash - pip3 install open3d + pip3 install open3d[gui,ml] # or - pip install --user open3d + pip install --user open3d[gui,ml] # or - python3 -m pip install --user open3d + python3 -m pip install --user open3d[gui,ml] Development version (pip) ------------------------- diff --git a/docs/tutorial/sensor/realsense.rst b/docs/tutorial/sensor/realsense.rst index b14f9e024747..b40d52b36e1e 100644 --- a/docs/tutorial/sensor/realsense.rst +++ b/docs/tutorial/sensor/realsense.rst @@ -18,7 +18,7 @@ Install Open3D from PyPI (a virtual environment is recommended): .. code-block:: sh - pip install open3d + pip install open3d[gui,ml] Compile from source (C++) ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -251,8 +251,8 @@ during capture and performing it while reading the bad file instead. This is a complete C++ example that shows visualizing live capture and recording to a bag file. The recording can be paused / resumed with [SPACE]. Use [ESC] to -stop capture and quit. You can download the -`rs_default_config.json `_ +stop capture and quit. You can download the +`rs_default_config.json `_ and use this example to capture your own dataset:: make RealSenseRecorder diff --git a/python/open3d/__init__.py b/python/open3d/__init__.py index abe40c00cb00..3f58fa1ce094 100644 --- a/python/open3d/__init__.py +++ b/python/open3d/__init__.py @@ -129,7 +129,14 @@ def _insert_pybind_names(skip_names=()): sys.modules.update(submodules) -import open3d.visualization +try: + import open3d.visualization +except ModuleNotFoundError: + warnings.warn( + "Open3D Python GUI Libraries not found. " + "Please make sure to install open3d[gui] if you wish to use " + "the open3d.visualization module.", RuntimeWarning) + _insert_pybind_names(skip_names=("ml",)) __version__ = "@PROJECT_VERSION@" @@ -162,10 +169,16 @@ def _insert_pybind_names(skip_names=()): if "OPEN3D_ML_ROOT" in os.environ: print("Using external Open3D-ML in {}".format(os.environ["OPEN3D_ML_ROOT"])) sys.path.append(os.environ["OPEN3D_ML_ROOT"]) -import open3d.ml -# Finally insert pybind names corresponding to ml -_insert_pybind_names() +try: + import open3d.ml + # Finally insert pybind names corresponding to ml + _insert_pybind_names() +except ModuleNotFoundError: + warnings.warn( + "Open3D Python ML Libraries not found. " + "Please make sure to install open3d[ml] if you wish to use " + "the open3d.ml module.", RuntimeWarning) def _jupyter_labextension_paths(): diff --git a/python/requirements.txt b/python/requirements.txt index 1affea6046d4..1ef04f80f3bc 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -1,5 +1,3 @@ numpy>=1.18.0 -dash>=2.6.0 -werkzeug>=2.2.3 nbformat==5.7.0 configargparse diff --git a/python/requirements_gui.txt b/python/requirements_gui.txt new file mode 100644 index 000000000000..817af47a1d5f --- /dev/null +++ b/python/requirements_gui.txt @@ -0,0 +1,2 @@ +dash>=2.6.0 +werkzeug>=2.2.3 \ No newline at end of file diff --git a/python/setup.py b/python/setup.py index bc27b0e10ff3..4f2c1969029d 100644 --- a/python/setup.py +++ b/python/setup.py @@ -24,6 +24,7 @@ # IN THE SOFTWARE. # ---------------------------------------------------------------------------- +from collections import defaultdict from setuptools import setup, find_packages from setuptools.command.install import install as _install import os @@ -118,10 +119,44 @@ def finalize_options(self): with open('requirements.txt', 'r') as f: install_requires = [line.strip() for line in f.readlines() if line] -# Read requirements for ML. +# Read extra requirements for GUI/ML/TF/Torch. +extra_requires = defaultdict(list) + +# GUI Deps +with open('requirements_gui.txt', 'r') as f: + extra_requires['gui'] += [line.strip() for line in f.readlines() if line] + +# Make sure GUI deps is part of "all" +extra_requires['all'] += extra_requires['gui'] + if '@BUNDLE_OPEN3D_ML@' == 'ON': + # ML Deps with open('@OPEN3D_ML_ROOT@/requirements.txt', 'r') as f: - install_requires += [line.strip() for line in f.readlines() if line] + extra_requires['ml'] += [line.strip() for line in f.readlines() if line] + extra_requires['all'] += extra_requires['ml'] + + # ML + TF Deps + with open('@OPEN3D_ML_ROOT@/requirements-tensorflow.txt', 'r') as f: + extra_requires['tf'] += extra_requires['ml'] + extra_requires['tf'] += [line.strip() for line in f.readlines() if line] + extra_requires['all'] += extra_requires['tf'] + + # ML + Torch Deps + if '@BUILD_CUDA_MODULE@' == 'ON': + torch_reqs = '@OPEN3D_ML_ROOT@/requirements-torch-cuda.txt' + else: + torch_reqs = '@OPEN3D_ML_ROOT@/requirements-torch.txt' + with open(torch_reqs, 'r') as f: + extra_requires['torch'] += extra_requires['ml'] + extra_requires['torch'] += [ + line.strip() + for line in f.readlines() + if line and not line.startswith("-") + ] + extra_requires['all'] += extra_requires['torch'] + +# Dedupe Extras w/ All +extra_requires['all'] = list(set(extra_requires['all'])) entry_points = { 'console_scripts': ['open3d = @PYPI_PACKAGE_NAME@.tools.cli:main',] @@ -139,6 +174,7 @@ def finalize_options(self): python_requires='>=3.6', include_package_data=True, install_requires=install_requires, + extras_require=extra_requires, packages=find_packages(), entry_points=entry_points, zip_safe=False, diff --git a/util/ci_utils.sh b/util/ci_utils.sh index a0b91a66bdac..cd3d3d0de731 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -83,6 +83,7 @@ install_python_dependencies() { # TODO: modify other locations to use requirements.txt python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements.txt" + python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements_gui.txt" if [[ "with-jupyter" =~ ^($options)$ ]]; then python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements_jupyter_build.txt" fi @@ -258,7 +259,7 @@ test_wheel() { echo -n "Using pip: " python -m pip --version echo "Installing Open3D wheel $wheel_path in virtual environment..." - python -m pip install "$wheel_path" + python -m pip install "${wheel_path}[gui,ml]" python -c "import open3d; print('Installed:', open3d)" python -c "import open3d; print('CUDA enabled: ', open3d.core.cuda.is_available())" echo @@ -387,6 +388,7 @@ install_docs_dependencies() { fi echo python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements.txt" + python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements_gui.txt" python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements_jupyter_build.txt" python -m pip install -r "${OPEN3D_SOURCE_ROOT}/docs/requirements.txt" }