From a6cab5b2f5f1915a83ed1fcd4ae3426a2104d3d3 Mon Sep 17 00:00:00 2001 From: Olivier Le Doeuff Date: Fri, 28 Jul 2023 17:08:41 +0200 Subject: [PATCH 01/19] Add support for python3.11 - Update the linux/mac/windows CI to handle python 3.11 --- .github/workflows/macos.yml | 4 ++-- .github/workflows/ubuntu-wheel.yml | 8 ++++++-- .github/workflows/windows.yml | 4 ++-- 3rdparty/pybind11/pybind11.cmake | 4 ++-- CHANGELOG.md | 1 + docs/arm.rst | 1 + docs/getting_started.in.rst | 5 +++++ python/README.rst | 1 + python/setup.py | 1 + 9 files changed, 21 insertions(+), 8 deletions(-) diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 516e72df47b..2a68079ab37 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -127,7 +127,7 @@ jobs: fail-fast: false # https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6 matrix: - python_version: ['3.7', '3.8', '3.9', '3.10'] + python_version: ['3.7', '3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: @@ -234,7 +234,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.7', '3.8', '3.9', '3.10'] + python_version: ['3.7', '3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index 0c42b06e67b..ef5e9246ba6 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -27,7 +27,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.7', '3.8', '3.9', '3.10'] + python_version: ['3.7', '3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: @@ -61,6 +61,8 @@ jobs: docker/docker_build.sh cuda_wheel_py39_dev elif [ "${{ env.PYTHON_VERSION }}" = "3.10" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then docker/docker_build.sh cuda_wheel_py310_dev + elif [ "${{ env.PYTHON_VERSION }}" = "3.11" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then + docker/docker_build.sh cuda_wheel_py311_dev elif [ "${{ env.PYTHON_VERSION }}" = "3.7" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then docker/docker_build.sh cuda_wheel_py37 elif [ "${{ env.PYTHON_VERSION }}" = "3.8" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then @@ -69,6 +71,8 @@ jobs: docker/docker_build.sh cuda_wheel_py39 elif [ "${{ env.PYTHON_VERSION }}" = "3.10" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then docker/docker_build.sh cuda_wheel_py310 + elif [ "${{ env.PYTHON_VERSION }}" = "3.11" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then + docker/docker_build.sh cuda_wheel_py311 fi PIP_PKG_NAME="$(basename ${GITHUB_WORKSPACE}/open3d-[0-9]*.whl)" PIP_CPU_PKG_NAME="$(basename ${GITHUB_WORKSPACE}/open3d_cpu*.whl)" @@ -114,7 +118,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.7', '3.8', '3.9', '3.10'] + python_version: ['3.7', '3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 3e13d7854c1..2aceb15569a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -246,7 +246,7 @@ jobs: fail-fast: false # https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6 matrix: - python_version: ['3.7', '3.8', '3.9', '3.10'] + python_version: ['3.7', '3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: @@ -351,7 +351,7 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.7', '3.8', '3.9', '3.10'] + python_version: ['3.7', '3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: diff --git a/3rdparty/pybind11/pybind11.cmake b/3rdparty/pybind11/pybind11.cmake index 5ed710aa58b..9a9c1b6df77 100644 --- a/3rdparty/pybind11/pybind11.cmake +++ b/3rdparty/pybind11/pybind11.cmake @@ -3,8 +3,8 @@ include(FetchContent) FetchContent_Declare( ext_pybind11 PREFIX pybind11 - URL https://github.com/pybind/pybind11/archive/refs/tags/v2.6.2.tar.gz - URL_HASH SHA256=8ff2fff22df038f5cd02cea8af56622bc67f5b64534f1b83b9f133b8366acff2 + URL https://github.com/pybind/pybind11/archive/refs/tags/v2.11.1.tar.gz + URL_HASH SHA256=d475978da0cdc2d43b73f30910786759d593a9d8ee05b1b6846d1eb16c6d2e0c DOWNLOAD_DIR "${OPEN3D_THIRD_PARTY_DOWNLOAD_DIR}/pybind11" ) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2164a63d139..2b46dc419df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ * Fix raycasting scene: Allow setting of number of threads that are used for building a raycasting scene * Fix Python bindings for CUDA device synchronization, voxel grid saving (PR #5425) * Support msgpack versions without cmake +* Python 3.11 support. bump pybind11 v2.6.2 -> v2.11.1 ## 0.13 diff --git a/docs/arm.rst b/docs/arm.rst index dddb894987d..92e26ef8b78 100644 --- a/docs/arm.rst +++ b/docs/arm.rst @@ -71,6 +71,7 @@ commands: ./docker_build.sh openblas-arm64-py38 # Python 3.8 ./docker_build.sh openblas-arm64-py39 # Python 3.9 ./docker_build.sh openblas-arm64-py310 # Python 3.10 + ./docker_build.sh openblas-arm64-py311 # Python 3.11 After running ``docker_build.sh``, you shall see a ``.whl`` file generated the current directly on the host. Then simply install the ``.whl`` file by: diff --git a/docs/getting_started.in.rst b/docs/getting_started.in.rst index e3ff7529fa6..fc79078ef17 100644 --- a/docs/getting_started.in.rst +++ b/docs/getting_started.in.rst @@ -20,6 +20,7 @@ Supported Python versions: * 3.8 * 3.9 * 3.10 + * 3.11 Supported operating systems: @@ -75,24 +76,28 @@ version (``HEAD`` of ``master`` branch): - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ + - `Python 3.11 `__ * - Linux (CPU) - `Python 3.7 `__ - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ + - `Python 3.11 `__ * - MacOS - `Python 3.7 `__ - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ + - `Python 3.11 `__ * - Windows - `Python 3.7 `__ - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ + - `Python 3.11 `__ Please use these links from the `latest version of this page `__ only. You can also diff --git a/python/README.rst b/python/README.rst index 64c5cb00c60..707ee14c138 100644 --- a/python/README.rst +++ b/python/README.rst @@ -48,6 +48,7 @@ With Python versions: * 3.8 * 3.9 * 3.10 +* 3.11 Resources ====================== diff --git a/python/setup.py b/python/setup.py index d2fea21ef7e..225be4ce98b 100644 --- a/python/setup.py +++ b/python/setup.py @@ -131,6 +131,7 @@ def finalize_options(self): "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Topic :: Education", "Topic :: Multimedia :: Graphics :: 3D Modeling", "Topic :: Multimedia :: Graphics :: 3D Rendering", From 95ed4d12f84223be56e261d746a7660acb384a2c Mon Sep 17 00:00:00 2001 From: Olivier Le Doeuff Date: Thu, 17 Aug 2023 08:19:38 +0200 Subject: [PATCH 02/19] =?UTF-8?q?=EF=B8=8Fupgrade=20pytorch=20to=202.0.1?= =?UTF-8?q?=20with=20cuda=2011.7=20in=20ci=5Futils?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/Dockerfile.wheel | 2 +- docker/README.md | 2 +- docker/docker_build.sh | 15 ++++++++------- util/ci_utils.sh | 12 ++++++------ 4 files changed, 16 insertions(+), 15 deletions(-) diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index 59d33503ab4..3da293478ba 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -1,5 +1,5 @@ # FROM must be called before other ARGS except for ARG BASE_IMAGE -ARG BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 +ARG BASE_IMAGE=nvidia/cuda:11.7.1-cudnn8-devel-ubuntu18.04 FROM ${BASE_IMAGE} # Customizable build arguments from cuda.yml diff --git a/docker/README.md b/docker/README.md index 89c5d34cabf..d14d9521c19 100644 --- a/docker/README.md +++ b/docker/README.md @@ -26,7 +26,7 @@ to install Nvidia Docker to run the CUDA container. To verify that the Nvidia Docker is working, run: ```bash -docker run --rm --gpus all nvidia/cuda:11.0-base nvidia-smi +docker run --rm --gpus all nvidia/cuda:11.7-base nvidia-smi ``` ### ARM64 Docker diff --git a/docker/docker_build.sh b/docker/docker_build.sh index c30d6571788..e62ae7a5c29 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -78,6 +78,7 @@ HOST_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null 2>&1 && pw CCACHE_VERSION=4.3 CMAKE_VERSION=cmake-3.20.6-linux-x86_64 CMAKE_VERSION_AARCH64=cmake-3.20.6-linux-aarch64 +CUDA_VERSION=11.7.1-cudnn8 print_usage_and_exit_docker_build() { echo "$__usage_docker_build" @@ -168,7 +169,7 @@ openblas_build() { } cuda_wheel_build() { - BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 + BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 CCACHE_TAR_NAME=open3d-ubuntu-1804-cuda-ci-ccache options="$(echo "$@" | tr ' ' '|')" @@ -256,7 +257,7 @@ ci_build() { 2-bionic_export_env() { export DOCKER_TAG=open3d-ci:2-bionic - export BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 + export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-2-bionic export PYTHON_VERSION=3.7 @@ -271,7 +272,7 @@ ci_build() { 3-ml-shared-bionic_export_env() { export DOCKER_TAG=open3d-ci:3-ml-shared-bionic - export BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 + export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-bionic export PYTHON_VERSION=3.7 @@ -286,7 +287,7 @@ ci_build() { 3-ml-shared-bionic-release_export_env() { export DOCKER_TAG=open3d-ci:3-ml-shared-bionic - export BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 + export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=OFF export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-bionic export PYTHON_VERSION=3.7 @@ -301,7 +302,7 @@ ci_build() { 4-shared-bionic_export_env() { export DOCKER_TAG=open3d-ci:4-shared-bionic - export BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 + export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-4-shared-bionic export PYTHON_VERSION=3.7 @@ -316,7 +317,7 @@ ci_build() { 4-shared-bionic-release_export_env() { export DOCKER_TAG=open3d-ci:4-shared-bionic - export BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04 + export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=OFF export CCACHE_TAR_NAME=open3d-ci-4-shared-bionic export PYTHON_VERSION=3.7 @@ -331,7 +332,7 @@ ci_build() { 5-ml-focal_export_env() { export DOCKER_TAG=open3d-ci:5-ml-focal - export BASE_IMAGE=nvidia/cuda:11.6.2-cudnn8-devel-ubuntu20.04 + export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-5-ml-focal export PYTHON_VERSION=3.7 diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 80ca4469a14..9f6d9e7e015 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -27,9 +27,9 @@ LOW_MEM_USAGE=${LOW_MEM_USAGE:-OFF} if [[ $BUILD_TENSORFLOW_OPS == ON || $BUILD_PYTORCH_OPS == ON || $UBUNTU_VERSION != bionic ]]; then # CUDA version in sync with PyTorch and Tensorflow - CUDA_VERSION=("11-6" "11.6") + CUDA_VERSION=("11-7" "11.7") CUDNN_MAJOR_VERSION=8 - CUDNN_VERSION="8.4.1.50_cuda11.6" + CUDNN_VERSION="8.0.5.39_cuda11.7" GCC_MAX_VER=9 else # Without MLOps, ensure Open3D works with the lowest supported CUDA version @@ -42,11 +42,11 @@ fi # ML TENSORFLOW_VER="2.8.4" TENSORBOARD_VER="2.8.0" -TORCH_CPU_GLNX_VER="1.13.1+cpu" -TORCH_CUDA_GLNX_VER="1.13.1+cu116" +TORCH_VER="2.0.1" +TORCH_CPU_GLNX_VER="${TORCH_VER}+cpu" +TORCH_CUDA_GLNX_VER="${TORCH_VER}+cu117" PYTHON_VER=$(python -c 'import sys; ver=f"{sys.version_info.major}{sys.version_info.minor}"; print(f"cp{ver}-cp{ver}{sys.abiflags}")' 2>/dev/null || true) -# TORCH_CUDA_GLNX_URL="https://github.com/isl-org/open3d_downloads/releases/download/torch1.8.2/torch-1.8.2-${PYTHON_VER}-linux_x86_64.whl" -TORCH_MACOS_VER="1.13.1" +TORCH_MACOS_VER="${TORCH_VER}" TORCH_REPO_URL="https://download.pytorch.org/whl/torch/" # Python PIP_VER="21.1.1" From 32a1f1e808abb30b92c850da5a7b10a2b8e52434 Mon Sep 17 00:00:00 2001 From: Olivier Le Doeuff Date: Thu, 17 Aug 2023 08:36:13 +0200 Subject: [PATCH 03/19] ci(linux): introduce python 3.11 in docker tests script --- docker/docker_build.sh | 33 +++++++++++++++++++++++++++++++++ docker/docker_test.sh | 24 ++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/docker/docker_build.sh b/docker/docker_build.sh index e62ae7a5c29..8dd58614f58 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -27,20 +27,24 @@ OPTION: openblas-amd64-py38-dev : OpenBLAS AMD64 3.8 wheel, developer mode openblas-amd64-py39-dev : OpenBLAS AMD64 3.9 wheel, developer mode openblas-amd64-py310-dev : OpenBLAS AMD64 3.10 wheel, developer mode + openblas-amd64-py311-dev : OpenBLAS AMD64 3.11 wheel, developer mode openblas-amd64-py37 : OpenBLAS AMD64 3.7 wheel, release mode openblas-amd64-py38 : OpenBLAS AMD64 3.8 wheel, release mode openblas-amd64-py39 : OpenBLAS AMD64 3.9 wheel, release mode openblas-amd64-py310 : OpenBLAS AMD64 3.10 wheel, release mode + openblas-amd64-py311 : OpenBLAS AMD64 3.11 wheel, release mode # OpenBLAS ARM64 (Dockerfile.openblas) openblas-arm64-py37-dev : OpenBLAS ARM64 3.7 wheel, developer mode openblas-arm64-py38-dev : OpenBLAS ARM64 3.8 wheel, developer mode openblas-arm64-py39-dev : OpenBLAS ARM64 3.9 wheel, developer mode openblas-arm64-py310-dev : OpenBLAS ARM64 3.10 wheel, developer mode + openblas-arm64-py311-dev : OpenBLAS ARM64 3.11 wheel, developer mode openblas-arm64-py37 : OpenBLAS ARM64 3.7 wheel, release mode openblas-arm64-py38 : OpenBLAS ARM64 3.8 wheel, release mode openblas-arm64-py39 : OpenBLAS ARM64 3.9 wheel, release mode openblas-arm64-py310 : OpenBLAS ARM64 3.10 wheel, release mode + openblas-arm64-py311 : OpenBLAS ARM64 3.11 wheel, release mode # Ubuntu CPU CI (Dockerfile.ci) cpu-static : Ubuntu CPU static @@ -66,10 +70,12 @@ OPTION: cuda_wheel_py38_dev : CUDA Python 3.8 wheel, developer mode cuda_wheel_py39_dev : CUDA Python 3.9 wheel, developer mode cuda_wheel_py310_dev : CUDA Python 3.10 wheel, developer mode + cuda_wheel_py311_dev : CUDA Python 3.11 wheel, developer mode cuda_wheel_py37 : CUDA Python 3.7 wheel, release mode cuda_wheel_py38 : CUDA Python 3.8 wheel, release mode cuda_wheel_py39 : CUDA Python 3.9 wheel, release mode cuda_wheel_py310 : CUDA Python 3.10 wheel, release mode + cuda_wheel_py311 : CUDA Python 3.11 wheel, release mode " HOST_OPEN3D_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null 2>&1 && pwd)" @@ -127,6 +133,9 @@ openblas_export_env() { elif [[ "py310" =~ ^($options)$ ]]; then export PYTHON_VERSION=3.10 export DOCKER_TAG=${DOCKER_TAG}-py310 + elif [[ "py311" =~ ^($options)$ ]]; then + export PYTHON_VERSION=3.11 + export DOCKER_TAG=${DOCKER_TAG}-py311 else echo "Invalid python version." print_usage_and_exit_docker_build @@ -182,6 +191,8 @@ cuda_wheel_build() { PYTHON_VERSION=3.9 elif [[ "py310" =~ ^($options)$ ]]; then PYTHON_VERSION=3.10 + elif [[ "py311" =~ ^($options)$ ]]; then + PYTHON_VERSION=3.11 else echo "Invalid python version." print_usage_and_exit_docker_build @@ -477,6 +488,10 @@ function main() { openblas_export_env amd64 py310 dev openblas_build ;; + openblas-amd64-py311-dev) + openblas_export_env amd64 py311 dev + openblas_build + ;; openblas-amd64-py37) openblas_export_env amd64 py37 openblas_build @@ -493,6 +508,10 @@ function main() { openblas_export_env amd64 py310 openblas_build ;; + openblas-amd64-py311) + openblas_export_env amd64 py311 + openblas_build + ;; # OpenBLAS ARM64 openblas-arm64-py37-dev) @@ -511,6 +530,10 @@ function main() { openblas_export_env arm64 py310 dev openblas_build ;; + openblas-arm64-py311-dev) + openblas_export_env arm64 py311 dev + openblas_build + ;; openblas-arm64-py37) openblas_export_env arm64 py37 openblas_build @@ -527,6 +550,10 @@ function main() { openblas_export_env arm64 py310 openblas_build ;; + openblas-arm64-py311) + openblas_export_env arm64 py311 + openblas_build + ;; # CPU CI cpu-static) @@ -573,6 +600,9 @@ function main() { cuda_wheel_py310_dev) cuda_wheel_build py310 dev ;; + cuda_wheel_py311_dev) + cuda_wheel_build py311 dev + ;; cuda_wheel_py37) cuda_wheel_build py37 ;; @@ -585,6 +615,9 @@ function main() { cuda_wheel_py310) cuda_wheel_build py310 ;; + cuda_wheel_py311) + cuda_wheel_build py311 + ;; # ML CIs 2-bionic) diff --git a/docker/docker_test.sh b/docker/docker_test.sh index 9993024a90f..6f26aaddcfd 100755 --- a/docker/docker_test.sh +++ b/docker/docker_test.sh @@ -20,20 +20,24 @@ OPTION: openblas-amd64-py38-dev : OpenBLAS AMD64 3.8 wheel, developer mode openblas-amd64-py39-dev : OpenBLAS AMD64 3.9 wheel, developer mode openblas-amd64-py310-dev : OpenBLAS AMD64 3.10 wheel, developer mode + openblas-amd64-py311-dev : OpenBLAS AMD64 3.11 wheel, developer mode openblas-amd64-py37 : OpenBLAS AMD64 3.7 wheel, release mode openblas-amd64-py38 : OpenBLAS AMD64 3.8 wheel, release mode openblas-amd64-py39 : OpenBLAS AMD64 3.9 wheel, release mode openblas-amd64-py310 : OpenBLAS AMD64 3.10 wheel, release mode + openblas-amd64-py311 : OpenBLAS AMD64 3.11 wheel, release mode # OpenBLAS ARM64 (Dockerfile.openblas) openblas-arm64-py37-dev : OpenBLAS ARM64 3.7 wheel, developer mode openblas-arm64-py38-dev : OpenBLAS ARM64 3.8 wheel, developer mode openblas-arm64-py39-dev : OpenBLAS ARM64 3.9 wheel, developer mode openblas-arm64-py310-dev : OpenBLAS ARM64 3.10 wheel, developer mode + openblas-arm64-py311-dev : OpenBLAS ARM64 3.11 wheel, developer mode openblas-arm64-py37 : OpenBLAS ARM64 3.7 wheel, release mode openblas-arm64-py38 : OpenBLAS ARM64 3.8 wheel, release mode openblas-arm64-py39 : OpenBLAS ARM64 3.9 wheel, release mode openblas-arm64-py310 : OpenBLAS ARM64 3.10 wheel, release mode + openblas-arm64-py311 : OpenBLAS ARM64 3.11 wheel, release mode # Ubuntu CPU CI (Dockerfile.ci) cpu-static : Ubuntu CPU static @@ -228,6 +232,11 @@ openblas-amd64-py310-dev) openblas_print_env cpp_python_linking_uninstall_test ;; +openblas-amd64-py311-dev) + openblas_export_env amd64 py311 dev + openblas_print_env + cpp_python_linking_uninstall_test + ;; openblas-amd64-py37) openblas_export_env amd64 py37 openblas_print_env @@ -248,6 +257,11 @@ openblas-amd64-py310) openblas_print_env cpp_python_linking_uninstall_test ;; +openblas-amd64-py311) + openblas_export_env amd64 py311 + openblas_print_env + cpp_python_linking_uninstall_test + ;; # OpenBLAS ARM64 openblas-arm64-py37-dev) @@ -270,6 +284,11 @@ openblas-arm64-py310-dev) openblas_print_env cpp_python_linking_uninstall_test ;; +openblas-arm64-py311-dev) + openblas_export_env arm64 py311 dev + openblas_print_env + cpp_python_linking_uninstall_test + ;; openblas-arm64-py37) openblas_export_env arm64 py37 openblas_print_env @@ -290,6 +309,11 @@ openblas-arm64-py310) openblas_print_env cpp_python_linking_uninstall_test ;; +openblas-arm64-py311) + openblas_export_env arm64 py311 + openblas_print_env + cpp_python_linking_uninstall_test + ;; # CPU CI cpu-static) From 8f7b13825fe4fa02131840dc2e8a32d4d36bdd0e Mon Sep 17 00:00:00 2001 From: Olivier Le Doeuff Date: Thu, 17 Aug 2023 08:42:13 +0200 Subject: [PATCH 04/19] ci(linux): delete unused env variable BUILD_TENSORFLOW_OPS & BUILD_PYTORCH_OPS --- .github/workflows/ubuntu.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index d41d3a2a1ce..b49c4502020 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -35,8 +35,6 @@ jobs: env: BUILD_SHARED_LIBS: ${{ matrix.BUILD_SHARED_LIBS }} BUILD_CUDA_MODULE: OFF - BUILD_TENSORFLOW_OPS: ${{ matrix.MLOPS }} - BUILD_PYTORCH_OPS: ${{ matrix.MLOPS }} DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} OPEN3D_CPU_RENDERING: true steps: From 03e33924616ab98ec181135577f1a5c6d5fdd04a Mon Sep 17 00:00:00 2001 From: Olivier Le Doeuff Date: Thu, 17 Aug 2023 08:48:55 +0200 Subject: [PATCH 05/19] ci: upgrade tensorflow to 2.12.0 & tensorboard to 2.14.0 This also require protobuf to be updated to 3.24.0 --- util/ci_utils.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 9f6d9e7e015..004e9726bfb 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -40,8 +40,8 @@ else GCC_MAX_VER=7 fi # ML -TENSORFLOW_VER="2.8.4" -TENSORBOARD_VER="2.8.0" +TENSORFLOW_VER="2.12.0" +TENSORBOARD_VER="2.14.0" TORCH_VER="2.0.1" TORCH_CPU_GLNX_VER="${TORCH_VER}+cpu" TORCH_CUDA_GLNX_VER="${TORCH_VER}+cu117" @@ -56,7 +56,7 @@ PYTEST_VER="7.1.2" PYTEST_RANDOMLY_VER="3.8.0" SCIPY_VER="1.7.3" YAPF_VER="0.30.0" -PROTOBUF_VER="3.19.0" +PROTOBUF_VER="4.24.0" OPEN3D_INSTALL_DIR=~/open3d_install OPEN3D_SOURCE_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. >/dev/null 2>&1 && pwd)" From d892962d48d18ea1755294fbc241ab5f74e127c1 Mon Sep 17 00:00:00 2001 From: Olivier Le Doeuff Date: Thu, 17 Aug 2023 08:49:34 +0200 Subject: [PATCH 06/19] ci(linux): disable tensorflow build, due to cxx abi issue --- docker/Dockerfile.wheel | 3 ++- docker/docker_build.sh | 15 ++++++++++----- util/ci_utils.sh | 7 ++++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index 3da293478ba..cf10ca879f9 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -125,7 +125,8 @@ WORKDIR /root/Open3D # Build python wheel RUN export NPROC=$(nproc) \ && export BUILD_SHARED_LIBS=OFF \ - && export BUILD_TENSORFLOW_OPS=ON \ + # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch + && export BUILD_TENSORFLOW_OPS=OFF \ && export BUILD_PYTORCH_OPS=ON \ && source /root/Open3D/util/ci_utils.sh \ && build_pip_package build_azure_kinect build_jupyter diff --git a/docker/docker_build.sh b/docker/docker_build.sh index 8dd58614f58..8bf7c6a1a45 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -289,7 +289,8 @@ ci_build() { export PYTHON_VERSION=3.7 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON - export BUILD_TENSORFLOW_OPS=ON + # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch + export BUILD_TENSORFLOW_OPS=OFF export BUILD_PYTORCH_OPS=ON export PACKAGE=ON export BUILD_SYCL_MODULE=OFF @@ -304,7 +305,8 @@ ci_build() { export PYTHON_VERSION=3.7 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON - export BUILD_TENSORFLOW_OPS=ON + # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch + export BUILD_TENSORFLOW_OPS=OFF export BUILD_PYTORCH_OPS=ON export PACKAGE=ON export BUILD_SYCL_MODULE=OFF @@ -349,7 +351,8 @@ ci_build() { export PYTHON_VERSION=3.7 export BUILD_SHARED_LIBS=OFF export BUILD_CUDA_MODULE=ON - export BUILD_TENSORFLOW_OPS=ON + # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch + export BUILD_TENSORFLOW_OPS=OFF export BUILD_PYTORCH_OPS=ON export PACKAGE=OFF export BUILD_SYCL_MODULE=OFF @@ -394,7 +397,8 @@ cpu-shared-ml_export_env() { export PYTHON_VERSION=3.7 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF - export BUILD_TENSORFLOW_OPS=ON + # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch + export BUILD_TENSORFLOW_OPS=OFF export BUILD_PYTORCH_OPS=ON export PACKAGE=ON export BUILD_SYCL_MODULE=OFF @@ -424,7 +428,8 @@ cpu-shared-ml-release_export_env() { export PYTHON_VERSION=3.7 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF - export BUILD_TENSORFLOW_OPS=ON + # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch + export BUILD_TENSORFLOW_OPS=OFF export BUILD_PYTORCH_OPS=ON export PACKAGE=ON export BUILD_SYCL_MODULE=OFF diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 004e9726bfb..84e2f046d87 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -204,8 +204,8 @@ build_pip_package() { "-DBUILD_AZURE_KINECT=$BUILD_AZURE_KINECT" "-DBUILD_LIBREALSENSE=ON" "-DGLIBCXX_USE_CXX11_ABI=OFF" - "-DBUILD_TENSORFLOW_OPS=ON" - "-DBUILD_PYTORCH_OPS=ON" + "-DBUILD_TENSORFLOW_OPS=$BUILD_TENSORFLOW_OPS" + "-DBUILD_PYTORCH_OPS=$BUILD_PYTORCH_OPS" "-DBUILD_FILAMENT_FROM_SOURCE=$BUILD_FILAMENT_FROM_SOURCE" "-DBUILD_JUPYTER_EXTENSION=$BUILD_JUPYTER_EXTENSION" "-DCMAKE_INSTALL_PREFIX=$OPEN3D_INSTALL_DIR" @@ -422,7 +422,8 @@ build_docs() { "-DBUILD_AZURE_KINECT=ON" "-DBUILD_LIBREALSENSE=ON" "-DGLIBCXX_USE_CXX11_ABI=OFF" - "-DBUILD_TENSORFLOW_OPS=ON" + # TODO: PyTorch still use old CXX ABI, re-enable Tensorflow when PyTorch is updated to use new ABI + "-DBUILD_TENSORFLOW_OPS=OFF" "-DBUILD_PYTORCH_OPS=ON" "-DBUILD_EXAMPLES=OFF" ) From a5c2c66eb6f2a3f7a6db0e37915add37851a8e22 Mon Sep 17 00:00:00 2001 From: Olivier Le Doeuff Date: Thu, 17 Aug 2023 08:52:22 +0200 Subject: [PATCH 07/19] ci(linux): always use cxx11_abi This is possible because - tensorflow support is temporarily dropped - pytorch got updated to 2.0.1 --- docker/Dockerfile.ci | 11 +++++------ util/ci_utils.sh | 7 ++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 5474b1aee58..d3d506dca84 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -166,17 +166,16 @@ RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ && yarn --version # Build all -RUN if [ "${BUILD_PYTORCH_OPS}" = "ON" ] || [ "${BUILD_TENSORFLOW_OPS}" = "ON" ]; then \ - export GLIBCXX_USE_CXX11_ABI=OFF; \ - else \ - export GLIBCXX_USE_CXX11_ABI=ON; \ - fi \ - && if [ "${BUILD_SYCL_MODULE}" = "ON" ]; then \ +RUN \ + if [ "${BUILD_SYCL_MODULE}" = "ON" ]; then \ export CMAKE_CXX_COMPILER=icpx; \ export CMAKE_C_COMPILER=icx; \ + export GLIBCXX_USE_CXX11_ABI=ON; \ else \ export CMAKE_CXX_COMPILER=g++; \ export CMAKE_C_COMPILER=gcc; \ + # TODO: PyTorch still use old CXX ABI, remove this line when PyTorch is updated + export GLIBCXX_USE_CXX11_ABI=OFF; \ fi \ && mkdir build \ && cd build \ diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 84e2f046d87..9b606e4dc55 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -124,10 +124,6 @@ build_all() { mkdir -p build cd build - GLIBCXX_USE_CXX11_ABI=ON - if [ "$BUILD_PYTORCH_OPS" == ON ] || [ "$BUILD_TENSORFLOW_OPS" == ON ]; then - GLIBCXX_USE_CXX11_ABI=OFF - fi cmakeOptions=( -DDEVELOPER_BUILD=$DEVELOPER_BUILD @@ -137,7 +133,8 @@ build_all() { -DBUILD_CUDA_MODULE="$BUILD_CUDA_MODULE" -DBUILD_COMMON_CUDA_ARCHS=ON -DBUILD_COMMON_ISPC_ISAS=ON - -DGLIBCXX_USE_CXX11_ABI="$GLIBCXX_USE_CXX11_ABI" + # TODO: PyTorch still use old CXX ABI, remove this line when PyTorch is updated + -DGLIBCXX_USE_CXX11_ABI=OFF -DBUILD_TENSORFLOW_OPS="$BUILD_TENSORFLOW_OPS" -DBUILD_PYTORCH_OPS="$BUILD_PYTORCH_OPS" -DCMAKE_INSTALL_PREFIX="$OPEN3D_INSTALL_DIR" From f8db0bdfa13087c2d7d346c757f9ea2a2fa24bb1 Mon Sep 17 00:00:00 2001 From: Olivier Le Doeuff Date: Thu, 17 Aug 2023 09:00:57 +0200 Subject: [PATCH 08/19] drop support for python3.7 --- .github/workflows/documentation.yml | 2 +- .github/workflows/macos.yml | 10 ++--- .github/workflows/style.yml | 2 +- .github/workflows/ubuntu-wheel.yml | 14 ++----- .github/workflows/windows.yml | 10 ++--- docker/docker_build.sh | 63 +++++++---------------------- docker/docker_test.sh | 24 ----------- docs/arm.rst | 1 - docs/getting_started.in.rst | 5 --- python/README.rst | 1 - python/setup.py | 3 +- 11 files changed, 27 insertions(+), 108 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index a0bf14fffe8..d48d5815971 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -58,7 +58,7 @@ jobs: - name: Set up Python version uses: actions/setup-python@v4 with: - python-version: '3.7' + python-version: '3.8' - name: Install dependencies env: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 2a68079ab37..905bd65e31c 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -56,7 +56,7 @@ jobs: - name: Set up Python version uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.8 - name: Install dependencies run: | brew install ccache pkg-config @@ -127,12 +127,10 @@ jobs: fail-fast: false # https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6 matrix: - python_version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: - - is_master: false - python_version: '3.7' - is_master: false python_version: '3.8' - is_master: false @@ -234,12 +232,10 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: - - is_master: false - python_version: '3.7' - is_master: false python_version: '3.8' - is_master: false diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 740ede78b16..f6dbcac232a 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Python version uses: actions/setup-python@v4 with: - python-version: '3.7' + python-version: '3.8' - name: Install dependencies run: | pip install -U clang-format~=10.0.0 yapf==0.30.0 nbformat diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index ef5e9246ba6..e0958994614 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -27,12 +27,10 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: - - is_master: false - python_version: '3.7' - is_master: false python_version: '3.8' - is_master: false @@ -53,9 +51,7 @@ jobs: # `docker/docker_build.sh xxx` command to execute locally. - name: Docker build run: | - if [ "${{ env.PYTHON_VERSION }}" = "3.7" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then - docker/docker_build.sh cuda_wheel_py37_dev - elif [ "${{ env.PYTHON_VERSION }}" = "3.8" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then + if [ "${{ env.PYTHON_VERSION }}" = "3.8" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then docker/docker_build.sh cuda_wheel_py38_dev elif [ "${{ env.PYTHON_VERSION }}" = "3.9" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then docker/docker_build.sh cuda_wheel_py39_dev @@ -63,8 +59,6 @@ jobs: docker/docker_build.sh cuda_wheel_py310_dev elif [ "${{ env.PYTHON_VERSION }}" = "3.11" ] && [ "${{ env.DEVELOPER_BUILD }}" = "ON" ]; then docker/docker_build.sh cuda_wheel_py311_dev - elif [ "${{ env.PYTHON_VERSION }}" = "3.7" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then - docker/docker_build.sh cuda_wheel_py37 elif [ "${{ env.PYTHON_VERSION }}" = "3.8" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then docker/docker_build.sh cuda_wheel_py38 elif [ "${{ env.PYTHON_VERSION }}" = "3.9" ] && [ "${{ env.DEVELOPER_BUILD }}" = "OFF" ]; then @@ -118,12 +112,10 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: - - is_master: false - python_version: '3.7' - is_master: false python_version: '3.8' - is_master: false diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 2aceb15569a..bd1074e543a 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -101,7 +101,7 @@ jobs: - name: Set up Python version uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: 3.8 - name: Config # Move build directory to C: https://github.com/actions/virtual-environments/issues/1341 @@ -246,12 +246,10 @@ jobs: fail-fast: false # https://github.community/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6 matrix: - python_version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: - - is_master: false - python_version: '3.7' - is_master: false python_version: '3.8' - is_master: false @@ -351,12 +349,10 @@ jobs: strategy: fail-fast: false matrix: - python_version: ['3.7', '3.8', '3.9', '3.10', '3.11'] + python_version: ['3.8', '3.9', '3.10', '3.11'] is_master: - ${{ github.ref == 'refs/heads/master' }} exclude: - - is_master: false - python_version: '3.7' - is_master: false python_version: '3.8' - is_master: false diff --git a/docker/docker_build.sh b/docker/docker_build.sh index 8bf7c6a1a45..6af0d3ce5f3 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -23,24 +23,20 @@ __usage_docker_build="USAGE: OPTION: # OpenBLAS AMD64 (Dockerfile.openblas) - openblas-amd64-py37-dev : OpenBLAS AMD64 3.7 wheel, developer mode openblas-amd64-py38-dev : OpenBLAS AMD64 3.8 wheel, developer mode openblas-amd64-py39-dev : OpenBLAS AMD64 3.9 wheel, developer mode openblas-amd64-py310-dev : OpenBLAS AMD64 3.10 wheel, developer mode openblas-amd64-py311-dev : OpenBLAS AMD64 3.11 wheel, developer mode - openblas-amd64-py37 : OpenBLAS AMD64 3.7 wheel, release mode openblas-amd64-py38 : OpenBLAS AMD64 3.8 wheel, release mode openblas-amd64-py39 : OpenBLAS AMD64 3.9 wheel, release mode openblas-amd64-py310 : OpenBLAS AMD64 3.10 wheel, release mode openblas-amd64-py311 : OpenBLAS AMD64 3.11 wheel, release mode # OpenBLAS ARM64 (Dockerfile.openblas) - openblas-arm64-py37-dev : OpenBLAS ARM64 3.7 wheel, developer mode openblas-arm64-py38-dev : OpenBLAS ARM64 3.8 wheel, developer mode openblas-arm64-py39-dev : OpenBLAS ARM64 3.9 wheel, developer mode openblas-arm64-py310-dev : OpenBLAS ARM64 3.10 wheel, developer mode openblas-arm64-py311-dev : OpenBLAS ARM64 3.11 wheel, developer mode - openblas-arm64-py37 : OpenBLAS ARM64 3.7 wheel, release mode openblas-arm64-py38 : OpenBLAS ARM64 3.8 wheel, release mode openblas-arm64-py39 : OpenBLAS ARM64 3.9 wheel, release mode openblas-arm64-py310 : OpenBLAS ARM64 3.10 wheel, release mode @@ -66,12 +62,10 @@ OPTION: 5-ml-focal : CUDA CI, 5-ml-focal, developer mode # CUDA wheels (Dockerfile.wheel) - cuda_wheel_py37_dev : CUDA Python 3.7 wheel, developer mode cuda_wheel_py38_dev : CUDA Python 3.8 wheel, developer mode cuda_wheel_py39_dev : CUDA Python 3.9 wheel, developer mode cuda_wheel_py310_dev : CUDA Python 3.10 wheel, developer mode cuda_wheel_py311_dev : CUDA Python 3.11 wheel, developer mode - cuda_wheel_py37 : CUDA Python 3.7 wheel, release mode cuda_wheel_py38 : CUDA Python 3.8 wheel, release mode cuda_wheel_py39 : CUDA Python 3.9 wheel, release mode cuda_wheel_py310 : CUDA Python 3.10 wheel, release mode @@ -121,10 +115,7 @@ openblas_export_env() { print_usage_and_exit_docker_build fi - if [[ "py37" =~ ^($options)$ ]]; then - export PYTHON_VERSION=3.7 - export DOCKER_TAG=${DOCKER_TAG}-py37 - elif [[ "py38" =~ ^($options)$ ]]; then + if [[ "py38" =~ ^($options)$ ]]; then export PYTHON_VERSION=3.8 export DOCKER_TAG=${DOCKER_TAG}-py38 elif [[ "py39" =~ ^($options)$ ]]; then @@ -183,9 +174,7 @@ cuda_wheel_build() { options="$(echo "$@" | tr ' ' '|')" echo "[cuda_wheel_build()] options: ${options}" - if [[ "py37" =~ ^($options)$ ]]; then - PYTHON_VERSION=3.7 - elif [[ "py38" =~ ^($options)$ ]]; then + if [[ "py38" =~ ^($options)$ ]]; then PYTHON_VERSION=3.8 elif [[ "py39" =~ ^($options)$ ]]; then PYTHON_VERSION=3.9 @@ -271,7 +260,7 @@ ci_build() { export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-2-bionic - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=OFF export BUILD_CUDA_MODULE=ON export BUILD_TENSORFLOW_OPS=OFF @@ -286,7 +275,7 @@ ci_build() { export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-bionic - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch @@ -302,7 +291,7 @@ ci_build() { export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=OFF export CCACHE_TAR_NAME=open3d-ci-3-ml-shared-bionic - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch @@ -318,7 +307,7 @@ ci_build() { export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-4-shared-bionic - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON export BUILD_TENSORFLOW_OPS=OFF @@ -333,7 +322,7 @@ ci_build() { export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu18.04 export DEVELOPER_BUILD=OFF export CCACHE_TAR_NAME=open3d-ci-4-shared-bionic - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=ON export BUILD_TENSORFLOW_OPS=OFF @@ -348,7 +337,7 @@ ci_build() { export BASE_IMAGE=nvidia/cuda:${CUDA_VERSION}-devel-ubuntu20.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-5-ml-focal - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=OFF export BUILD_CUDA_MODULE=ON # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch @@ -364,7 +353,7 @@ cpu-static_export_env() { export BASE_IMAGE=ubuntu:18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-cpu - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=OFF export BUILD_CUDA_MODULE=OFF export BUILD_TENSORFLOW_OPS=OFF @@ -379,7 +368,7 @@ cpu-shared_export_env() { export BASE_IMAGE=ubuntu:18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-cpu - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF export BUILD_TENSORFLOW_OPS=OFF @@ -394,7 +383,7 @@ cpu-shared-ml_export_env() { export BASE_IMAGE=ubuntu:18.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-cpu - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch @@ -410,7 +399,7 @@ cpu-shared-release_export_env() { export BASE_IMAGE=ubuntu:18.04 export DEVELOPER_BUILD=OFF export CCACHE_TAR_NAME=open3d-ci-cpu - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF export BUILD_TENSORFLOW_OPS=OFF @@ -425,7 +414,7 @@ cpu-shared-ml-release_export_env() { export BASE_IMAGE=ubuntu:18.04 export DEVELOPER_BUILD=OFF export CCACHE_TAR_NAME=open3d-ci-cpu - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch @@ -443,7 +432,7 @@ sycl-shared_export_env() { export BASE_IMAGE=intel/oneapi-basekit:2022.2-devel-ubuntu20.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-sycl - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=ON export BUILD_CUDA_MODULE=OFF export BUILD_TENSORFLOW_OPS=OFF @@ -460,7 +449,7 @@ sycl-static_export_env() { export BASE_IMAGE=intel/oneapi-basekit:2022.2-devel-ubuntu20.04 export DEVELOPER_BUILD=ON export CCACHE_TAR_NAME=open3d-ci-sycl - export PYTHON_VERSION=3.7 + export PYTHON_VERSION=3.8 export BUILD_SHARED_LIBS=OFF export BUILD_CUDA_MODULE=OFF export BUILD_TENSORFLOW_OPS=OFF @@ -477,10 +466,6 @@ function main() { echo "[$(basename $0)] building $1" case "$1" in # OpenBLAS AMD64 - openblas-amd64-py37-dev) - openblas_export_env amd64 py37 dev - openblas_build - ;; openblas-amd64-py38-dev) openblas_export_env amd64 py38 dev openblas_build @@ -497,10 +482,6 @@ function main() { openblas_export_env amd64 py311 dev openblas_build ;; - openblas-amd64-py37) - openblas_export_env amd64 py37 - openblas_build - ;; openblas-amd64-py38) openblas_export_env amd64 py38 openblas_build @@ -519,10 +500,6 @@ function main() { ;; # OpenBLAS ARM64 - openblas-arm64-py37-dev) - openblas_export_env arm64 py37 dev - openblas_build - ;; openblas-arm64-py38-dev) openblas_export_env arm64 py38 dev openblas_build @@ -539,10 +516,6 @@ function main() { openblas_export_env arm64 py311 dev openblas_build ;; - openblas-arm64-py37) - openblas_export_env arm64 py37 - openblas_build - ;; openblas-arm64-py38) openblas_export_env arm64 py38 openblas_build @@ -593,9 +566,6 @@ function main() { ;; # CUDA wheels - cuda_wheel_py37_dev) - cuda_wheel_build py37 dev - ;; cuda_wheel_py38_dev) cuda_wheel_build py38 dev ;; @@ -608,9 +578,6 @@ function main() { cuda_wheel_py311_dev) cuda_wheel_build py311 dev ;; - cuda_wheel_py37) - cuda_wheel_build py37 - ;; cuda_wheel_py38) cuda_wheel_build py38 ;; diff --git a/docker/docker_test.sh b/docker/docker_test.sh index 6f26aaddcfd..877bbe1e5f8 100755 --- a/docker/docker_test.sh +++ b/docker/docker_test.sh @@ -16,24 +16,20 @@ __usage_docker_test="USAGE: OPTION: # OpenBLAS AMD64 (Dockerfile.openblas) - openblas-amd64-py37-dev : OpenBLAS AMD64 3.7 wheel, developer mode openblas-amd64-py38-dev : OpenBLAS AMD64 3.8 wheel, developer mode openblas-amd64-py39-dev : OpenBLAS AMD64 3.9 wheel, developer mode openblas-amd64-py310-dev : OpenBLAS AMD64 3.10 wheel, developer mode openblas-amd64-py311-dev : OpenBLAS AMD64 3.11 wheel, developer mode - openblas-amd64-py37 : OpenBLAS AMD64 3.7 wheel, release mode openblas-amd64-py38 : OpenBLAS AMD64 3.8 wheel, release mode openblas-amd64-py39 : OpenBLAS AMD64 3.9 wheel, release mode openblas-amd64-py310 : OpenBLAS AMD64 3.10 wheel, release mode openblas-amd64-py311 : OpenBLAS AMD64 3.11 wheel, release mode # OpenBLAS ARM64 (Dockerfile.openblas) - openblas-arm64-py37-dev : OpenBLAS ARM64 3.7 wheel, developer mode openblas-arm64-py38-dev : OpenBLAS ARM64 3.8 wheel, developer mode openblas-arm64-py39-dev : OpenBLAS ARM64 3.9 wheel, developer mode openblas-arm64-py310-dev : OpenBLAS ARM64 3.10 wheel, developer mode openblas-arm64-py311-dev : OpenBLAS ARM64 3.11 wheel, developer mode - openblas-arm64-py37 : OpenBLAS ARM64 3.7 wheel, release mode openblas-arm64-py38 : OpenBLAS ARM64 3.8 wheel, release mode openblas-arm64-py39 : OpenBLAS ARM64 3.9 wheel, release mode openblas-arm64-py310 : OpenBLAS ARM64 3.10 wheel, release mode @@ -212,11 +208,6 @@ echo "[$(basename $0)] building $1" source "${HOST_OPEN3D_ROOT}/docker/docker_build.sh" case "$1" in # OpenBLAS AMD64 -openblas-amd64-py37-dev) - openblas_export_env amd64 py37 dev - openblas_print_env - cpp_python_linking_uninstall_test - ;; openblas-amd64-py38-dev) openblas_export_env amd64 py38 dev openblas_print_env @@ -237,11 +228,6 @@ openblas-amd64-py311-dev) openblas_print_env cpp_python_linking_uninstall_test ;; -openblas-amd64-py37) - openblas_export_env amd64 py37 - openblas_print_env - cpp_python_linking_uninstall_test - ;; openblas-amd64-py38) openblas_export_env amd64 py38 openblas_print_env @@ -264,11 +250,6 @@ openblas-amd64-py311) ;; # OpenBLAS ARM64 -openblas-arm64-py37-dev) - openblas_export_env arm64 py37 dev - openblas_print_env - cpp_python_linking_uninstall_test - ;; openblas-arm64-py38-dev) openblas_export_env arm64 py38 dev openblas_print_env @@ -289,11 +270,6 @@ openblas-arm64-py311-dev) openblas_print_env cpp_python_linking_uninstall_test ;; -openblas-arm64-py37) - openblas_export_env arm64 py37 - openblas_print_env - cpp_python_linking_uninstall_test - ;; openblas-arm64-py38) openblas_export_env arm64 py38 openblas_print_env diff --git a/docs/arm.rst b/docs/arm.rst index 92e26ef8b78..584d0d007ad 100644 --- a/docs/arm.rst +++ b/docs/arm.rst @@ -67,7 +67,6 @@ commands: cd docker - ./docker_build.sh openblas-arm64-py37 # Python 3.7 ./docker_build.sh openblas-arm64-py38 # Python 3.8 ./docker_build.sh openblas-arm64-py39 # Python 3.9 ./docker_build.sh openblas-arm64-py310 # Python 3.10 diff --git a/docs/getting_started.in.rst b/docs/getting_started.in.rst index fc79078ef17..09bbe26b839 100644 --- a/docs/getting_started.in.rst +++ b/docs/getting_started.in.rst @@ -16,7 +16,6 @@ Supported Python versions: .. hlist:: :columns: 4 - * 3.7 * 3.8 * 3.9 * 3.10 @@ -72,28 +71,24 @@ version (``HEAD`` of ``master`` branch): :widths: auto * - Linux - - `Python 3.7 `__ - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ - `Python 3.11 `__ * - Linux (CPU) - - `Python 3.7 `__ - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ - `Python 3.11 `__ * - MacOS - - `Python 3.7 `__ - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ - `Python 3.11 `__ * - Windows - - `Python 3.7 `__ - `Python 3.8 `__ - `Python 3.9 `__ - `Python 3.10 `__ diff --git a/python/README.rst b/python/README.rst index 707ee14c138..f55cf004b2a 100644 --- a/python/README.rst +++ b/python/README.rst @@ -44,7 +44,6 @@ The package has been tested on: With Python versions: -* 3.7 * 3.8 * 3.9 * 3.10 diff --git a/python/setup.py b/python/setup.py index 225be4ce98b..07651bf1158 100644 --- a/python/setup.py +++ b/python/setup.py @@ -127,7 +127,6 @@ def finalize_options(self): "Programming Language :: C", "Programming Language :: C++", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", @@ -158,7 +157,7 @@ def finalize_options(self): setup_args = dict( name=name, version="@PROJECT_VERSION@", - python_requires=">=3.6", + python_requires=">=3.8", include_package_data=True, install_requires=install_requires, packages=find_packages(), From 83a2f410b9b30fcf41014a247d6a6c4c98eac296 Mon Sep 17 00:00:00 2001 From: Olivier Le Doeuff Date: Thu, 17 Aug 2023 09:29:32 +0200 Subject: [PATCH 09/19] temp commit to test open3d-ml update --- .github/workflows/documentation.yml | 2 +- .github/workflows/macos.yml | 2 +- .github/workflows/ubuntu-wheel.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index d48d5815971..74908674ece 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -39,7 +39,7 @@ jobs: - name: Checkout Open3D-ML source code uses: actions/checkout@v3 with: - repository: isl-org/Open3D-ML + repository: OlivierLDff/Open3D-ML path: ${{ env.OPEN3D_ML_ROOT }} - name: Setup cache diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 905bd65e31c..bce1332fba4 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -250,7 +250,7 @@ jobs: - name: Checkout Open3D-ML source code uses: actions/checkout@v3 with: - repository: isl-org/Open3D-ML + repository: OlivierLDff/Open3D-ML path: ${{ env.OPEN3D_ML_ROOT }} - name: Download wheels diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index e0958994614..b3e679c890d 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -132,7 +132,7 @@ jobs: - name: Checkout Open3D-ML source code uses: actions/checkout@v3 with: - repository: isl-org/Open3D-ML + repository: OlivierLDff/Open3D-ML path: ${{ env.OPEN3D_ML_ROOT }} - name: Download wheels uses: actions/download-artifact@v3 From 3f683a6b438b4c697bf59d2db32cf7f7c3fc863b Mon Sep 17 00:00:00 2001 From: Olivier Le Doeuff Date: Thu, 17 Aug 2023 11:46:00 +0200 Subject: [PATCH 10/19] temp: try to ignore macos framework? --- 3rdparty/cmake/FindTensorflow.cmake | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/3rdparty/cmake/FindTensorflow.cmake b/3rdparty/cmake/FindTensorflow.cmake index 815fa497cde..037266a0bac 100644 --- a/3rdparty/cmake/FindTensorflow.cmake +++ b/3rdparty/cmake/FindTensorflow.cmake @@ -41,12 +41,12 @@ if(NOT Tensorflow_FOUND) string(REGEX REPLACE "::" ";" Tensorflow_DEFINITIONS ${Tensorflow_DEFINITIONS}) # Get Tensorflow_FRAMEWORK_LIB - find_library( - Tensorflow_FRAMEWORK_LIB - NAMES tensorflow_framework libtensorflow_framework.so.2 - PATHS "${Tensorflow_LIB_DIR}" - NO_DEFAULT_PATH - ) + # find_library( + # Tensorflow_FRAMEWORK_LIB + # NAMES tensorflow_framework libtensorflow_framework.so.2 + # PATHS "${Tensorflow_LIB_DIR}" + # NO_DEFAULT_PATH + # ) endif() message(STATUS "TensorFlow version: ${Tensorflow_VERSION}") @@ -76,4 +76,4 @@ endif() include(FindPackageHandleStandardArgs) find_package_handle_standard_args( Tensorflow DEFAULT_MSG Tensorflow_INCLUDE_DIR Tensorflow_LIB_DIR - Tensorflow_FRAMEWORK_LIB Tensorflow_DEFINITIONS) + Tensorflow_DEFINITIONS) From 83b0643654b6cc0dbd31a51e6e7eec7c13e81f11 Mon Sep 17 00:00:00 2001 From: Olivier Le Doeuff Date: Thu, 17 Aug 2023 14:09:34 +0200 Subject: [PATCH 11/19] temp: try to update cmake osx min version to see if it can fix some build --- 3rdparty/vtk/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/3rdparty/vtk/CMakeLists.txt b/3rdparty/vtk/CMakeLists.txt index 7188cb76325..dd74c820c86 100644 --- a/3rdparty/vtk/CMakeLists.txt +++ b/3rdparty/vtk/CMakeLists.txt @@ -30,7 +30,7 @@ if(WIN32) endif() if (APPLE) -set (CMAKE_OSX_DEPLOYMENT_TARGET "10.15" CACHE STRING +set (CMAKE_OSX_DEPLOYMENT_TARGET "12.6" CACHE STRING "Minimum OS X deployment version" FORCE) endif() From cb6efb07fc9d0dd12268012137409c46a87e5519 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Fri, 22 Sep 2023 15:31:26 -0700 Subject: [PATCH 12/19] Python 3.11 on macOS Update to C++ 17 TensorFlow 2.13 Tensorboard 2.13 scipy 1.11.2 --- .gitignore | 1 + 3rdparty/cmake/FindPytorch.cmake | 22 ++++++------ 3rdparty/cmake/FindTensorflow.cmake | 34 ++++++++++--------- CMakeLists.txt | 6 +--- cpp/open3d/ml/tensorflow/CMakeLists.txt | 3 ++ .../ContinuousConvBackpropFilterOps.cpp | 4 +-- .../continuous_conv/ContinuousConvOps.cpp | 4 +-- ...ntinuousConvTransposeBackpropFilterOps.cpp | 4 +-- .../ContinuousConvTransposeOps.cpp | 4 +-- .../visualization/tensorboard_plugin/util.py | 2 +- python/requirements_test.txt | 4 +-- python/test/core/test_core.py | 8 ++--- 12 files changed, 50 insertions(+), 46 deletions(-) diff --git a/.gitignore b/.gitignore index cb1dbd7c1a6..8a041878cc6 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ docs/tutorial/**/*.ply docs/tutorial/**/*.pcd docs/tutorial/**/*.json docs/_out/ +docs/_build/ docs/python_api/ docs/python_example/ docs/conf.py diff --git a/3rdparty/cmake/FindPytorch.cmake b/3rdparty/cmake/FindPytorch.cmake index 202af7c461a..9d85ef4bbfa 100644 --- a/3rdparty/cmake/FindPytorch.cmake +++ b/3rdparty/cmake/FindPytorch.cmake @@ -97,18 +97,20 @@ foreach(lib ${TORCH_LIBRARIES}) endforeach(lib) # Check if the c++11 ABI is compatible on Linux -if(UNIX AND NOT APPLE AND ((Pytorch_CXX11_ABI AND (NOT GLIBCXX_USE_CXX11_ABI)) OR - (NOT Pytorch_CXX11_ABI AND GLIBCXX_USE_CXX11_ABI))) - if(Pytorch_CXX11_ABI) - set(NEEDED_ABI_FLAG "ON") +if(UNIX AND NOT APPLE) + if((Pytorch_CXX11_ABI AND (NOT GLIBCXX_USE_CXX11_ABI)) OR + (NOT Pytorch_CXX11_ABI AND GLIBCXX_USE_CXX11_ABI)) + if(Pytorch_CXX11_ABI) + set(NEEDED_ABI_FLAG "ON") + else() + set(NEEDED_ABI_FLAG "OFF") + endif() + message(FATAL_ERROR "PyTorch and Open3D ABI mismatch: ${Pytorch_CXX11_ABI} != ${GLIBCXX_USE_CXX11_ABI}.\n" + "Please use -DGLIBCXX_USE_CXX11_ABI=${NEEDED_ABI_FLAG} " + "in the cmake config command to change the Open3D ABI.") else() - set(NEEDED_ABI_FLAG "OFF") + message(STATUS "PyTorch matches Open3D ABI: ${Pytorch_CXX11_ABI} == ${GLIBCXX_USE_CXX11_ABI}") endif() - message(FATAL_ERROR "PyTorch and Open3D ABI mismatch: ${Pytorch_CXX11_ABI} != ${GLIBCXX_USE_CXX11_ABI}.\n" - "Please use -DGLIBCXX_USE_CXX11_ABI=${NEEDED_ABI_FLAG} " - "in the cmake config command to change the Open3D ABI.") -else() - message(STATUS "PyTorch matches Open3D ABI: ${Pytorch_CXX11_ABI} == ${GLIBCXX_USE_CXX11_ABI}") endif() include(FindPackageHandleStandardArgs) diff --git a/3rdparty/cmake/FindTensorflow.cmake b/3rdparty/cmake/FindTensorflow.cmake index 037266a0bac..6dc3d3676a2 100644 --- a/3rdparty/cmake/FindTensorflow.cmake +++ b/3rdparty/cmake/FindTensorflow.cmake @@ -41,12 +41,12 @@ if(NOT Tensorflow_FOUND) string(REGEX REPLACE "::" ";" Tensorflow_DEFINITIONS ${Tensorflow_DEFINITIONS}) # Get Tensorflow_FRAMEWORK_LIB - # find_library( - # Tensorflow_FRAMEWORK_LIB - # NAMES tensorflow_framework libtensorflow_framework.so.2 - # PATHS "${Tensorflow_LIB_DIR}" - # NO_DEFAULT_PATH - # ) + find_library( + Tensorflow_FRAMEWORK_LIB + NAMES tensorflow_framework tensorflow_framework.2 libtensorflow_framework.so.2 + PATHS "${Tensorflow_LIB_DIR}" + NO_DEFAULT_PATH + ) endif() message(STATUS "TensorFlow version: ${Tensorflow_VERSION}") @@ -59,18 +59,20 @@ if (UNIX AND NOT APPLE) endif() # Check if the c++11 ABI is compatible -if(UNIX AND NOT APPLE AND ((Tensorflow_CXX11_ABI AND (NOT GLIBCXX_USE_CXX11_ABI)) OR - (NOT Tensorflow_CXX11_ABI AND GLIBCXX_USE_CXX11_ABI))) - if(TensorFlow_CXX11_ABI) - set(NEEDED_ABI_FLAG "ON") +if (UNIX AND NOT APPLE) + if(((Tensorflow_CXX11_ABI AND (NOT GLIBCXX_USE_CXX11_ABI)) OR + (NOT Tensorflow_CXX11_ABI AND GLIBCXX_USE_CXX11_ABI))) + if(TensorFlow_CXX11_ABI) + set(NEEDED_ABI_FLAG "ON") + else() + set(NEEDED_ABI_FLAG "OFF") + endif() + message(FATAL_ERROR "TensorFlow and Open3D ABI mismatch: ${Tensorflow_CXX11_ABI} != ${GLIBCXX_USE_CXX11_ABI}.\n" + "Please use -D GLIBCXX_USE_CXX11_ABI=${NEEDED_ABI_FLAG} " + "in the cmake config command to change the Open3D ABI.") else() - set(NEEDED_ABI_FLAG "OFF") + message(STATUS "TensorFlow matches Open3D ABI: ${Tensorflow_CXX11_ABI} == ${GLIBCXX_USE_CXX11_ABI}") endif() - message(FATAL_ERROR "TensorFlow and Open3D ABI mismatch: ${Tensorflow_CXX11_ABI} != ${GLIBCXX_USE_CXX11_ABI}.\n" - "Please use -D GLIBCXX_USE_CXX11_ABI=${NEEDED_ABI_FLAG} " - "in the cmake config command to change the Open3D ABI.") -else() - message(STATUS "TensorFlow matches Open3D ABI: ${Tensorflow_CXX11_ABI} == ${GLIBCXX_USE_CXX11_ABI}") endif() include(FindPackageHandleStandardArgs) diff --git a/CMakeLists.txt b/CMakeLists.txt index e8c97dbf848..bce94eb0968 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -290,11 +290,7 @@ endif() # Global flag to set CXX standard. # This does not affect 3rd party libraries. -if (BUILD_SYCL_MODULE) - set(CMAKE_CXX_STANDARD 17) -else() - set(CMAKE_CXX_STANDARD 14) -endif() +set(CMAKE_CXX_STANDARD 17) # FIXME: Remove this workaround once a fixed Visual Studio 16.10 version is released. if (BUILD_CUDA_MODULE diff --git a/cpp/open3d/ml/tensorflow/CMakeLists.txt b/cpp/open3d/ml/tensorflow/CMakeLists.txt index 756e76b07b5..cecd4152c49 100644 --- a/cpp/open3d/ml/tensorflow/CMakeLists.txt +++ b/cpp/open3d/ml/tensorflow/CMakeLists.txt @@ -136,6 +136,9 @@ set_target_properties(open3d_tf_ops PROPERTIES set_target_properties(open3d_tf_ops PROPERTIES PREFIX "") set_target_properties(open3d_tf_ops PROPERTIES DEBUG_POSTFIX "_debug") +# _GLIBCXX_USER_CXX11_ABI is set separately +list(REMOVE_ITEM Tensorflow_DEFINITIONS "_GLIBCXX_USE_CXX11_ABI=0" + "_GLIBCXX_USE_CXX11_ABI=1") target_compile_definitions(open3d_tf_ops PRIVATE "${Tensorflow_DEFINITIONS}") if (BUILD_CUDA_MODULE) diff --git a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvBackpropFilterOps.cpp b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvBackpropFilterOps.cpp index 3fa594026e2..06a4df2b463 100644 --- a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvBackpropFilterOps.cpp +++ b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvBackpropFilterOps.cpp @@ -120,8 +120,8 @@ REGISTER_OP("Open3DContinuousConvBackpropFilter") if (c->ValueKnown(c->Dim(filters_shape, i))) { int64_t n = c->Value(c->Dim(filters_shape, i)); if (n < 1) - return Status(error::INVALID_ARGUMENT, - "Each filter dimension must be >= 1"); + return absl::InvalidArgumentError( + "Each filter dimension must be >= 1"); } } diff --git a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvOps.cpp b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvOps.cpp index 154ed9e0813..a86f9dc7396 100644 --- a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvOps.cpp +++ b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvOps.cpp @@ -116,8 +116,8 @@ REGISTER_OP("Open3DContinuousConv") if (c->ValueKnown(c->Dim(filters_shape, i))) { int64_t n = c->Value(c->Dim(filters_shape, i)); if (n < 1) - return Status(error::INVALID_ARGUMENT, - "Each filter dimension must be >= 1"); + return absl::InvalidArgumentError( + "Each filter dimension must be >= 1"); } } diff --git a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeBackpropFilterOps.cpp b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeBackpropFilterOps.cpp index ae13c7fd12c..613224b562c 100644 --- a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeBackpropFilterOps.cpp +++ b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeBackpropFilterOps.cpp @@ -139,8 +139,8 @@ REGISTER_OP("Open3DContinuousConvTransposeBackpropFilter") if (c->ValueKnown(c->Dim(filters_shape, i))) { int64_t n = c->Value(c->Dim(filters_shape, i)); if (n < 1) - return Status(error::INVALID_ARGUMENT, - "Each filter dimension must be >= 1"); + return absl::InvalidArgumentError( + "Each filter dimension must be >= 1"); } } diff --git a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeOps.cpp b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeOps.cpp index 19a41bb40b3..b04c5b62afa 100644 --- a/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeOps.cpp +++ b/cpp/open3d/ml/tensorflow/continuous_conv/ContinuousConvTransposeOps.cpp @@ -145,8 +145,8 @@ REGISTER_OP("Open3DContinuousConvTranspose") if (c->ValueKnown(c->Dim(filters_shape, i))) { int64_t n = c->Value(c->Dim(filters_shape, i)); if (n < 1) - return Status(error::INVALID_ARGUMENT, - "Each filter dimension must be >= 1"); + return absl::InvalidArgumentError( + "Each filter dimension must be >= 1"); } } diff --git a/python/open3d/visualization/tensorboard_plugin/util.py b/python/open3d/visualization/tensorboard_plugin/util.py index b057ee1a5b0..8c8a5beda34 100644 --- a/python/open3d/visualization/tensorboard_plugin/util.py +++ b/python/open3d/visualization/tensorboard_plugin/util.py @@ -72,7 +72,7 @@ def release_read(self): try: self._readers -= 1 if not self._readers: - self._read_ready.notifyAll() + self._read_ready.notify_all() finally: self._read_ready.release() diff --git a/python/requirements_test.txt b/python/requirements_test.txt index 7eafb00b60a..1d86929df65 100644 --- a/python/requirements_test.txt +++ b/python/requirements_test.txt @@ -1,6 +1,6 @@ pytest==7.1.2 pytest-randomly==3.8.0 -scipy==1.7.3 -tensorboard==2.8.0 +scipy==1.11.2 +tensorboard==2.13.0 oauthlib==3.2.2 certifi==2022.12.7 diff --git a/python/test/core/test_core.py b/python/test/core/test_core.py index b0fac5c8f0c..47200dfe338 100644 --- a/python/test/core/test_core.py +++ b/python/test/core/test_core.py @@ -61,8 +61,8 @@ def to_numpy_dtype(dtype: o3c.Dtype): o3c.uint16: np.uint16, o3c.uint32: np.uint32, o3c.uint64: np.uint64, - o3c.bool8: np.bool8, # np.bool deprecated - o3c.bool: np.bool8, # o3c.bool is an alias for o3c.bool8 + o3c.bool8: np.bool_, + o3c.bool: np.bool_, # o3c.bool is an alias for o3c.bool8 } return conversions[dtype] @@ -163,7 +163,7 @@ def test_tensor_constructor(dtype, device): np.testing.assert_equal(np_t, o3_t.cpu().numpy()) # Boolean - np_t = np.array([True, False, True], dtype=np.bool8) + np_t = np.array([True, False, True], dtype=np.bool_) o3_t = o3c.Tensor([True, False, True], o3c.bool, device) np.testing.assert_equal(np_t, o3_t.cpu().numpy()) o3_t = o3c.Tensor(np_t, o3c.bool, device) @@ -801,7 +801,7 @@ def test_setitem(device): np.testing.assert_equal(o3_t.cpu().numpy(), np_t) # Scalar boolean set item - np_t = np.eye(4, dtype=np.bool8) + np_t = np.eye(4, dtype=np.bool_) o3_t = o3c.Tensor.eye(4, dtype=o3c.bool) np_t[2, 2] = False o3_t[2, 2] = False From 6b7f0950d41f8605f6cb93408fa3b1bc527412bd Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Fri, 22 Sep 2023 16:25:10 -0700 Subject: [PATCH 13/19] TensorFlow / Tensorboard 2.13 --- util/ci_utils.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 9b606e4dc55..d7f44d412ef 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -40,8 +40,8 @@ else GCC_MAX_VER=7 fi # ML -TENSORFLOW_VER="2.12.0" -TENSORBOARD_VER="2.14.0" +TENSORFLOW_VER="2.13.0" +TENSORBOARD_VER="2.13.0" TORCH_VER="2.0.1" TORCH_CPU_GLNX_VER="${TORCH_VER}+cpu" TORCH_CUDA_GLNX_VER="${TORCH_VER}+cu117" From 7236e334a98123433dc284c576806a18b528028a Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Sun, 24 Sep 2023 22:34:31 -0700 Subject: [PATCH 14/19] Ignore tensorflow tests if TF / open3d.ml.tf is not available. Move PyTorch compile warning to where PyTorch version is available. --- 3rdparty/cmake/FindPytorch.cmake | 23 +++++++++++++++++++ cmake/Open3DPrintConfigurationSummary.cmake | 21 ----------------- python/test/ml_ops/test_cconv_python.py | 8 +++---- .../test/ml_ops/test_fixed_radius_search.py | 6 +++-- python/test/ml_ops/test_knn_search.py | 6 +++-- python/test/ml_ops/test_radius_search.py | 6 +++-- 6 files changed, 39 insertions(+), 31 deletions(-) diff --git a/3rdparty/cmake/FindPytorch.cmake b/3rdparty/cmake/FindPytorch.cmake index 9d85ef4bbfa..eb2a53e2ec5 100644 --- a/3rdparty/cmake/FindPytorch.cmake +++ b/3rdparty/cmake/FindPytorch.cmake @@ -113,6 +113,29 @@ if(UNIX AND NOT APPLE) endif() endif() +message(STATUS "Pytorch_VERSION: ${Pytorch_VERSION}, CUDAToolkit_VERSION: ${CUDAToolkit_VERSION}") +if (BUILD_PYTORCH_OPS AND BUILD_CUDA_MODULE AND CUDAToolkit_VERSION + VERSION_GREATER_EQUAL "11.0" AND Pytorch_VERSION VERSION_LESS + "1.9") + message(WARNING + "--------------------------------------------------------------------------------\n" + " \n" + " You are compiling PyTorch ops with CUDA 11 with PyTorch version < 1.9. This \n" + " configuration may have stability issues. See \n" + " https://github.com/isl-org/Open3D/issues/3324 and \n" + " https://github.com/pytorch/pytorch/issues/52663 for more information on this \n" + " problem. \n" + " \n" + " We recommend to compile PyTorch from source with compile flags \n" + " '-Xcompiler -fno-gnu-unique' \n" + " \n" + " or use the PyTorch wheels at \n" + " https://github.com/isl-org/open3d_downloads/releases/tag/torch1.8.2 \n" + " \n" + "--------------------------------------------------------------------------------\n" + ) +endif() + include(FindPackageHandleStandardArgs) find_package_handle_standard_args(Pytorch DEFAULT_MSG Pytorch_VERSION Pytorch_ROOT) diff --git a/cmake/Open3DPrintConfigurationSummary.cmake b/cmake/Open3DPrintConfigurationSummary.cmake index 9427d08c0c7..c8e8a891a8b 100644 --- a/cmake/Open3DPrintConfigurationSummary.cmake +++ b/cmake/Open3DPrintConfigurationSummary.cmake @@ -40,27 +40,6 @@ function(open3d_print_configuration_summary) open3d_aligned_print("Build Jupyter Extension" "${BUILD_JUPYTER_EXTENSION}") open3d_aligned_print("Build TensorFlow Ops" "${BUILD_TENSORFLOW_OPS}") open3d_aligned_print("Build PyTorch Ops" "${BUILD_PYTORCH_OPS}") - if (BUILD_PYTORCH_OPS AND BUILD_CUDA_MODULE AND CUDAToolkit_VERSION - VERSION_GREATER_EQUAL "11.0" AND Pytorch_VERSION VERSION_LESS - "1.9") - message(WARNING - "--------------------------------------------------------------------------------\n" - " \n" - " You are compiling PyTorch ops with CUDA 11 with PyTorch version < 1.9. This \n" - " configuration may have stability issues. See \n" - " https://github.com/isl-org/Open3D/issues/3324 and \n" - " https://github.com/pytorch/pytorch/issues/52663 for more information on this \n" - " problem. \n" - " \n" - " We recommend to compile PyTorch from source with compile flags \n" - " '-Xcompiler -fno-gnu-unique' \n" - " \n" - " or use the PyTorch wheels at \n" - " https://github.com/isl-org/open3d_downloads/releases/tag/torch1.8.2 \n" - " \n" - "--------------------------------------------------------------------------------\n" - ) - endif() open3d_aligned_print("Build Benchmarks" "${BUILD_BENCHMARKS}") open3d_aligned_print("Bundle Open3D-ML" "${BUNDLE_OPEN3D_ML}") if(GLIBCXX_USE_CXX11_ABI) diff --git a/python/test/ml_ops/test_cconv_python.py b/python/test/ml_ops/test_cconv_python.py index 32a30aba4e7..2461d84b114 100644 --- a/python/test/ml_ops/test_cconv_python.py +++ b/python/test/ml_ops/test_cconv_python.py @@ -28,8 +28,8 @@ def test_compare_to_conv3d(dtype, filter_size, out_channels, in_channels, with_inp_importance, with_normalization): """Compares to the 3D convolution in tensorflow""" - import tensorflow as tf - import open3d.ml.tf as ml3d + tf = pytest.importorskip('tensorflow') + ml3d = pytest.importorskip('open3d.ml.tf') np.random.seed(0) conv_attrs = { @@ -123,8 +123,8 @@ def test_cconv_gradient(filter_size, out_channels, in_channels, with_inp_importance, with_neighbors_importance, with_individual_extent, with_normalization, align_corners, coordinate_mapping, interpolation): - import tensorflow as tf - import open3d.ml.tf as ml3d + tf = pytest.importorskip('tensorflow') + ml3d = pytest.importorskip('open3d.ml.tf') dtype = np.float64 np.random.seed(0) diff --git a/python/test/ml_ops/test_fixed_radius_search.py b/python/test/ml_ops/test_fixed_radius_search.py index 71ce5e1270a..9d2af9b58e3 100644 --- a/python/test/ml_ops/test_fixed_radius_search.py +++ b/python/test/ml_ops/test_fixed_radius_search.py @@ -10,8 +10,10 @@ from scipy.spatial import cKDTree import pytest import mltest -import torch -import tensorflow as tf +if o3d._build_config['BUILD_PYTORCH_OPS']: + import torch +if o3d._build_config['BUILD_TENSORFLOW_OPS']: + import tensorflow as tf # skip all tests if the ml ops were not built pytestmark = mltest.default_marks diff --git a/python/test/ml_ops/test_knn_search.py b/python/test/ml_ops/test_knn_search.py index e4bdae38827..93f32aab9f7 100644 --- a/python/test/ml_ops/test_knn_search.py +++ b/python/test/ml_ops/test_knn_search.py @@ -10,8 +10,10 @@ from scipy.spatial import cKDTree import pytest import mltest -import torch -import tensorflow as tf +if o3d._build_config['BUILD_PYTORCH_OPS']: + import torch +if o3d._build_config['BUILD_TENSORFLOW_OPS']: + import tensorflow as tf # skip all tests if the ml ops were not built pytestmark = mltest.default_marks diff --git a/python/test/ml_ops/test_radius_search.py b/python/test/ml_ops/test_radius_search.py index 6ce8da8b478..f573b2cd2fe 100644 --- a/python/test/ml_ops/test_radius_search.py +++ b/python/test/ml_ops/test_radius_search.py @@ -10,8 +10,10 @@ from scipy.spatial import cKDTree import pytest import mltest -import torch -import tensorflow as tf +if o3d._build_config['BUILD_PYTORCH_OPS']: + import torch +if o3d._build_config['BUILD_TENSORFLOW_OPS']: + import tensorflow as tf # skip all tests if the tf ops were not built and disable warnings caused by # tensorflow From 0ca03ad40ce69e455bb4e59d5f7be2e1570fb936 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Mon, 25 Sep 2023 11:55:02 -0700 Subject: [PATCH 15/19] Style fix and... Update pip, scipy versions. Use requirements_test.txt in CI tensorflow-gpu -> tensorflow for 2.13+ remove CUDA_VERSION setting in ci_utils.sh. THis happens in docker/build_docker.sh now. --- .github/workflows/documentation.yml | 2 +- .github/workflows/macos.yml | 6 ++- .github/workflows/ubuntu-wheel.yml | 6 ++- .github/workflows/vtk_packages.yml | 1 + .github/workflows/windows.yml | 15 +++----- python/requirements_build.txt | 2 +- python/requirements_test.txt | 2 +- .../test/ml_ops/test_fixed_radius_search.py | 2 +- python/test/ml_ops/test_knn_search.py | 2 +- python/test/ml_ops/test_radius_search.py | 2 +- util/ci_utils.sh | 37 ++++--------------- 11 files changed, 30 insertions(+), 47 deletions(-) diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 74908674ece..d48d5815971 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -39,7 +39,7 @@ jobs: - name: Checkout Open3D-ML source code uses: actions/checkout@v3 with: - repository: OlivierLDff/Open3D-ML + repository: isl-org/Open3D-ML path: ${{ env.OPEN3D_ML_ROOT }} - name: Setup cache diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index bce1332fba4..bfd53f70547 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -135,6 +135,8 @@ jobs: python_version: '3.8' - is_master: false python_version: '3.9' + - is_master: false + python_version: '3.10' env: BUILD_CUDA_MODULE: OFF @@ -240,6 +242,8 @@ jobs: python_version: '3.8' - is_master: false python_version: '3.9' + - is_master: false + python_version: '3.10' env: OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML @@ -250,7 +254,7 @@ jobs: - name: Checkout Open3D-ML source code uses: actions/checkout@v3 with: - repository: OlivierLDff/Open3D-ML + repository: isl-org/Open3D-ML path: ${{ env.OPEN3D_ML_ROOT }} - name: Download wheels diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index b3e679c890d..a19726fa4ff 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -35,6 +35,8 @@ jobs: python_version: '3.8' - is_master: false python_version: '3.9' + - is_master: false + python_version: '3.10' env: DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }} PYTHON_VERSION: ${{ matrix.python_version }} @@ -120,6 +122,8 @@ jobs: python_version: '3.8' - is_master: false python_version: '3.9' + - is_master: false + python_version: '3.10' env: OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML steps: @@ -132,7 +136,7 @@ jobs: - name: Checkout Open3D-ML source code uses: actions/checkout@v3 with: - repository: OlivierLDff/Open3D-ML + repository: isl-org/Open3D-ML path: ${{ env.OPEN3D_ML_ROOT }} - name: Download wheels uses: actions/download-artifact@v3 diff --git a/.github/workflows/vtk_packages.yml b/.github/workflows/vtk_packages.yml index bcde81876f0..30347cc2941 100644 --- a/.github/workflows/vtk_packages.yml +++ b/.github/workflows/vtk_packages.yml @@ -9,6 +9,7 @@ on: jobs: Linux: + # TODO: Convert to docker runs-on: ubuntu-18.04 steps: - name: Checkout source code diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index bd1074e543a..66c7e5ba084 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -19,15 +19,11 @@ concurrency: cancel-in-progress: true env: - PIP_VER: "21.1.1" + PIP_VER: "23.2.1" WHEEL_VER: "0.38.4" STOOLS_VER: "67.3.2" - PYTEST_VER: "7.1.2" - PYTEST_RANDOMLY_VER: "3.8.0" - SCIPY_VER: "1.7.3" JEDI_VER: "0.17.2" # https://github.com/ipython/ipython/issues/12740 IDNA_VER: "2.8" # https://github.com/psf/requests/issues/5710 - TENSORBOARD_VER: "2.8" SRC_DIR: "D:\\a\\open3d\\open3d" BUILD_DIR: "C:\\Open3D\\build" NPROC: 2 @@ -254,6 +250,8 @@ jobs: python_version: '3.8' - is_master: false python_version: '3.9' + - is_master: false + python_version: '3.10' steps: - name: Checkout source code @@ -357,6 +355,8 @@ jobs: python_version: '3.8' - is_master: false python_version: '3.9' + - is_master: false + python_version: '3.10' steps: - name: Checkout source code @@ -384,10 +384,7 @@ jobs: python -m pip install --upgrade pip==${{ env.PIP_VER }} ` wheel==${{ env.WHEEL_VER }} ` setuptools==${{ env.STOOLS_VER }} - python -m pip install -U pytest==${{ env.PYTEST_VER }} - python -m pip install -U pytest-randomly==${{ env.PYTEST_RANDOMLY_VER }} - python -m pip install -U scipy==${{ env.SCIPY_VER }} ` - tensorboard==${{ env.TENSORBOARD_VER }} + python -m pip install -U -r python/requirements_test.txt $py_tag=(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')") if (Test-Path -Path "pip_package") { $PIP_PKG_NAME=(Get-ChildItem pip_package\open3d*-$py_tag-*.whl).Name diff --git a/python/requirements_build.txt b/python/requirements_build.txt index 308feb49aeb..b9bb3b1f28a 100644 --- a/python/requirements_build.txt +++ b/python/requirements_build.txt @@ -1,3 +1,3 @@ -setuptools>=50.3.2 +setuptools>=67.3.2 wheel==0.38.4 yapf==0.30.0 diff --git a/python/requirements_test.txt b/python/requirements_test.txt index f50add8b83d..4a2f418b6bb 100644 --- a/python/requirements_test.txt +++ b/python/requirements_test.txt @@ -1,6 +1,6 @@ pytest==7.1.2 pytest-randomly==3.8.0 -scipy==1.11.2 +scipy==1.10.1 tensorboard==2.13.0 oauthlib==3.2.2 certifi==2023.7.22 diff --git a/python/test/ml_ops/test_fixed_radius_search.py b/python/test/ml_ops/test_fixed_radius_search.py index 9d2af9b58e3..f0ccf87eada 100644 --- a/python/test/ml_ops/test_fixed_radius_search.py +++ b/python/test/ml_ops/test_fixed_radius_search.py @@ -12,7 +12,7 @@ import mltest if o3d._build_config['BUILD_PYTORCH_OPS']: import torch -if o3d._build_config['BUILD_TENSORFLOW_OPS']: +if o3d._build_config['BUILD_TENSORFLOW_OPS']: import tensorflow as tf # skip all tests if the ml ops were not built diff --git a/python/test/ml_ops/test_knn_search.py b/python/test/ml_ops/test_knn_search.py index 93f32aab9f7..175835f6593 100644 --- a/python/test/ml_ops/test_knn_search.py +++ b/python/test/ml_ops/test_knn_search.py @@ -12,7 +12,7 @@ import mltest if o3d._build_config['BUILD_PYTORCH_OPS']: import torch -if o3d._build_config['BUILD_TENSORFLOW_OPS']: +if o3d._build_config['BUILD_TENSORFLOW_OPS']: import tensorflow as tf # skip all tests if the ml ops were not built diff --git a/python/test/ml_ops/test_radius_search.py b/python/test/ml_ops/test_radius_search.py index f573b2cd2fe..85b2af0ad0d 100644 --- a/python/test/ml_ops/test_radius_search.py +++ b/python/test/ml_ops/test_radius_search.py @@ -12,7 +12,7 @@ import mltest if o3d._build_config['BUILD_PYTORCH_OPS']: import torch -if o3d._build_config['BUILD_TENSORFLOW_OPS']: +if o3d._build_config['BUILD_TENSORFLOW_OPS']: import tensorflow as tf # skip all tests if the tf ops were not built and disable warnings caused by diff --git a/util/ci_utils.sh b/util/ci_utils.sh index d7f44d412ef..09ab1720141 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -23,38 +23,19 @@ BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS:-ON} LOW_MEM_USAGE=${LOW_MEM_USAGE:-OFF} # Dependency versions: -# CUDA -if [[ $BUILD_TENSORFLOW_OPS == ON || $BUILD_PYTORCH_OPS == ON || - $UBUNTU_VERSION != bionic ]]; then - # CUDA version in sync with PyTorch and Tensorflow - CUDA_VERSION=("11-7" "11.7") - CUDNN_MAJOR_VERSION=8 - CUDNN_VERSION="8.0.5.39_cuda11.7" - GCC_MAX_VER=9 -else - # Without MLOps, ensure Open3D works with the lowest supported CUDA version - # Not available in Nvidia focal repos - CUDA_VERSION=("10-1" "10.1") - CUDNN_MAJOR_VERSION=8 - CUDNN_VERSION="8.0.5.39-1+cuda10.1" - GCC_MAX_VER=7 -fi +# CUDA: see docker/docker_build.sh # ML TENSORFLOW_VER="2.13.0" -TENSORBOARD_VER="2.13.0" TORCH_VER="2.0.1" TORCH_CPU_GLNX_VER="${TORCH_VER}+cpu" -TORCH_CUDA_GLNX_VER="${TORCH_VER}+cu117" +TORCH_CUDA_GLNX_VER="${TORCH_VER}+cu117" # match CUDA_VERSION in docker/docker_build.sh PYTHON_VER=$(python -c 'import sys; ver=f"{sys.version_info.major}{sys.version_info.minor}"; print(f"cp{ver}-cp{ver}{sys.abiflags}")' 2>/dev/null || true) TORCH_MACOS_VER="${TORCH_VER}" TORCH_REPO_URL="https://download.pytorch.org/whl/torch/" # Python -PIP_VER="21.1.1" +PIP_VER="23.2.1" WHEEL_VER="0.38.4" STOOLS_VER="67.3.2" -PYTEST_VER="7.1.2" -PYTEST_RANDOMLY_VER="3.8.0" -SCIPY_VER="1.7.3" YAPF_VER="0.30.0" PROTOBUF_VER="4.24.0" @@ -68,16 +49,15 @@ install_python_dependencies() { python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \ setuptools=="$STOOLS_VER" if [[ "with-unit-test" =~ ^($options)$ ]]; then - python -m pip install -U scipy=="$SCIPY_VER" pytest=="$PYTEST_VER" \ - pytest-randomly=="$PYTEST_RANDOMLY_VER" + python -m pip install -U -r python/requirements_test.txt fi if [[ "with-cuda" =~ ^($options)$ ]]; then - TF_ARCH_NAME=tensorflow-gpu + TF_ARCH_NAME=tensorflow TF_ARCH_DISABLE_NAME=tensorflow-cpu TORCH_GLNX="torch==$TORCH_CUDA_GLNX_VER" else TF_ARCH_NAME=tensorflow-cpu - TF_ARCH_DISABLE_NAME=tensorflow-gpu + TF_ARCH_DISABLE_NAME=tensorflow TORCH_GLNX="torch==$TORCH_CPU_GLNX_VER" fi @@ -300,10 +280,7 @@ test_wheel() { run_python_tests() { # shellcheck disable=SC1091 source open3d_test.venv/bin/activate - python -m pip install -U pytest=="$PYTEST_VER" \ - pytest-randomly=="$PYTEST_RANDOMLY_VER" \ - scipy=="$SCIPY_VER" \ - tensorboard=="$TENSORBOARD_VER" + python -m pip install -U -r python/requirements_test.txt echo Add --randomly-seed=SEED to the test command to reproduce test order. pytest_args=("$OPEN3D_SOURCE_ROOT"/python/test/) if [ "$BUILD_PYTORCH_OPS" == "OFF" ] || [ "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then From c61fb71795e7c86e6639a7f30d5c3f6d46bc3bbd Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Tue, 3 Oct 2023 15:08:13 -0700 Subject: [PATCH 16/19] C++17 only if TF ops are enabled --- CMakeLists.txt | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bce94eb0968..c3d64e4b95d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -290,7 +290,14 @@ endif() # Global flag to set CXX standard. # This does not affect 3rd party libraries. -set(CMAKE_CXX_STANDARD 17) +# Tensorflow 2.9+ requires cxx_17, but MSVC 19.29 throws errors with C++17 +# enabled. +if (BUILD_SYCL_MODULE OR BUILD_TENSORFLOW_OPS) + set(CMAKE_CXX_STANDARD 17) +else() + set(CMAKE_CXX_STANDARD 14) +endif() +set(CMAKE_CXX_EXTENSIONS OFF) # Improved compatibility # FIXME: Remove this workaround once a fixed Visual Studio 16.10 version is released. if (BUILD_CUDA_MODULE From 7f9a3bec6be699847b1ff1fe7b56b7232b6d2165 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Thu, 12 Oct 2023 21:25:42 -0700 Subject: [PATCH 17/19] Don't build tf_ops explicitly. --- docker/Dockerfile.wheel | 2 +- util/ci_utils.sh | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index cf10ca879f9..e1c184dc202 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -123,9 +123,9 @@ COPY . /root/Open3D WORKDIR /root/Open3D # Build python wheel + # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch RUN export NPROC=$(nproc) \ && export BUILD_SHARED_LIBS=OFF \ - # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch && export BUILD_TENSORFLOW_OPS=OFF \ && export BUILD_PYTORCH_OPS=ON \ && source /root/Open3D/util/ci_utils.sh \ diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 09ab1720141..d18e0084d94 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -195,7 +195,6 @@ build_pip_package() { cmake -DBUILD_CUDA_MODULE=OFF "${cmakeOptions[@]}" .. set +x # Echo commands off echo - make VERBOSE=1 -j"$NPROC" pybind open3d_tf_ops open3d_torch_ops echo "Packaging Open3D CPU pip package..." make VERBOSE=1 -j"$NPROC" pip-package From 52206a567014faf5b60d5b5e90e663b472c6c1b0 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Fri, 13 Oct 2023 11:10:03 -0700 Subject: [PATCH 18/19] Move BUILD_TF/TORCH_OPS settings to github actions file Enable users to build wheel with tfops locally with docker container. --- .github/workflows/ubuntu-wheel.yml | 3 +++ docker/Dockerfile.wheel | 9 +++++---- docker/docker_build.sh | 4 ++++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ubuntu-wheel.yml b/.github/workflows/ubuntu-wheel.yml index a19726fa4ff..16cce0b8272 100644 --- a/.github/workflows/ubuntu-wheel.yml +++ b/.github/workflows/ubuntu-wheel.yml @@ -19,6 +19,9 @@ concurrency: env: GCE_CLI_GHA_VERSION: '416.0.0' # Fixed to avoid dependency on API changes + BUILD_CUDA_MODULE: 'ON' + BUILD_PYTORCH_OPS: 'ON' + BUILD_TENSORFLOW_OPS: 'OFF' # Turn ON when cxx11_abi is same for TF and PyTorch jobs: build-wheel: diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index e1c184dc202..9b9fac01935 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -8,14 +8,18 @@ ARG CCACHE_TAR_NAME ARG CMAKE_VERSION ARG CCACHE_VERSION ARG PYTHON_VERSION +ARG BUILD_TENSORFLOW_OPS +ARG BUILD_PYTORCH_OPS # Forward all ARG to ENV # ci_utils.sh requires these environment variables ENV DEVELOPER_BUILD=${DEVELOPER_BUILD} ENV CCACHE_TAR_NAME=${CCACHE_TAR_NAME} ENV CMAKE_VERSION=${CMAKE_VERSION} -ARG CCACHE_VERSION=${CCACHE_VERSION} +ENV CCACHE_VERSION=${CCACHE_VERSION} ENV PYTHON_VERSION=${PYTHON_VERSION} +ENV BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS} +ENV BUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS} # Prevent interactive inputs when installing packages ENV DEBIAN_FRONTEND=noninteractive @@ -123,11 +127,8 @@ COPY . /root/Open3D WORKDIR /root/Open3D # Build python wheel - # TODO: re-enable tensorflow support, off due to due to cxx11_abi issue with PyTorch RUN export NPROC=$(nproc) \ && export BUILD_SHARED_LIBS=OFF \ - && export BUILD_TENSORFLOW_OPS=OFF \ - && export BUILD_PYTORCH_OPS=ON \ && source /root/Open3D/util/ci_utils.sh \ && build_pip_package build_azure_kinect build_jupyter diff --git a/docker/docker_build.sh b/docker/docker_build.sh index 15be9ef7f6a..b0b405d9de2 100755 --- a/docker/docker_build.sh +++ b/docker/docker_build.sh @@ -193,6 +193,8 @@ cuda_wheel_build() { fi echo "[cuda_wheel_build()] PYTHON_VERSION: ${PYTHON_VERSION}" echo "[cuda_wheel_build()] DEVELOPER_BUILD: ${DEVELOPER_BUILD}" + echo "[cuda_wheel_build()] BUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS:?'env var must be set.'}" + echo "[cuda_wheel_build()] BUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS:?'env var must be set.'}" pushd "${HOST_OPEN3D_ROOT}" docker build \ @@ -203,6 +205,8 @@ cuda_wheel_build() { --build-arg CMAKE_VERSION="${CMAKE_VERSION}" \ --build-arg CCACHE_VERSION="${CCACHE_VERSION}" \ --build-arg PYTHON_VERSION="${PYTHON_VERSION}" \ + --build-arg BUILD_TENSORFLOW_OPS="${BUILD_TENSORFLOW_OPS}" \ + --build-arg BUILD_PYTORCH_OPS="${BUILD_PYTORCH_OPS}" \ -t open3d-ci:wheel \ -f docker/Dockerfile.wheel . popd From 8af2f8fbcb5c1c6d11e5d1ea6c813479e95318a6 Mon Sep 17 00:00:00 2001 From: Sameer Sheorey Date: Fri, 13 Oct 2023 13:18:49 -0700 Subject: [PATCH 19/19] Set cxx11_abi based on tf / torch ops in ci_utls.sh --- docs/docker.in.rst | 3 +++ util/ci_utils.sh | 24 +++++++++++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/docs/docker.in.rst b/docs/docker.in.rst index 25714d0f5d8..3647b99d48f 100644 --- a/docs/docker.in.rst +++ b/docs/docker.in.rst @@ -204,3 +204,6 @@ sample configuration options available are shown below. Run ``./docker_build.sh`` without arguments to get a list of all available build configurations. + +.. note:: You can control support for PyTorch and Tensorflow with environment variables: + `BUILD_PYTORCH_OPS=ON` and `BUILD_TENSORFLOW_OPS=ON` diff --git a/util/ci_utils.sh b/util/ci_utils.sh index d18e0084d94..3d9416d1f46 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -29,7 +29,6 @@ TENSORFLOW_VER="2.13.0" TORCH_VER="2.0.1" TORCH_CPU_GLNX_VER="${TORCH_VER}+cpu" TORCH_CUDA_GLNX_VER="${TORCH_VER}+cu117" # match CUDA_VERSION in docker/docker_build.sh -PYTHON_VER=$(python -c 'import sys; ver=f"{sys.version_info.major}{sys.version_info.minor}"; print(f"cp{ver}-cp{ver}{sys.abiflags}")' 2>/dev/null || true) TORCH_MACOS_VER="${TORCH_VER}" TORCH_REPO_URL="https://download.pytorch.org/whl/torch/" # Python @@ -71,14 +70,14 @@ install_python_dependencies() { if [ "$BUILD_TENSORFLOW_OPS" == "ON" ]; then # TF happily installs both CPU and GPU versions at the same time, so remove the other python -m pip uninstall --yes "$TF_ARCH_DISABLE_NAME" - python -m pip install -U "$TF_ARCH_NAME"=="$TENSORFLOW_VER" + python -m pip install -U "$TF_ARCH_NAME"=="$TENSORFLOW_VER" # ML/requirements-tensorflow.txt fi - if [ "$BUILD_PYTORCH_OPS" == "ON" ]; then + if [ "$BUILD_PYTORCH_OPS" == "ON" ]; then # ML/requirements-torch.txt if [[ "$OSTYPE" == "linux-gnu"* ]]; then - python -m pip install -U "${TORCH_GLNX}" -f "$TORCH_REPO_URL" + python -m pip install -U "${TORCH_GLNX}" -f "$TORCH_REPO_URL" tensorboard elif [[ "$OSTYPE" == "darwin"* ]]; then - python -m pip install -U torch=="$TORCH_MACOS_VER" -f "$TORCH_REPO_URL" + python -m pip install -U torch=="$TORCH_MACOS_VER" -f "$TORCH_REPO_URL" tensorboard else echo "unknown OS $OSTYPE" exit 1 @@ -106,7 +105,7 @@ build_all() { cd build cmakeOptions=( - -DDEVELOPER_BUILD=$DEVELOPER_BUILD + -DDEVELOPER_BUILD="$DEVELOPER_BUILD" -DBUILD_SHARED_LIBS="$BUILD_SHARED_LIBS" -DCMAKE_BUILD_TYPE=Release -DBUILD_LIBREALSENSE=ON @@ -169,6 +168,13 @@ build_pip_package() { echo "Jupyter extension disabled in Python wheel." BUILD_JUPYTER_EXTENSION=OFF fi + CXX11_ABI=ON + if [ "$BUILD_TENSORFLOW_OPS" == "ON" ]; then + CXX11_ABI=$(python -c "import tensorflow as tf; print('ON' if tf.__cxx11_abi_flag__ else 'OFF')") + elif [ "$BUILD_PYTORCH_OPS" == "ON" ]; then + CXX11_ABI=$(python -c "import torch; print('ON' if torch._C._GLIBCXX_USE_CXX11_ABI else 'OFF')") + fi + echo Building with GLIBCXX_USE_CXX11_ABI="$CXX11_ABI" set -u echo @@ -179,8 +185,8 @@ build_pip_package() { "-DDEVELOPER_BUILD=$DEVELOPER_BUILD" "-DBUILD_COMMON_ISPC_ISAS=ON" "-DBUILD_AZURE_KINECT=$BUILD_AZURE_KINECT" - "-DBUILD_LIBREALSENSE=ON" - "-DGLIBCXX_USE_CXX11_ABI=OFF" + "-DBUILD_LIBREALSENSE=OFF" + "-DGLIBCXX_USE_CXX11_ABI=$CXX11_ABI" "-DBUILD_TENSORFLOW_OPS=$BUILD_TENSORFLOW_OPS" "-DBUILD_PYTORCH_OPS=$BUILD_PYTORCH_OPS" "-DBUILD_FILAMENT_FROM_SOURCE=$BUILD_FILAMENT_FROM_SOURCE" @@ -282,7 +288,7 @@ run_python_tests() { python -m pip install -U -r python/requirements_test.txt echo Add --randomly-seed=SEED to the test command to reproduce test order. pytest_args=("$OPEN3D_SOURCE_ROOT"/python/test/) - if [ "$BUILD_PYTORCH_OPS" == "OFF" ] || [ "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then + if [ "$BUILD_PYTORCH_OPS" == "OFF" ] && [ "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then echo Testing ML Ops disabled pytest_args+=(--ignore "$OPEN3D_SOURCE_ROOT"/python/test/ml_ops/) fi