From e88c7b13270d961393ab1e64c03d1f7ccd047cce Mon Sep 17 00:00:00 2001 From: Sameer Sheorey <41028320+ssheorey@users.noreply.github.com> Date: Sun, 29 Sep 2024 23:07:07 -0700 Subject: [PATCH] Replace conda with pyenv to fix incorrect libstdc++ use in jammy CI. (#6966) * Replace conda with pyenv to fix incorrect libstdc++ use in jammy CI. * Use RPATH instead of RUNPATH to load libc++abi.so directly in Python. No need to find and load explicitly. * Use libc++11 to build in Ubuntu 22.04. Warn if newer version is used. * TODO: Solution for Ubuntu 24.04 --- 3rdparty/find_dependencies.cmake | 9 ++++++ cpp/open3d/core/nns/NanoFlannImpl.h | 25 ++++------------ cpp/pybind/CMakeLists.txt | 3 ++ cpp/tests/t/geometry/TensorMap.cpp | 7 +++++ docker/Dockerfile.ci | 44 +++++++++++++++++++---------- docker/docker_test.sh | 2 +- python/open3d/__init__.py | 8 ------ util/install_deps_ubuntu.sh | 11 ++++++-- 8 files changed, 63 insertions(+), 46 deletions(-) diff --git a/3rdparty/find_dependencies.cmake b/3rdparty/find_dependencies.cmake index 77b7085df69..d91377a3138 100644 --- a/3rdparty/find_dependencies.cmake +++ b/3rdparty/find_dependencies.cmake @@ -1338,6 +1338,7 @@ if(BUILD_GUI) if (CPP_LIBRARY AND CPPABI_LIBRARY) set(CLANG_LIBDIR ${llvm_lib_dir}) message(STATUS "CLANG_LIBDIR found in ubuntu-default: ${CLANG_LIBDIR}") + set(LIBCPP_VERSION ${llvm_ver}) break() endif() endforeach() @@ -1362,7 +1363,10 @@ if(BUILD_GUI) llvm-8/lib llvm-7/lib ) + file(REAL_PATH ${CPPABI_LIBRARY} CPPABI_LIBRARY) get_filename_component(CLANG_LIBDIR ${CPPABI_LIBRARY} DIRECTORY) + string(REGEX MATCH "llvm-([0-9]+)/lib" _ ${CLANG_LIBDIR}) + set(LIBCPP_VERSION ${CMAKE_MATCH_1}) endif() # Find clang libraries at the exact path ${CLANG_LIBDIR}. @@ -1378,6 +1382,11 @@ if(BUILD_GUI) target_link_libraries(3rdparty_filament INTERFACE -lstdc++ ${CPP_LIBRARY} ${CPPABI_LIBRARY}) message(STATUS "Filament C++ libraries: ${CPP_LIBRARY} ${CPPABI_LIBRARY}") + if (LIBCPP_VERSION GREATER 11) + message(WARNING "libc++ (LLVM) version ${LIBCPP_VERSION} > 11 includes libunwind that " + "interferes with the system libunwind.so.8 and may crash Python code when exceptions " + "are used. Please consider using libc++ (LLVM) v11.") + endif() endif() if (APPLE) find_library(CORE_VIDEO CoreVideo) diff --git a/cpp/open3d/core/nns/NanoFlannImpl.h b/cpp/open3d/core/nns/NanoFlannImpl.h index 027d6d0c4ee..9a9ceb100c4 100644 --- a/cpp/open3d/core/nns/NanoFlannImpl.h +++ b/cpp/open3d/core/nns/NanoFlannImpl.h @@ -118,13 +118,6 @@ void _KnnSearchCPU(NanoFlannIndexHolderBase *holder, return; } - auto points_equal = [](const T *const p1, const T *const p2, - size_t dimension) { - std::vector p1_vec(p1, p1 + dimension); - std::vector p2_vec(p2, p2 + dimension); - return p1_vec == p2_vec; - }; - std::vector> neighbors_indices(num_queries); std::vector> neighbors_distances(num_queries); std::vector neighbors_count(num_queries, 0); @@ -147,8 +140,9 @@ void _KnnSearchCPU(NanoFlannIndexHolderBase *holder, for (size_t valid_i = 0; valid_i < num_valid; ++valid_i) { TIndex idx = result_indices[valid_i]; if (ignore_query_point && - points_equal(&queries[i * dimension], - &points[idx * dimension], dimension)) { + std::equal(&queries[i * dimension], + &queries[i * dimension] + dimension, + &points[idx * dimension])) { continue; } neighbors_indices[i].push_back(idx); @@ -222,13 +216,6 @@ void _RadiusSearchCPU(NanoFlannIndexHolderBase *holder, return; } - auto points_equal = [](const T *const p1, const T *const p2, - size_t dimension) { - std::vector p1_vec(p1, p1 + dimension); - std::vector p2_vec(p2, p2 + dimension); - return p1_vec == p2_vec; - }; - std::vector> neighbors_indices(num_queries); std::vector> neighbors_distances(num_queries); std::vector neighbors_count(num_queries, 0); @@ -255,9 +242,9 @@ void _RadiusSearchCPU(NanoFlannIndexHolderBase *holder, int num_neighbors = 0; for (const auto &idx_dist : search_result) { if (ignore_query_point && - points_equal(&queries[i * dimension], - &points[idx_dist.first * dimension], - dimension)) { + std::equal(&queries[i * dimension], + &queries[i * dimension] + dimension, + &points[idx_dist.first * dimension])) { continue; } neighbors_indices[i].push_back(idx_dist.first); diff --git a/cpp/pybind/CMakeLists.txt b/cpp/pybind/CMakeLists.txt index e7a534a3eb3..c79bbd96719 100644 --- a/cpp/pybind/CMakeLists.txt +++ b/cpp/pybind/CMakeLists.txt @@ -80,6 +80,9 @@ set(PYTHON_COMPILED_MODULE_DIR if (APPLE) set_target_properties(pybind PROPERTIES BUILD_RPATH "@loader_path;@loader_path/..") elseif (UNIX) + # Use RPATH instead of RUNPATH in pybind so that needed libc++.so can find child dependant libc++abi.so in RPATH + # https://stackoverflow.com/questions/69662319/managing-secondary-dependencies-of-shared-libraries + target_link_options(pybind PRIVATE "LINKER:--disable-new-dtags") set_target_properties(pybind PROPERTIES BUILD_RPATH "$ORIGIN;$ORIGIN/..") endif() set_target_properties(pybind PROPERTIES diff --git a/cpp/tests/t/geometry/TensorMap.cpp b/cpp/tests/t/geometry/TensorMap.cpp index 372c6b82270..8512ce68ccb 100644 --- a/cpp/tests/t/geometry/TensorMap.cpp +++ b/cpp/tests/t/geometry/TensorMap.cpp @@ -32,6 +32,13 @@ TEST_P(TensorMapPermuteDevices, Constructor) { // Primary key is required. EXPECT_ANY_THROW(t::geometry::TensorMap()); + // Delete primary key. + EXPECT_ANY_THROW(tm0.Erase("positions")); + + // Reserved keys. + EXPECT_ANY_THROW(tm0.insert( + {"primary_key", core::Tensor::Zeros({2, 3}, dtype, device)})); + // Iterators. std::map tensor_map( {{"positions", core::Tensor::Zeros({10, 3}, dtype, device)}, diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index cd52a838882..73feb7bffd6 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -67,34 +67,48 @@ RUN if [ "${BUILD_SYCL_MODULE}" = "ON" ]; then \ rm -rf /etc/apt/sources.list.d/oneAPI.list; \ fi -# Dependencies: basic +# Dependencies: basic and python-build RUN apt-get update && apt-get install -y \ git \ wget \ curl \ build-essential \ pkg-config \ + zlib1g \ + zlib1g-dev \ + libssl-dev \ + libbz2-dev \ + libreadline-dev \ + libsqlite3-dev \ + libncursesw5-dev \ + xz-utils \ + tk-dev \ + libxml2-dev \ + libxmlsec1-dev \ + libffi-dev \ + liblzma-dev \ && rm -rf /var/lib/apt/lists/* -# Miniconda or Intel conda -# The **/open3d/bin paths are used during docker run, in this way docker run +# pyenv or Intel Python +# The pyenv python paths are used during docker run, in this way docker run # does not need to activate the environment again. -ENV PATH="/root/miniconda3/bin:${PATH}" -ENV PATH="/root/miniconda3/envs/open3d/bin:${PATH}" +# The soft link from the python patch level version to the python mino version +# ensures python wheel commands (i.e. open3d) are in PATH, since we don't know +# which patch level pyenv will install (latest). +ENV PYENV_ROOT=/root/.pyenv +ENV PATH="$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PYENV_ROOT/versions/$PYTHON_VERSION/bin:$PATH" ENV PATH="/opt/intel/oneapi/intelpython/latest/bin:${PATH}" -ENV PATH="/opt/intel/oneapi/intelpython/latest/envs/open3d/bin:${PATH}" RUN if [ "${BUILD_SYCL_MODULE}" = "OFF" ]; then \ - wget -q https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh; \ - bash Miniconda3-latest-Linux-x86_64.sh -b; \ - rm Miniconda3-latest-Linux-x86_64.sh; \ + curl https://pyenv.run | bash \ + && pyenv update \ + && pyenv install $PYTHON_VERSION \ + && pyenv global $PYTHON_VERSION \ + && pyenv rehash \ + && ln -s $PYENV_ROOT/versions/${PYTHON_VERSION}* $PYENV_ROOT/versions/${PYTHON_VERSION}; \ fi -RUN conda --version \ - && conda create -y -n open3d python=${PYTHON_VERSION} +RUN python --version && pip --version -# Activate open3d virtualenv -# This works during docker build. It becomes the prefix of all RUN commands. -# Ref: https://stackoverflow.com/a/60148365/1255535 -SHELL ["conda", "run", "-n", "open3d", "/bin/bash", "-o", "pipefail", "-c"] +SHELL ["/bin/bash", "-o", "pipefail", "-c"] # Dependencies: cmake ENV PATH=${HOME}/${CMAKE_VERSION}/bin:${PATH} diff --git a/docker/docker_test.sh b/docker/docker_test.sh index 441b17ed4e3..d328d14535d 100755 --- a/docker/docker_test.sh +++ b/docker/docker_test.sh @@ -141,7 +141,7 @@ cpp_python_linking_uninstall_test() { # Python test echo "pytest is randomized, add --randomly-seed=SEED to repeat the test sequence." ${docker_run} -i --rm "${DOCKER_TAG}" /bin/bash -c " \ - python -m pytest python/test ${pytest_args} -s" + python -W default -m pytest python/test ${pytest_args} -s" restart_docker_daemon_if_on_gcloud # Command-line tools test diff --git a/python/open3d/__init__.py b/python/open3d/__init__.py index 141bf426c50..f7354b2a90c 100644 --- a/python/open3d/__init__.py +++ b/python/open3d/__init__.py @@ -45,14 +45,6 @@ def load_cdll(path): if sys.platform == "win32": # Unix: Use rpath to find libraries _win32_dll_dir = os.add_dll_directory(str(Path(__file__).parent)) -if _build_config["BUILD_GUI"] and not (find_library("c++abi") or - find_library("c++")): - try: # Preload libc++.so and libc++abi.so (required by filament) - load_cdll(str(next((Path(__file__).parent).glob("*c++abi.*")))) - load_cdll(str(next((Path(__file__).parent).glob("*c++.*")))) - except StopIteration: # Not found: check system paths while loading - pass - __DEVICE_API__ = "cpu" if _build_config["BUILD_CUDA_MODULE"]: # Load CPU pybind dll gracefully without introducing new python variable. diff --git a/util/install_deps_ubuntu.sh b/util/install_deps_ubuntu.sh index 3e359a1a6e7..2786b19c23b 100755 --- a/util/install_deps_ubuntu.sh +++ b/util/install_deps_ubuntu.sh @@ -39,17 +39,22 @@ eval $( echo DISTRIB_ID="$DISTRIB_ID"; echo DISTRIB_RELEASE="$DISTRIB_RELEASE" ) +# To avoid dependence on libunwind, we don't want to use clang / libc++ versions later than 11. +# Ubuntu 20.04's has versions 8, 10 or 12 while Ubuntu 22.04 has versions 11 and later. if [ "$DISTRIB_ID" == "Ubuntu" -a "$DISTRIB_RELEASE" == "20.04" ]; then - # Ubuntu 20.04's clang/libc++-dev/libc++abi-dev are version 8, 10 or 12. - # To avoid dependence on libunwind, we don't want to use versions later than 10. deps=("${deps[@]/clang/clang-10}") deps=("${deps[@]/libc++-dev/libc++-10-dev}") deps=("${deps[@]/libc++abi-dev/libc++abi-10-dev}") fi +if [ "$DISTRIB_ID" == "Ubuntu" -a "$DISTRIB_RELEASE" == "22.04" ]; then + deps=("${deps[@]/clang/clang-11}") + deps=("${deps[@]/libc++-dev/libc++-11-dev}") + deps=("${deps[@]/libc++abi-dev/libc++abi-11-dev}") +fi # Special case for ARM64 if [ "$(uname -m)" == "aarch64" ]; then - # For compling LAPACK in OpenBLAS + # For compiling LAPACK in OpenBLAS deps+=("gfortran") fi