Issue with pcl library while executing catkin_make- vtkFiltersParallelDIY2_AutoInit_Construct

[ashwinjosephk]

Hi,
I was following the instructions to setup the tool using the instructions on https://koide3.github.io/direct_visual_lidar_calibration/installation/

Environment: Ubuntu 20.04 + ROS Noetic

After dealing successfully with the dependencies setup, I went ahead with the catkin_make command for direct_visual_lidar_calibration. I am getting some pcl library related undefined reference issues. Any help regarding how to fix it would be very helpful. Thank you.
My log looks like:

.
.
/usr/bin/ld: /usr/local/lib/libpcl_surface.so: undefined reference to `vtkFiltersParallelDIY2_AutoInit_Construct()'
/usr/bin/ld: /usr/local/lib/libpcl_surface.so: undefined reference to `vtkFiltersParallelDIY2_AutoInit_Destruct()'
collect2: error: ld returned 1 exit status
make[2]: *** [direct_visual_lidar_calibration/CMakeFiles/preprocess.dir/build.make:352: /home/knadmin/Ashwin/calib_ws/devel/lib/direct_visual_lidar_calibration/preprocess] Error 1
make[1]: *** [CMakeFiles/Makefile2:1372: direct_visual_lidar_calibration/CMakeFiles/preprocess.dir/all] Error 2
make[1]: *** Waiting for unfinished jobs....
[ 87%] Linking CXX executable /home/knadmin/Ashwin/calib_ws/devel/lib/direct_visual_lidar_calibration/viewer
/usr/bin/ld: /usr/local/lib/libpcl_surface.so: undefined reference to `vtkFiltersParallelDIY2_AutoInit_Construct()'
/usr/bin/ld: /usr/local/lib/libpcl_surface.so: undefined reference to `vtkFiltersParallelDIY2_AutoInit_Destruct()'
collect2: error: ld returned 1 exit status
make[2]: *** [direct_visual_lidar_calibration/CMakeFiles/viewer.dir/build.make:273: /home/knadmin/Ashwin/calib_ws/devel/lib/direct_visual_lidar_calibration/viewer] Error 1
make[1]: *** [CMakeFiles/Makefile2:1502: direct_visual_lidar_calibration/CMakeFiles/viewer.dir/all] Error 2
[ 90%] Linking CXX executable /home/knadmin/Ashwin/calib_ws/devel/lib/direct_visual_lidar_calibration/initial_guess_manual
/usr/bin/ld: /usr/local/lib/libpcl_surface.so: undefined reference to `vtkFiltersParallelDIY2_AutoInit_Construct()'
/usr/bin/ld: /usr/local/lib/libpcl_surface.so: undefined reference to `vtkFiltersParallelDIY2_AutoInit_Destruct()'
collect2: error: ld returned 1 exit status
make[2]: *** [direct_visual_lidar_calibration/CMakeFiles/initial_guess_manual.dir/build.make:273: /home/knadmin/Ashwin/calib_ws/devel/lib/direct_visual_lidar_calibration/initial_guess_manual] Error 1
make[1]: *** [CMakeFiles/Makefile2:1424: direct_visual_lidar_calibration/CMakeFiles/initial_guess_manual.dir/all] Error 2
[ 93%] Linking CXX executable /home/knadmin/Ashwin/calib_ws/devel/lib/direct_visual_lidar_calibration/calibrate
/usr/bin/ld: /usr/local/lib/libpcl_surface.so: undefined reference to `vtkFiltersParallelDIY2_AutoInit_Construct()'
/usr/bin/ld: /usr/local/lib/libpcl_surface.so: undefined reference to `vtkFiltersParallelDIY2_AutoInit_Destruct()'
collect2: error: ld returned 1 exit status
make[2]: *** [direct_visual_lidar_calibration/CMakeFiles/calibrate.dir/build.make:273: /home/knadmin/Ashwin/calib_ws/devel/lib/direct_visual_lidar_calibration/calibrate] Error 1
make[1]: *** [CMakeFiles/Makefile2:1476: direct_visual_lidar_calibration/CMakeFiles/calibrate.dir/all] Error 2
[ 96%] Linking CXX executable /home/knadmin/Ashwin/calib_ws/devel/lib/direct_visual_lidar_calibration/initial_guess_auto
/usr/bin/ld: /usr/local/lib/libpcl_surface.so: undefined reference to `vtkFiltersParallelDIY2_AutoInit_Construct()'
/usr/bin/ld: /usr/local/lib/libpcl_surface.so: undefined reference to `vtkFiltersParallelDIY2_AutoInit_Destruct()'
collect2: error: ld returned 1 exit status
make[2]: *** [direct_visual_lidar_calibration/CMakeFiles/initial_guess_auto.dir/build.make:273: /home/knadmin/Ashwin/calib_ws/devel/lib/direct_visual_lidar_calibration/initial_guess_auto] Error 1
make[1]: *** [CMakeFiles/Makefile2:1450: direct_visual_lidar_calibration/CMakeFiles/initial_guess_auto.dir/all] Error 2
[100%] Linking CXX executable /home/knadmin/Ashwin/calib_ws/devel/lib/direct_visual_lidar_calibration/preprocess_map
/usr/bin/ld: warning: libpcl_io_ply.so.1.14, needed by /usr/local/lib/libpcl_io.so, may conflict with libpcl_io_ply.so.1.10
[100%] Built target preprocess_map
make: *** [Makefile:146: all] Error 2
Invoking "make -j6" failed

(venv3) ashwin@KN-Precision-7540:~/Ashwin/calib_ws$ 

My CMakelists looks like:

cmake_minimum_required(VERSION 3.16)
project(direct_visual_lidar_calibration)

add_compile_options(-std=c++17)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_LIST_DIR}/cmake")

if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
  set(CMAKE_BUILD_TYPE "RelWithDebInfo" CACHE STRING "Choose the type of build." FORCE)
  set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
endif()

if($ENV{ROS_VERSION} EQUAL 1)
  # ROS1
  find_package(catkin REQUIRED COMPONENTS rosbag sensor_msgs cv_bridge)
  catkin_package()
else()
  # ROS2
  find_package(ament_cmake_auto REQUIRED)
  find_package(ament_cmake_python REQUIRED)
  ament_auto_find_build_dependencies()
endif()

find_package(PCL REQUIRED)
include_directories(/usr/local/include/pcl-1.14)
find_package(Ceres REQUIRED)
find_package(GTSAM REQUIRED)
find_package(OpenCV REQUIRED)
find_package(Boost REQUIRED COMPONENTS filesystem program_options)
find_package(Iridescence REQUIRED)

#set(PCL_INCLUDE_DIRS "/home/knadmin/Ashwin/pcl/build/include/pcl")
#set(PCL_LIBRARIES "/home/knadmin/Ashwin/pcl/build/lib/libpcl_common.so") # Adjust based on actual libraries installed
set(PCL_INCLUDE_DIRS "/usr/local/include/pcl-1.14/pcl")

set(PCL_LIBRARIES "/usr/local/lib/libpcl_common.so;/usr/local/lib/libpcl_search.so;/usr/local/lib/libpcl_io_ply.so;/usr/local/lib/libpcl_io.so;/usr/local/lib/libpcl_surface.so")




set(OLD_DISTRO "humble;galactic;foxy")
if($ENV{ROS_DISTRO} IN_LIST OLD_DISTRO)
  add_definitions(-DCV_BRIDGE_INCLUDE_H)
else()
  add_definitions(-DCV_BRIDGE_INCLUDE_HPP)
endif()

find_package(OpenMP)
if (OPENMP_FOUND)
  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
endif()

# direct_visual_lidar_calibration
add_library(direct_visual_lidar_calibration SHARED
  src/camera/create_camera.cpp
  src/vlcal/common/frame.cpp
  src/vlcal/common/frame_cpu.cpp
  src/vlcal/common/ivox.cpp
  src/vlcal/common/cloud_covariance_estimation.cpp
  src/vlcal/common/time_keeper.cpp
  src/vlcal/common/estimate_fov.cpp
  src/vlcal/common/estimate_pose.cpp
  src/vlcal/common/points_color_updater.cpp
  src/vlcal/common/visual_lidar_data.cpp
  src/vlcal/common/visual_lidar_visualizer.cpp
  src/vlcal/preprocess/preprocess.cpp
  src/vlcal/preprocess/generate_lidar_image.cpp
  src/vlcal/preprocess/static_point_cloud_integrator.cpp
  src/vlcal/preprocess/dynamic_point_cloud_integrator.cpp
  src/vlcal/calib/cost_calculator_nid.cpp
  src/vlcal/calib/view_culling.cpp
  src/vlcal/calib/visual_camera_calibration.cpp
)
target_include_directories(direct_visual_lidar_calibration PUBLIC
  include
  thirdparty/json/include
  thirdparty/Bonxai/include
  thirdparty/nanoflann/include
  thirdparty/Sophus
  ${Boost_INCLUDE_DIRS}
  ${CERES_INCLUDE_DIRS}
  ${GTSAM_INCLUDE_DIRS}
  ${PCL_INCLUDE_DIRS}
  ${OpenCV_INCLUDE_DIRS}
)
target_link_libraries(direct_visual_lidar_calibration
  fmt
  ${Boost_LIBRARIES}
  ${PCL_LIBRARIES}
  ${CERES_LIBRARIES}
  ${GTSAM_LIBRARIES}
  ${OpenCV_LIBRARIES}
  ${PCL_LIBRARIES}
  Iridescence::Iridescence
  pcl_common
  pcl_search
  pcl_filters
  pcl_surface
  pcl_io
  pcl_io_ply   # Link specific I/O module for PLY

)


if($ENV{ROS_VERSION} EQUAL 1)
  # ROS1
  add_executable(preprocess
    src/preprocess_ros1.cpp
  )
  target_link_libraries(preprocess
    direct_visual_lidar_calibration
  )
  target_include_directories(preprocess PUBLIC
    ${catkin_INCLUDE_DIRS}
  )
  target_link_libraries(preprocess
    ${catkin_LIBRARIES}
  )
else()
  # ROS2
  ament_auto_add_executable(preprocess
    src/preprocess_ros2.cpp
  )
  target_link_libraries(preprocess
    direct_visual_lidar_calibration
  )
endif()

# preprocess_map
add_executable(preprocess_map
  src/preprocess_map.cpp
)
target_include_directories(preprocess_map PRIVATE
  ${catkin_INCLUDE_DIRS}
)
target_link_libraries(preprocess_map
  direct_visual_lidar_calibration
)

# initial_guess_manual
add_executable(initial_guess_manual
  src/initial_guess_manual.cpp
)
target_include_directories(initial_guess_manual PRIVATE
  ${catkin_INCLUDE_DIRS}
)
target_link_libraries(initial_guess_manual
  direct_visual_lidar_calibration
)

# initial_guess_auto
add_executable(initial_guess_auto
  src/initial_guess_auto.cpp
)
target_include_directories(initial_guess_auto PRIVATE
  ${catkin_INCLUDE_DIRS}
)
target_link_libraries(initial_guess_auto
  direct_visual_lidar_calibration
)

# calibrate
add_executable(calibrate
  src/calibrate.cpp
)
target_include_directories(calibrate PRIVATE
  ${catkin_INCLUDE_DIRS}
)
target_link_libraries(calibrate
  direct_visual_lidar_calibration
)

# viewer
add_executable(viewer
  src/viewer.cpp
)
target_include_directories(viewer PRIVATE
  ${catkin_INCLUDE_DIRS}
)
target_link_libraries(viewer
  direct_visual_lidar_calibration
)

install(
  TARGETS
  direct_visual_lidar_calibration
)

install(
  TARGETS
  preprocess_map
  initial_guess_manual
  initial_guess_auto
  calibrate
  viewer
  DESTINATION
  lib/${PROJECT_NAME}
)

if($ENV{ROS_VERSION} EQUAL 1)
  # ROS1
else()
  # ROS2
  ament_python_install_package(${PROJECT_NAME})
  install(PROGRAMS scripts/find_matches_superglue.py DESTINATION lib/${PROJECT_NAME})
  ament_export_libraries(direct_visual_lidar_calibration)
  ament_auto_package()
endif()

[koide3]

What is the version of PCL, and how you installed it (via apt?)?

[harisbinyousaf24]

Its great to use this calibration tool @koide3. I successfully ran the tool using docker image and tested with my own data somehow. But i was trying to build this tool in my local system mentioned in the installation guide as mentioned by @ashwinjoseph95 and in the end step when i did catkin_make i got something like this:

Just wondering is it related to my older version of pcl which is 1.9. Which version of PCL is required by DVLC?

[koide3]

Yes, you need a PCL 1.2 or greater.

[harisbinyousaf24]

Thanks @koide3. I worked it out with pcl 1.9 and by making some changes in the scripts estimate_fov.cpp and preprocess_map.cpp. The package built successfully. The preprocessing worked but if i activate --visualize flag i get this new thing Operation aborted/ Segmentation fault core dumped. Do i need to have GPU for the visualization?
Also would have been more easier if open3d is used as a visualization tool or some light library for non-GPU users (in my opinion). Because i had to pass initial guess manually via calib.json file to avoid point selection process which gets real stucky because i have no GPU.
Other than that really appreciate your work. @koide3

[koide3]

The visualizer uses OpenGL that should work on non-nvidia integrated GPUs as well (e.g., ones on Intel CPUs and Raspberry Pi). Can you tell me how it got crashed?

[harisbinyousaf24]

Yeah sure @koide3 I am attaching a video in which i recorded my screen to run the tool:

error.mp4

In this video if you see when i try to run this command:
rosrun direct_visual_lidar_calibration preprocess --image_topic /Camera_8_synced/image_raw --points_topic /innovusion_lidar_front --camera_model plumb_bob --camera_intrinsic 1170.335979672321,1168.528989041779,957.0745338763222,543.629582572989 --camera_distortion_coeffs -0.3487691099377431,0.1406639011462764,0.0004765741803053816,0.0007804555751864781,-0.03061449907047344 /home/ty/Downloads/DATA/DVLC/data /home/ty/Downloads/DATA/DVLC/preprocessed --visualize

I get the Segmentation Fault Core dumped as if something is revoking to open the visualizer. Although Camera info topic is not available, i am providing intrinsics and distortion coefficients separately. It works fine if i remove the --visualize flag from the end.

My System: Ubuntu 20.04, ROS Noetic
Processor: Intel® Core™ i5-7200U CPU @ 2.50GHz × 4 ;
Graphics: Mesa Intel® HD Graphics 620 (KBL GT2)

Built this repo from source.
Below is snapshot of my terminal from preprocess(without visualize) and manual initial guess step:

Upon backtracing the error via gdb, i get this with the help of GPT:

The backtrace shows the exact source of the double free error. It's occurring in the VisualLiDARData constructor when trying to handle PLY data. Specifically:

1. The error is in visual_lidar_data.cpp line 19
2. It's related to a shared pointer (std::shared_ptr) managing a glk::PLYData object
3. The problem occurs during memory deallocation (free) suggesting the same memory is being freed twice