diff --git a/CMakeIntegratedOpenCL.cmake b/CMakeIntegratedOpenCL.cmake new file mode 100644 index 000000000000..8673f3bdead2 --- /dev/null +++ b/CMakeIntegratedOpenCL.cmake @@ -0,0 +1,85 @@ +set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) +set(BOOST_VERSION_DOT "1.74") +string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT}) + +set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git") +set(OPENCL_HEADER_TAG "1b2a1850f410aaaaeaa56cead5a179b5aea4918e") + +set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git") +set(OPENCL_LOADER_TAG "98ca71fb9f8484f1cd1999f55224bf9e8d18693b") + +set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git") +set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0") + +# Build Independent OpenCL library +include(FetchContent) +FetchContent_Declare(OpenCL-Headers GIT_REPOSITORY ${OPENCL_HEADER_REPOSITORY} GIT_TAG ${OPENCL_HEADER_TAG}) +FetchContent_GetProperties(OpenCL-Headers) +if(NOT OpenCL-Headers_POPULATED) + FetchContent_Populate(OpenCL-Headers) + message(STATUS "Populated OpenCL Headers") +endif() +set(OPENCL_ICD_LOADER_HEADERS_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for OpenCL ICD Loader +set(OpenCL_INCLUDE_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for Boost::Compute + +FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG}) +FetchContent_GetProperties(OpenCL-ICD-Loader) +if(NOT OpenCL-ICD-Loader_POPULATED) + FetchContent_Populate(OpenCL-ICD-Loader) + set(USE_DYNAMIC_VCXX_RUNTIME ON) + add_subdirectory(${opencl-icd-loader_SOURCE_DIR} ${opencl-icd-loader_BINARY_DIR} EXCLUDE_FROM_ALL) + message(STATUS "Populated OpenCL ICD Loader") +endif() +list(APPEND INTEGRATED_OPENCL_INCLUDES ${OPENCL_ICD_LOADER_HEADERS_DIR}) +list(APPEND INTEGRATED_OPENCL_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/Release/OpenCL.lib cfgmgr32.lib runtimeobject.lib) +list(APPEND INTEGRATED_OPENCL_DEFINITIONS CL_TARGET_OPENCL_VERSION=120) + +# Build Independent Boost libraries +include(ExternalProject) +include(ProcessorCount) +ProcessorCount(J) +set(BOOST_BASE "${PROJECT_BINARY_DIR}/Boost") +set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.bat") +set(BOOST_BUILD "${BOOST_BASE}/source/b2.exe") +set(BOOST_FLAGS "") +list(APPEND BOOST_SUBMODULES "libs/algorithm" "libs/align" "libs/any" "libs/array" "libs/assert" "libs/bind" "libs/chrono" "libs/compute" "libs/concept_check" "libs/config" "libs/container" "libs/container_hash" "libs/core" "libs/detail" "libs/filesystem" "libs/foreach" "libs/format" "libs/function" "libs/function_types" "libs/fusion" "libs/headers" "libs/integer" "libs/io" "libs/iterator" "libs/lexical_cast" "libs/math" "libs/move" "libs/mpl" "libs/multi_index" "libs/numeric/conversion" "libs/optional" "libs/predef" "libs/preprocessor" "libs/property_tree" "libs/range" "libs/ratio" "libs/serialization" "libs/smart_ptr" "libs/static_assert" "libs/system" "libs/throw_exception" "libs/tuple" "libs/typeof" "libs/type_index" "libs/type_traits" "libs/utility" "libs/uuid" "libs/winapi" "tools/boost_install" "tools/build") +ExternalProject_Add(Boost + TMP_DIR "${BOOST_BASE}/tmp" + STAMP_DIR "${BOOST_BASE}/stamp" + DOWNLOAD_DIR "${BOOST_BASE}/download" + SOURCE_DIR "${BOOST_BASE}/source" + BINARY_DIR "${BOOST_BASE}/source" + INSTALL_DIR "${BOOST_BASE}/install" + GIT_REPOSITORY ${BOOST_REPOSITORY} + GIT_TAG ${BOOST_TAG} + GIT_SUBMODULES ${BOOST_SUBMODULES} + GIT_SHALLOW ON + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND ${BOOST_BOOTSTRAP} + BUILD_COMMAND ${BOOST_BUILD} -sBOOST_ROOT=${BOOST_BASE}/source -a -q -j ${J} --with-headers --with-chrono --with-filesystem --with-system link=static runtime-link=shared variant=release threading=multi cxxflags="${BOOST_FLAGS}" + INSTALL_COMMAND "" +) +set(BOOST_INCLUDE "${BOOST_BASE}/source" CACHE PATH "") +set(BOOST_LIBRARY "${BOOST_BASE}/source/stage/lib" CACHE PATH "") +list(APPEND INTEGRATED_OPENCL_INCLUDES ${BOOST_INCLUDE}) +if(MSVC) + if(${MSVC_VERSION} GREATER 1929) + message(FATAL_ERROR "Unrecognized MSVC version number: ${MSVC_VERSION}") + elseif(${MSVC_VERSION} GREATER 1919) + set(MSVC_TOOLCHAIN_ID "142") + elseif(${MSVC_VERSION} GREATER 1909) + set(MSVC_TOOLCHAIN_ID "141") + elseif(${MSVC_VERSION} GREATER 1899) + set(MSVC_TOOLCHAIN_ID "140") + else() + message(FATAL_ERROR "Unrecognized MSVC version number: ${MSVC_VERSION}") + endif() + list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) + list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) + list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) +else() + message(FATAL_ERROR "Integrated OpenCL build is not yet available for MinGW") +endif() + +set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE) diff --git a/CMakeLists.txt b/CMakeLists.txt index b6cbc76b2653..04b8861b12ca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,6 @@ -if(USE_GPU OR APPLE) +if(__INTEGRATE_OPENCL) + cmake_minimum_required(VERSION 3.11) +elseif(USE_GPU OR APPLE) cmake_minimum_required(VERSION 3.2) elseif(USE_CUDA) cmake_minimum_required(VERSION 3.16) @@ -22,11 +24,18 @@ OPTION(USE_CUDA "Enable CUDA-accelerated training (EXPERIMENTAL)" OFF) OPTION(USE_DEBUG "Set to ON for Debug mode" OFF) OPTION(BUILD_STATIC_LIB "Build static library" OFF) OPTION(__BUILD_FOR_R "Set to ON if building lib_lightgbm for use with the R package" OFF) +OPTION(__INTEGRATE_OPENCL "Set to ON if building LightGBM with the OpenCL ICD Loader and its dependencies included" OFF) if(APPLE) OPTION(APPLE_OUTPUT_DYLIB "Output dylib shared library" OFF) endif(APPLE) +if(__INTEGRATE_OPENCL) + set(__INTEGRATE_OPENCL ON CACHE BOOL "" FORCE) + set(USE_GPU OFF CACHE BOOL "" FORCE) + message(STATUS "Building library with integrated OpenCL components") +endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.8.2") message(FATAL_ERROR "Insufficient gcc version") @@ -134,6 +143,15 @@ if(USE_GPU) ADD_DEFINITIONS(-DUSE_GPU) endif(USE_GPU) +if(__INTEGRATE_OPENCL) + if(WIN32) + include(CMakeIntegratedOpenCL.cmake) + ADD_DEFINITIONS(-DUSE_GPU) + else() + message(FATAL_ERROR "Integrated OpenCL build is available only for Windows") + endif(WIN32) +endif(__INTEGRATE_OPENCL) + if(USE_CUDA) find_package(CUDA REQUIRED) include_directories(${CUDA_INCLUDE_DIRS}) @@ -321,7 +339,7 @@ else() endif(BUILD_STATIC_LIB) if(MSVC) - set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm") + set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm") endif(MSVC) if(USE_SWIG) @@ -378,6 +396,19 @@ if(USE_GPU) TARGET_LINK_LIBRARIES(_lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES}) endif(USE_GPU) +if(__INTEGRATE_OPENCL) + # targets OpenCL and Boost are added in CMakeIntegratedOpenCL.cmake + add_dependencies(lightgbm OpenCL Boost) + add_dependencies(_lightgbm OpenCL Boost) + # variables INTEGRATED_OPENCL_* are set in CMakeIntegratedOpenCL.cmake + target_include_directories(lightgbm PRIVATE ${INTEGRATED_OPENCL_INCLUDES}) + target_include_directories(_lightgbm PRIVATE ${INTEGRATED_OPENCL_INCLUDES}) + target_compile_definitions(lightgbm PRIVATE ${INTEGRATED_OPENCL_DEFINITIONS}) + target_compile_definitions(_lightgbm PRIVATE ${INTEGRATED_OPENCL_DEFINITIONS}) + target_link_libraries(lightgbm PRIVATE ${INTEGRATED_OPENCL_LIBRARIES}) + target_link_libraries(_lightgbm PRIVATE ${INTEGRATED_OPENCL_LIBRARIES}) +endif() + if(USE_CUDA) set_target_properties(lightgbm PROPERTIES CUDA_RESOLVE_DEVICE_SYMBOLS ON) TARGET_LINK_LIBRARIES( @@ -396,12 +427,14 @@ if(USE_HDFS) TARGET_LINK_LIBRARIES(_lightgbm ${HDFS_CXX_LIBRARIES}) endif(USE_HDFS) -if(WIN32 AND (MINGW OR CYGWIN)) - TARGET_LINK_LIBRARIES(lightgbm Ws2_32) - TARGET_LINK_LIBRARIES(_lightgbm Ws2_32) - TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI) - TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI) -endif() +if(WIN32) + if(MINGW OR CYGWIN) + TARGET_LINK_LIBRARIES(lightgbm Ws2_32) + TARGET_LINK_LIBRARIES(_lightgbm Ws2_32) + TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI) + TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI) + endif(MINGW OR CYGWIN) +endif(WIN32) if(__BUILD_FOR_R) if(MSVC) diff --git a/python-package/setup.py b/python-package/setup.py index 1faffdd1e53c..4e3242fe2191 100644 --- a/python-package/setup.py +++ b/python-package/setup.py @@ -29,7 +29,7 @@ def find_lib(): return LIB_PATH -def copy_files(use_gpu=False): +def copy_files(integrated_opencl=False, use_gpu=False): def copy_files_helper(folder_name): src = os.path.join(CURRENT_DIR, os.path.pardir, folder_name) @@ -51,14 +51,18 @@ def copy_files_helper(folder_name): copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.vcxproj"), os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.vcxproj"), verbose=0) - if use_gpu: - copy_files_helper('compute') - copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"), - os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"), - verbose=0) copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "LICENSE"), os.path.join(CURRENT_DIR, "LICENSE"), verbose=0) + copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"), + os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"), + verbose=0) + if integrated_opencl: + copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeIntegratedOpenCL.cmake"), + os.path.join(CURRENT_DIR, "compile", "CMakeIntegratedOpenCL.cmake"), + verbose=0) + if use_gpu: + copy_files_helper('compute') def clear_path(path): @@ -91,7 +95,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_cuda=False, use_mpi=False, use_hdfs=False, boost_root=None, boost_dir=None, boost_include_dir=None, boost_librarydir=None, opencl_include_dir=None, opencl_library=None, - nomp=False, bit32=False): + nomp=False, bit32=False, integrated_opencl=False): if os.path.exists(os.path.join(CURRENT_DIR, "build_cpp")): shutil.rmtree(os.path.join(CURRENT_DIR, "build_cpp")) @@ -101,6 +105,9 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_cuda=False, use_mpi=False, logger.info("Starting to compile the library.") cmake_cmd = ["cmake", "../compile/"] + if integrated_opencl: + use_gpu = False + cmake_cmd.append("-D__INTEGRATE_OPENCL=ON") if use_gpu: cmake_cmd.append("-DUSE_GPU=ON") if boost_root: @@ -136,7 +143,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_cuda=False, use_mpi=False, else: status = 1 lib_path = os.path.join(CURRENT_DIR, "compile", "windows", "x64", "DLL", "lib_lightgbm.dll") - if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32)): + if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32, integrated_opencl)): logger.info("Starting to compile with MSBuild from existing solution file.") platform_toolsets = ("v142", "v141", "v140") for pt in platform_toolsets: @@ -189,6 +196,7 @@ class CustomInstall(install): user_options = install.user_options + [ ('mingw', 'm', 'Compile with MinGW'), + ('integrated-opencl', None, 'Compile integrated OpenCL version'), ('gpu', 'g', 'Compile GPU version'), ('cuda', None, 'Compile CUDA version'), ('mpi', None, 'Compile MPI version'), @@ -207,6 +215,7 @@ class CustomInstall(install): def initialize_options(self): install.initialize_options(self) self.mingw = 0 + self.integrated_opencl = 0 self.gpu = 0 self.cuda = 0 self.boost_root = None @@ -231,12 +240,12 @@ def run(self): "please use 64-bit Python instead.") open(LOG_PATH, 'wb').close() if not self.precompile: - copy_files(use_gpu=self.gpu) + copy_files(integrated_opencl=self.integrated_opencl, use_gpu=self.gpu) compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_cuda=self.cuda, use_mpi=self.mpi, use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir, boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir, opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library, - nomp=self.nomp, bit32=self.bit32) + nomp=self.nomp, bit32=self.bit32, integrated_opencl=self.integrated_opencl) install.run(self) if os.path.isfile(LOG_PATH): os.remove(LOG_PATH) @@ -245,7 +254,7 @@ def run(self): class CustomSdist(sdist): def run(self): - copy_files(use_gpu=True) + copy_files(integrated_opencl=True, use_gpu=True) open(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'), 'w').close() if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')): shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'Release'))