From b1841bd598ad0e609c4703c70aadbd2cb0160904 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Wed, 27 May 2020 23:14:50 +0000 Subject: [PATCH 01/22] Add specialized OpenCL/Python package build path --- CMakeLists.txt | 62 +++++++++++++---- CMakeOpenCLPythonPackage.txt | 110 ++++++++++++++++++++++++++++++ CMakeOpenCLPythonPackageCache.txt | 13 ++++ python-package/setup.py | 39 +++++++---- 4 files changed, 200 insertions(+), 24 deletions(-) create mode 100644 CMakeOpenCLPythonPackage.txt create mode 100644 CMakeOpenCLPythonPackageCache.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index a07c3fe79b6f..bac1141e27d3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,6 @@ -if(USE_GPU OR APPLE) +if(__OPENCL_PYTHON_PACKAGE) + cmake_minimum_required(VERSION 3.11) +elseif(USE_GPU OR APPLE) cmake_minimum_required(VERSION 3.2) else() cmake_minimum_required(VERSION 2.8) @@ -20,6 +22,11 @@ if(APPLE) OPTION(APPLE_OUTPUT_DYLIB "Output dylib shared library" OFF) endif(APPLE) +if(__OPENCL_PYTHON_PACKAGE) + set(USE_GPU OFF CACHE BOOL "" FORCE) + message(WARNING "\nWARNING: building integrated OpenCL enabled components intended for Python package") +endif() + if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.8.2") message(FATAL_ERROR "Insufficient gcc version") @@ -127,6 +134,11 @@ if(USE_GPU) ADD_DEFINITIONS(-DUSE_GPU) endif(USE_GPU) +if(__OPENCL_PYTHON_PACKAGE) + include(CMakeOpenCLPythonPackage.txt) + ADD_DEFINITIONS(-DUSE_GPU) +endif() + if(USE_HDFS) find_package(JNI REQUIRED) find_path(HDFS_INCLUDE_DIR hdfs.h REQUIRED) @@ -224,7 +236,7 @@ if(USE_MPI) include_directories(${MPI_CXX_INCLUDE_PATH}) endif(USE_MPI) -file(GLOB SOURCES +file(GLOB SHARED_SOURCES src/application/*.cpp src/boosting/*.cpp src/io/*.cpp @@ -234,19 +246,27 @@ file(GLOB SOURCES src/treelearner/*.cpp ) -add_executable(lightgbm src/main.cpp ${SOURCES}) -list(APPEND SOURCES "src/c_api.cpp") +add_library(L OBJECT ${SHARED_SOURCES}) + +add_executable(lightgbm src/main.cpp $) +list(APPEND LIBRARY_SOURCES "src/c_api.cpp") # Only build the R part of the library if building for # use with the R package if(BUILD_FOR_R) - list(APPEND SOURCES "src/lightgbm_R.cpp") + list(APPEND LIBRARY_SOURCES "src/lightgbm_R.cpp") endif(BUILD_FOR_R) -add_library(_lightgbm SHARED ${SOURCES}) +add_library(_lightgbm SHARED ${LIBRARY_SOURCES} $) if(MSVC) - set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm") + set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm") + # TODO: use uniquely named dll (in setup.py, lightgbm/libpath.py) to avoid runtime conflicts + #if(__OPENCL_PYTHON_PACKAGE) + # set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm-${OPENCL_PYTHON_PACKAGE_STAMP}") + #else() + # set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm") + #endif() endif(MSVC) if(USE_SWIG) @@ -303,16 +323,34 @@ if(USE_GPU) TARGET_LINK_LIBRARIES(_lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES}) endif(USE_GPU) +if(__OPENCL_PYTHON_PACKAGE) + # targets OpenCL and Boost are added in CMakeOpenCLPythonPackage.txt + add_dependencies(L OpenCL Boost) + add_dependencies(lightgbm L) + add_dependencies(_lightgbm L) + # variables OPENCL_PYTHON_PACKAGE_* are set in CMakeOpenCLPythonPackage.txt + target_include_directories(L PRIVATE ${OPENCL_PYTHON_PACKAGE_INCLUDES}) + target_compile_definitions(L PRIVATE ${OPENCL_PYTHON_PACKAGE_DEFINITIONS}) + target_link_libraries(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_LIBRARIES}) + target_link_libraries(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_LIBRARIES}) +endif() + if(USE_HDFS) TARGET_LINK_LIBRARIES(lightgbm ${HDFS_CXX_LIBRARIES}) TARGET_LINK_LIBRARIES(_lightgbm ${HDFS_CXX_LIBRARIES}) endif(USE_HDFS) -if(WIN32 AND (MINGW OR CYGWIN)) - TARGET_LINK_LIBRARIES(lightgbm Ws2_32) - TARGET_LINK_LIBRARIES(_lightgbm Ws2_32) - TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI) - TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI) +if(WIN32) + if (MINGW OR CYGWIN) + TARGET_LINK_LIBRARIES(lightgbm Ws2_32) + TARGET_LINK_LIBRARIES(_lightgbm Ws2_32) + TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI) + TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI) + endif() +else() + if(__OPENCL_PYTHON_PACKAGE) + target_link_libraries(lightgbm PRIVATE pthread dl) + endif() endif() if(BUILD_FOR_R) diff --git a/CMakeOpenCLPythonPackage.txt b/CMakeOpenCLPythonPackage.txt new file mode 100644 index 000000000000..edcf3c6ec12c --- /dev/null +++ b/CMakeOpenCLPythonPackage.txt @@ -0,0 +1,110 @@ + +set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) +set(BOOST_VERSION_DOT "1.73") +string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT}) + +set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git") +set(OPENCL_HEADER_TAG "3f7c1f77f8e1a3c9b48765c713ca19013e95b91e") +#set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git") +#set(OPENCL_LOADER_TAG "bbdf079426d859fb8a68c332b41b714f9c87d6ad") +set(OPENCL_LOADER_REPOSITORY "https://github.com/tpboudreau/OpenCL-ICD-Loader.git") +set(OPENCL_LOADER_TAG "e42b3464e63f30cb0b818d88b836dd9e02ddbd7b") +set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git") +set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0") +execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE LIGHTGBM_TAG OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) +string(SHA1 OPENCL_PYTHON_PACKAGE_STAMP "${OPENCL_HEADER_REPOSITORY}@${OPENCL_HEADER_TAG};${OPENCL_LOADER_REPOSITORY}@${OPENCL_LOADER_TAG};${BOOST_REPOSITORY}@${BOOST_TAG};lightgbm@${LIGHTGBM_TAG}") +message(STATUS "OpenCL Python package build stamp: ${OPENCL_PYTHON_PACKAGE_STAMP}") + +# Build Independent OpenCL library +include(FetchContent) +FetchContent_Declare(OpenCL-Headers GIT_REPOSITORY ${OPENCL_HEADER_REPOSITORY} GIT_TAG ${OPENCL_HEADER_TAG}) +FetchContent_GetProperties(OpenCL-Headers) +if(NOT OpenCL-Headers_POPULATED) + FetchContent_Populate(OpenCL-Headers) + message(STATUS "Populated OpenCL Headers") +endif() +set(OPENCL_ICD_LOADER_HEADERS_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for OpenCL ICD Loader +set(OpenCL_INCLUDE_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for Boost::Compute + +FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG}) +FetchContent_GetProperties(OpenCL-ICD-Loader) +if(NOT OpenCL-ICD-Loader_POPULATED) + FetchContent_Populate(OpenCL-ICD-Loader) + if(WIN32) + set(USE_DYNAMIC_VCXX_RUNTIME ON) + else() + set(OPENCL_ICD_LOADER_PIC ON) + endif() + add_subdirectory(${opencl-icd-loader_SOURCE_DIR} ${opencl-icd-loader_BINARY_DIR} EXCLUDE_FROM_ALL) + message(STATUS "Populated OpenCL ICD Loader") +endif() +list(APPEND OPENCL_PYTHON_PACKAGE_INCLUDES ${OPENCL_ICD_LOADER_HEADERS_DIR}) +if(WIN32) + list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/Release/OpenCL.lib cfgmgr32.lib runtimeobject.lib) +else() + list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/libOpenCL.a) +endif() +list(APPEND OPENCL_PYTHON_PACKAGE_DEFINITIONS CL_TARGET_OPENCL_VERSION=220) + +# Build Independent Boost libraries +include(ExternalProject) +include(ProcessorCount) +ProcessorCount(J) +set(BOOST_BASE "${PROJECT_BINARY_DIR}/Boost") +if(WIN32) + set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.bat") + set(BOOST_BUILD "${BOOST_BASE}/source/b2.exe") + set(BOOST_FLAGS "") + list(APPEND BOOST_SUBMODULES "libs/*" "tools/*") +else() + set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.sh") + set(BOOST_BUILD "${BOOST_BASE}/source/b2") + set(BOOST_FLAGS -fPIC) + list(APPEND BOOST_SUBMODULES "libs/compute" "libs/chrono" "libs/config" "libs/filesystem" "libs/headers" "libs/io" "libs/predef" "libs/system" "libs/uuid" "tools/*") +endif() +ExternalProject_Add(Boost + TMP_DIR "${BOOST_BASE}/tmp" + STAMP_DIR "${BOOST_BASE}/stamp" + DOWNLOAD_DIR "${BOOST_BASE}/download" + SOURCE_DIR "${BOOST_BASE}/source" + BINARY_DIR "${BOOST_BASE}/source" + INSTALL_DIR "${BOOST_BASE}/install" + GIT_REPOSITORY ${BOOST_REPOSITORY} + GIT_TAG ${BOOST_TAG} + GIT_SUBMODULES ${BOOST_SUBMODULES} + GIT_SHALLOW ON + UPDATE_COMMAND "" + PATCH_COMMAND "" + CONFIGURE_COMMAND ${BOOST_BOOTSTRAP} + BUILD_COMMAND ${BOOST_BUILD} -sBOOST_ROOT=${BOOST_BASE}/source -a -q -j ${J} --with-headers --with-chrono --with-filesystem --with-system link=static runtime-link=shared variant=release threading=multi cxxflags="${BOOST_FLAGS}" + INSTALL_COMMAND "" +) +set(BOOST_INCLUDE ${BOOST_BASE}/source CACHE PATH "") +set(BOOST_LIBRARY ${BOOST_BASE}/source/stage/lib CACHE PATH "") +list(APPEND OPENCL_PYTHON_PACKAGE_INCLUDES ${BOOST_INCLUDE}) +if(WIN32) + if(MSVC) + if(${MSVC_VERSION} GREATER 1929) + set(MSVC_TOOLCHAIN_ID "unknown") + elseif(${MSVC_VERSION} GREATER 1919) + set(MSVC_TOOLCHAIN_ID "142") + elseif(${MSVC_VERSION} GREATER 1909) + set(MSVC_TOOLCHAIN_ID "141") + elseif(${MSVC_VERSION} GREATER 1899) + set(MSVC_TOOLCHAIN_ID "140") + else() + set(MSVC_TOOLCHAIN_ID "unknown") + endif() + list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) + list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) + list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) + else() + message(WARNING "MinGW Boost library names not yet specified") + endif() +else() + list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem.a) + list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_system.a) +endif() + +set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE) + diff --git a/CMakeOpenCLPythonPackageCache.txt b/CMakeOpenCLPythonPackageCache.txt new file mode 100644 index 000000000000..8831b292cd87 --- /dev/null +++ b/CMakeOpenCLPythonPackageCache.txt @@ -0,0 +1,13 @@ + +set(__OPENCL_PYTHON_PACKAGE ON CACHE BOOL "" FORCE) + +set(USE_MPI OFF CACHE BOOL "") +set(USE_OPENMP ON CACHE BOOL "") +set(USE_GPU OFF CACHE BOOL "") +set(USE_SWIG OFF CACHE BOOL "") +set(USE_HDFS OFF CACHE BOOL "") +set(USE_R35 OFF CACHE BOOL "") +set(USE_TIMETAG OFF CACHE BOOL "") +set(USE_DEBUG OFF CACHE BOOL "") +set(BUILD_FOR_R OFF CACHE BOOL "") + diff --git a/python-package/setup.py b/python-package/setup.py index 73f123baf42d..f653a044d8ae 100644 --- a/python-package/setup.py +++ b/python-package/setup.py @@ -28,7 +28,7 @@ def find_lib(): return LIB_PATH -def copy_files(use_gpu=False): +def copy_files(opencl_python_package=False, use_gpu=False): def copy_files_helper(folder_name): src = os.path.join(CURRENT_DIR, os.path.pardir, folder_name) @@ -50,14 +50,21 @@ def copy_files_helper(folder_name): distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.vcxproj"), os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.vcxproj"), verbose=0) - if use_gpu: - copy_files_helper('compute') - distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"), - os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"), - verbose=0) distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "LICENSE"), os.path.join(CURRENT_DIR, "LICENSE"), verbose=0) + distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"), + os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"), + verbose=0) + if opencl_python_package: + distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeOpenCLPythonPackage.txt"), + os.path.join(CURRENT_DIR, "compile", "CMakeOpenCLPythonPackage.txt"), + verbose=0) + distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeOpenCLPythonPackageCache.txt"), + os.path.join(CURRENT_DIR, "compile", "CMakeOpenCLPythonPackageCache.txt"), + verbose=0) + if use_gpu: + copy_files_helper('compute') def clear_path(path): @@ -90,7 +97,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, use_hdfs=False, boost_root=None, boost_dir=None, boost_include_dir=None, boost_librarydir=None, opencl_include_dir=None, opencl_library=None, - nomp=False, bit32=False): + nomp=False, bit32=False, opencl_python_package=False): if os.path.exists(os.path.join(CURRENT_DIR, "build_cpp")): shutil.rmtree(os.path.join(CURRENT_DIR, "build_cpp")) @@ -100,6 +107,12 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, logger.info("Starting to compile the library.") cmake_cmd = ["cmake", "../compile/"] + if opencl_python_package: + use_gpu = False + use_mpi = False + nomp = False + use_hdfs = False + cmake_cmd.append("-C../compile/CMakeOpenCLPythonPackageCache.txt") if use_gpu: cmake_cmd.append("-DUSE_GPU=ON") if boost_root: @@ -133,7 +146,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, else: status = 1 lib_path = os.path.join(CURRENT_DIR, "compile", "windows", "x64", "DLL", "lib_lightgbm.dll") - if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32)): + if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32, opencl_python_package)): logger.info("Starting to compile with MSBuild from existing solution file.") platform_toolsets = ("v142", "v141", "v140") for pt in platform_toolsets: @@ -166,7 +179,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, else: # Linux, Darwin (macOS), etc. logger.info("Starting to compile with CMake.") silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first') - silent_call(["make", "_lightgbm", "-j4"], raise_error=True, + silent_call(["make", "_lightgbm", "-j2"], raise_error=True, error_msg='An error has occurred while building lightgbm library file') os.chdir(CURRENT_DIR) @@ -186,6 +199,7 @@ class CustomInstall(install): user_options = install.user_options + [ ('mingw', 'm', 'Compile with MinGW'), + ('opencl-python-package', None, 'Compile integrated OpenCL version'), ('gpu', 'g', 'Compile GPU version'), ('mpi', None, 'Compile MPI version'), ('nomp', None, 'Compile version without OpenMP support'), @@ -203,6 +217,7 @@ class CustomInstall(install): def initialize_options(self): install.initialize_options(self) self.mingw = 0 + self.opencl_python_package = 0 self.gpu = 0 self.boost_root = None self.boost_dir = None @@ -226,12 +241,12 @@ def run(self): "please use 64-bit Python instead.") open(LOG_PATH, 'wb').close() if not self.precompile: - copy_files(use_gpu=self.gpu) + copy_files(opencl_python_package=self.opencl_python_package, use_gpu=self.gpu) compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_mpi=self.mpi, use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir, boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir, opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library, - nomp=self.nomp, bit32=self.bit32) + nomp=self.nomp, bit32=self.bit32, opencl_python_package=self.opencl_python_package) install.run(self) if os.path.isfile(LOG_PATH): os.remove(LOG_PATH) @@ -240,7 +255,7 @@ def run(self): class CustomSdist(sdist): def run(self): - copy_files(use_gpu=True) + copy_files(opencl_python_package=True, use_gpu=True) open(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'), 'w').close() if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')): shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')) From 27dffcb53ac8203cc5a6a4ad168cb4442a99f504 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Mon, 1 Jun 2020 21:57:12 -0700 Subject: [PATCH 02/22] Refer to upstream OpenCL repository --- CMakeOpenCLPythonPackage.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeOpenCLPythonPackage.txt b/CMakeOpenCLPythonPackage.txt index edcf3c6ec12c..c144ed64205d 100644 --- a/CMakeOpenCLPythonPackage.txt +++ b/CMakeOpenCLPythonPackage.txt @@ -5,10 +5,10 @@ string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT}) set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git") set(OPENCL_HEADER_TAG "3f7c1f77f8e1a3c9b48765c713ca19013e95b91e") -#set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git") -#set(OPENCL_LOADER_TAG "bbdf079426d859fb8a68c332b41b714f9c87d6ad") -set(OPENCL_LOADER_REPOSITORY "https://github.com/tpboudreau/OpenCL-ICD-Loader.git") -set(OPENCL_LOADER_TAG "e42b3464e63f30cb0b818d88b836dd9e02ddbd7b") +set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git") +set(OPENCL_LOADER_TAG "bbdf079426d859fb8a68c332b41b714f9c87d6ad") +#set(OPENCL_LOADER_REPOSITORY "https://github.com/tpboudreau/OpenCL-ICD-Loader.git") +#set(OPENCL_LOADER_TAG "e42b3464e63f30cb0b818d88b836dd9e02ddbd7b") set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git") set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0") execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE LIGHTGBM_TAG OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) From 2958084381ebeea59e3ab0fc021451d4922648c6 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Wed, 3 Jun 2020 16:52:35 +0000 Subject: [PATCH 03/22] Reset build job count in setup.py --- python-package/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-package/setup.py b/python-package/setup.py index f653a044d8ae..afac0bf80f9c 100644 --- a/python-package/setup.py +++ b/python-package/setup.py @@ -179,7 +179,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, else: # Linux, Darwin (macOS), etc. logger.info("Starting to compile with CMake.") silent_call(cmake_cmd, raise_error=True, error_msg='Please install CMake and all required dependencies first') - silent_call(["make", "_lightgbm", "-j2"], raise_error=True, + silent_call(["make", "_lightgbm", "-j4"], raise_error=True, error_msg='An error has occurred while building lightgbm library file') os.chdir(CURRENT_DIR) From 519a5b9dc54ed45b6843af40775d7d6ed0a29027 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Wed, 3 Jun 2020 18:50:20 +0000 Subject: [PATCH 04/22] TEMPORARY: refer to OpenCL fork to ensure Linux CI builds succeed --- CMakeOpenCLPythonPackage.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeOpenCLPythonPackage.txt b/CMakeOpenCLPythonPackage.txt index c144ed64205d..edcf3c6ec12c 100644 --- a/CMakeOpenCLPythonPackage.txt +++ b/CMakeOpenCLPythonPackage.txt @@ -5,10 +5,10 @@ string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT}) set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git") set(OPENCL_HEADER_TAG "3f7c1f77f8e1a3c9b48765c713ca19013e95b91e") -set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git") -set(OPENCL_LOADER_TAG "bbdf079426d859fb8a68c332b41b714f9c87d6ad") -#set(OPENCL_LOADER_REPOSITORY "https://github.com/tpboudreau/OpenCL-ICD-Loader.git") -#set(OPENCL_LOADER_TAG "e42b3464e63f30cb0b818d88b836dd9e02ddbd7b") +#set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git") +#set(OPENCL_LOADER_TAG "bbdf079426d859fb8a68c332b41b714f9c87d6ad") +set(OPENCL_LOADER_REPOSITORY "https://github.com/tpboudreau/OpenCL-ICD-Loader.git") +set(OPENCL_LOADER_TAG "e42b3464e63f30cb0b818d88b836dd9e02ddbd7b") set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git") set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0") execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE LIGHTGBM_TAG OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) From 84b6804ee4b0f44dbeabefe041358237a48a86e8 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Wed, 3 Jun 2020 19:45:52 +0000 Subject: [PATCH 05/22] Remove intermediate cmake target --- CMakeLists.txt | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bac1141e27d3..d0839d9e8ab3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -236,7 +236,7 @@ if(USE_MPI) include_directories(${MPI_CXX_INCLUDE_PATH}) endif(USE_MPI) -file(GLOB SHARED_SOURCES +file(GLOB SOURCES src/application/*.cpp src/boosting/*.cpp src/io/*.cpp @@ -246,18 +246,16 @@ file(GLOB SHARED_SOURCES src/treelearner/*.cpp ) -add_library(L OBJECT ${SHARED_SOURCES}) +add_executable(lightgbm src/main.cpp ${SOURCES}) +list(APPEND SOURCES "src/c_api.cpp") -add_executable(lightgbm src/main.cpp $) - -list(APPEND LIBRARY_SOURCES "src/c_api.cpp") # Only build the R part of the library if building for # use with the R package if(BUILD_FOR_R) - list(APPEND LIBRARY_SOURCES "src/lightgbm_R.cpp") + list(APPEND SOURCES "src/lightgbm_R.cpp") endif(BUILD_FOR_R) -add_library(_lightgbm SHARED ${LIBRARY_SOURCES} $) +add_library(_lightgbm SHARED ${SOURCES}) if(MSVC) set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm") @@ -325,12 +323,13 @@ endif(USE_GPU) if(__OPENCL_PYTHON_PACKAGE) # targets OpenCL and Boost are added in CMakeOpenCLPythonPackage.txt - add_dependencies(L OpenCL Boost) - add_dependencies(lightgbm L) - add_dependencies(_lightgbm L) + add_dependencies(lightgbm OpenCL Boost) + add_dependencies(_lightgbm OpenCL Boost) # variables OPENCL_PYTHON_PACKAGE_* are set in CMakeOpenCLPythonPackage.txt - target_include_directories(L PRIVATE ${OPENCL_PYTHON_PACKAGE_INCLUDES}) - target_compile_definitions(L PRIVATE ${OPENCL_PYTHON_PACKAGE_DEFINITIONS}) + target_include_directories(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_INCLUDES}) + target_include_directories(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_INCLUDES}) + target_compile_definitions(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_DEFINITIONS}) + target_compile_definitions(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_DEFINITIONS}) target_link_libraries(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_LIBRARIES}) target_link_libraries(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_LIBRARIES}) endif() From ec4b3f4880088ba7fbc2f42539b3a7297c2f9491 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Thu, 4 Jun 2020 20:22:04 +0000 Subject: [PATCH 06/22] Restrict OpenCL headers to documented API version --- CMakeOpenCLPythonPackage.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeOpenCLPythonPackage.txt b/CMakeOpenCLPythonPackage.txt index edcf3c6ec12c..b63989e71376 100644 --- a/CMakeOpenCLPythonPackage.txt +++ b/CMakeOpenCLPythonPackage.txt @@ -44,7 +44,7 @@ if(WIN32) else() list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/libOpenCL.a) endif() -list(APPEND OPENCL_PYTHON_PACKAGE_DEFINITIONS CL_TARGET_OPENCL_VERSION=220) +list(APPEND OPENCL_PYTHON_PACKAGE_DEFINITIONS CL_TARGET_OPENCL_VERSION=120) # Build Independent Boost libraries include(ExternalProject) From 1fc78782badbe3e5015981e8e65f28937691ad43 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Mon, 8 Jun 2020 20:06:35 +0000 Subject: [PATCH 07/22] Use command line definition to activate integrated build --- CMakeLists.txt | 9 +++++++++ CMakeOpenCLPythonPackageCache.txt | 13 ------------- python-package/setup.py | 5 +---- 3 files changed, 10 insertions(+), 17 deletions(-) delete mode 100644 CMakeOpenCLPythonPackageCache.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index d0839d9e8ab3..5853baca67db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,16 @@ if(APPLE) endif(APPLE) if(__OPENCL_PYTHON_PACKAGE) + set(__OPENCL_PYTHON_PACKAGE ON CACHE BOOL "" FORCE) set(USE_GPU OFF CACHE BOOL "" FORCE) + set(USE_MPI OFF CACHE BOOL "") + set(USE_OPENMP ON CACHE BOOL "") + set(USE_SWIG OFF CACHE BOOL "") + set(USE_HDFS OFF CACHE BOOL "") + set(USE_R35 OFF CACHE BOOL "") + set(USE_TIMETAG OFF CACHE BOOL "") + set(USE_DEBUG OFF CACHE BOOL "") + set(BUILD_FOR_R OFF CACHE BOOL "") message(WARNING "\nWARNING: building integrated OpenCL enabled components intended for Python package") endif() diff --git a/CMakeOpenCLPythonPackageCache.txt b/CMakeOpenCLPythonPackageCache.txt deleted file mode 100644 index 8831b292cd87..000000000000 --- a/CMakeOpenCLPythonPackageCache.txt +++ /dev/null @@ -1,13 +0,0 @@ - -set(__OPENCL_PYTHON_PACKAGE ON CACHE BOOL "" FORCE) - -set(USE_MPI OFF CACHE BOOL "") -set(USE_OPENMP ON CACHE BOOL "") -set(USE_GPU OFF CACHE BOOL "") -set(USE_SWIG OFF CACHE BOOL "") -set(USE_HDFS OFF CACHE BOOL "") -set(USE_R35 OFF CACHE BOOL "") -set(USE_TIMETAG OFF CACHE BOOL "") -set(USE_DEBUG OFF CACHE BOOL "") -set(BUILD_FOR_R OFF CACHE BOOL "") - diff --git a/python-package/setup.py b/python-package/setup.py index afac0bf80f9c..2a219a5d80fb 100644 --- a/python-package/setup.py +++ b/python-package/setup.py @@ -60,9 +60,6 @@ def copy_files_helper(folder_name): distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeOpenCLPythonPackage.txt"), os.path.join(CURRENT_DIR, "compile", "CMakeOpenCLPythonPackage.txt"), verbose=0) - distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeOpenCLPythonPackageCache.txt"), - os.path.join(CURRENT_DIR, "compile", "CMakeOpenCLPythonPackageCache.txt"), - verbose=0) if use_gpu: copy_files_helper('compute') @@ -112,7 +109,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, use_mpi = False nomp = False use_hdfs = False - cmake_cmd.append("-C../compile/CMakeOpenCLPythonPackageCache.txt") + cmake_cmd.append("-D__OPENCL_PYTHON_PACKAGE=ON") if use_gpu: cmake_cmd.append("-DUSE_GPU=ON") if boost_root: From af176a6cfdc7a24ae3d690a05e6ef6c131b171bc Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Tue, 9 Jun 2020 18:00:14 +0000 Subject: [PATCH 08/22] Flag reference to unofficial repo with FIXME --- CMakeOpenCLPythonPackage.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeOpenCLPythonPackage.txt b/CMakeOpenCLPythonPackage.txt index b63989e71376..de27e4cb33d7 100644 --- a/CMakeOpenCLPythonPackage.txt +++ b/CMakeOpenCLPythonPackage.txt @@ -5,10 +5,13 @@ string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT}) set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git") set(OPENCL_HEADER_TAG "3f7c1f77f8e1a3c9b48765c713ca19013e95b91e") + +# FIXME -- remove reference to OpenCL ICD Loader fork after decision on whether or not to include Linux build #set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git") #set(OPENCL_LOADER_TAG "bbdf079426d859fb8a68c332b41b714f9c87d6ad") set(OPENCL_LOADER_REPOSITORY "https://github.com/tpboudreau/OpenCL-ICD-Loader.git") set(OPENCL_LOADER_TAG "e42b3464e63f30cb0b818d88b836dd9e02ddbd7b") + set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git") set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0") execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE LIGHTGBM_TAG OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) From 180c92924a6ab401ead9f766eb6aa69b790d5dfc Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Thu, 27 Aug 2020 21:14:13 +0000 Subject: [PATCH 09/22] TEMPORARY: update private repo tag for dependency --- CMakeOpenCLPythonPackage.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeOpenCLPythonPackage.txt b/CMakeOpenCLPythonPackage.txt index de27e4cb33d7..5f2ecc709214 100644 --- a/CMakeOpenCLPythonPackage.txt +++ b/CMakeOpenCLPythonPackage.txt @@ -10,7 +10,7 @@ set(OPENCL_HEADER_TAG "3f7c1f77f8e1a3c9b48765c713ca19013e95b91e") #set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git") #set(OPENCL_LOADER_TAG "bbdf079426d859fb8a68c332b41b714f9c87d6ad") set(OPENCL_LOADER_REPOSITORY "https://github.com/tpboudreau/OpenCL-ICD-Loader.git") -set(OPENCL_LOADER_TAG "e42b3464e63f30cb0b818d88b836dd9e02ddbd7b") +set(OPENCL_LOADER_TAG "3d93e6b630a3592e4ab40f3fcc69a70d5ab8cc42") set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git") set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0") From 1782aed222726af05398ba1a936d515660a3312b Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Thu, 3 Sep 2020 05:11:47 +0000 Subject: [PATCH 10/22] Remove integrated build for non-Win32 and related cleanup --- CMakeLists.txt | 20 +++--- ...kage.txt => CMakeOpenCLPythonPackage.cmake | 69 +++++++------------ python-package/setup.py | 4 +- 3 files changed, 35 insertions(+), 58 deletions(-) rename CMakeOpenCLPythonPackage.txt => CMakeOpenCLPythonPackage.cmake (59%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 95db443d3a45..930bb811c15f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,7 +33,7 @@ if(__OPENCL_PYTHON_PACKAGE) set(USE_TIMETAG OFF CACHE BOOL "") set(USE_DEBUG OFF CACHE BOOL "") set(BUILD_FOR_R OFF CACHE BOOL "") - message(WARNING "\nWARNING: building integrated OpenCL enabled components intended for Python package") + message(WARNING "WARNING: building integrated OpenCL enabled components intended for Python package") endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") @@ -140,8 +140,12 @@ if(USE_GPU) endif(USE_GPU) if(__OPENCL_PYTHON_PACKAGE) - include(CMakeOpenCLPythonPackage.txt) - ADD_DEFINITIONS(-DUSE_GPU) + if (WIN32) + include(CMakeOpenCLPythonPackage.cmake) + ADD_DEFINITIONS(-DUSE_GPU) + else() + message(FATAL_ERROR "Integrated OpenCL build available only on WIN32") + endif() endif() if(USE_HDFS) @@ -268,12 +272,6 @@ endif(BUILD_STATIC_LIB) if(MSVC) set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm") - # TODO: use uniquely named dll (in setup.py, lightgbm/libpath.py) to avoid runtime conflicts - #if(__OPENCL_PYTHON_PACKAGE) - # set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm-${OPENCL_PYTHON_PACKAGE_STAMP}") - #else() - # set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm") - #endif() endif(MSVC) if(USE_SWIG) @@ -331,10 +329,10 @@ if(USE_GPU) endif(USE_GPU) if(__OPENCL_PYTHON_PACKAGE) - # targets OpenCL and Boost are added in CMakeOpenCLPythonPackage.txt + # targets OpenCL and Boost are added in CMakeOpenCLPythonPackage.cmake add_dependencies(lightgbm OpenCL Boost) add_dependencies(_lightgbm OpenCL Boost) - # variables OPENCL_PYTHON_PACKAGE_* are set in CMakeOpenCLPythonPackage.txt + # variables OPENCL_PYTHON_PACKAGE_* are set in CMakeOpenCLPythonPackage.cmake target_include_directories(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_INCLUDES}) target_include_directories(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_INCLUDES}) target_compile_definitions(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_DEFINITIONS}) diff --git a/CMakeOpenCLPythonPackage.txt b/CMakeOpenCLPythonPackage.cmake similarity index 59% rename from CMakeOpenCLPythonPackage.txt rename to CMakeOpenCLPythonPackage.cmake index 5f2ecc709214..7f2c4f47351a 100644 --- a/CMakeOpenCLPythonPackage.txt +++ b/CMakeOpenCLPythonPackage.cmake @@ -4,13 +4,12 @@ set(BOOST_VERSION_DOT "1.73") string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT}) set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git") -set(OPENCL_HEADER_TAG "3f7c1f77f8e1a3c9b48765c713ca19013e95b91e") +#set(OPENCL_HEADER_TAG "3f7c1f77f8e1a3c9b48765c713ca19013e95b91e") +set(OPENCL_HEADER_TAG "1b2a1850f410aaaaeaa56cead5a179b5aea4918e") -# FIXME -- remove reference to OpenCL ICD Loader fork after decision on whether or not to include Linux build -#set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git") +set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git") #set(OPENCL_LOADER_TAG "bbdf079426d859fb8a68c332b41b714f9c87d6ad") -set(OPENCL_LOADER_REPOSITORY "https://github.com/tpboudreau/OpenCL-ICD-Loader.git") -set(OPENCL_LOADER_TAG "3d93e6b630a3592e4ab40f3fcc69a70d5ab8cc42") +set(OPENCL_LOADER_TAG "862eebe7ca733c398334a8db8481172a7d3a3c47") set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git") set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0") @@ -33,20 +32,12 @@ FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY FetchContent_GetProperties(OpenCL-ICD-Loader) if(NOT OpenCL-ICD-Loader_POPULATED) FetchContent_Populate(OpenCL-ICD-Loader) - if(WIN32) - set(USE_DYNAMIC_VCXX_RUNTIME ON) - else() - set(OPENCL_ICD_LOADER_PIC ON) - endif() + set(USE_DYNAMIC_VCXX_RUNTIME ON) add_subdirectory(${opencl-icd-loader_SOURCE_DIR} ${opencl-icd-loader_BINARY_DIR} EXCLUDE_FROM_ALL) message(STATUS "Populated OpenCL ICD Loader") endif() list(APPEND OPENCL_PYTHON_PACKAGE_INCLUDES ${OPENCL_ICD_LOADER_HEADERS_DIR}) -if(WIN32) - list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/Release/OpenCL.lib cfgmgr32.lib runtimeobject.lib) -else() - list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/libOpenCL.a) -endif() +list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/Release/OpenCL.lib cfgmgr32.lib runtimeobject.lib) list(APPEND OPENCL_PYTHON_PACKAGE_DEFINITIONS CL_TARGET_OPENCL_VERSION=120) # Build Independent Boost libraries @@ -54,17 +45,10 @@ include(ExternalProject) include(ProcessorCount) ProcessorCount(J) set(BOOST_BASE "${PROJECT_BINARY_DIR}/Boost") -if(WIN32) - set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.bat") - set(BOOST_BUILD "${BOOST_BASE}/source/b2.exe") - set(BOOST_FLAGS "") - list(APPEND BOOST_SUBMODULES "libs/*" "tools/*") -else() - set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.sh") - set(BOOST_BUILD "${BOOST_BASE}/source/b2") - set(BOOST_FLAGS -fPIC) - list(APPEND BOOST_SUBMODULES "libs/compute" "libs/chrono" "libs/config" "libs/filesystem" "libs/headers" "libs/io" "libs/predef" "libs/system" "libs/uuid" "tools/*") -endif() +set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.bat") +set(BOOST_BUILD "${BOOST_BASE}/source/b2.exe") +set(BOOST_FLAGS "") +list(APPEND BOOST_SUBMODULES "libs/*" "tools/*") ExternalProject_Add(Boost TMP_DIR "${BOOST_BASE}/tmp" STAMP_DIR "${BOOST_BASE}/stamp" @@ -85,28 +69,23 @@ ExternalProject_Add(Boost set(BOOST_INCLUDE ${BOOST_BASE}/source CACHE PATH "") set(BOOST_LIBRARY ${BOOST_BASE}/source/stage/lib CACHE PATH "") list(APPEND OPENCL_PYTHON_PACKAGE_INCLUDES ${BOOST_INCLUDE}) -if(WIN32) - if(MSVC) - if(${MSVC_VERSION} GREATER 1929) - set(MSVC_TOOLCHAIN_ID "unknown") - elseif(${MSVC_VERSION} GREATER 1919) - set(MSVC_TOOLCHAIN_ID "142") - elseif(${MSVC_VERSION} GREATER 1909) - set(MSVC_TOOLCHAIN_ID "141") - elseif(${MSVC_VERSION} GREATER 1899) - set(MSVC_TOOLCHAIN_ID "140") - else() - set(MSVC_TOOLCHAIN_ID "unknown") - endif() - list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) - list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) - list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) +if(MSVC) + if(${MSVC_VERSION} GREATER 1929) + set(MSVC_TOOLCHAIN_ID "unknown") + elseif(${MSVC_VERSION} GREATER 1919) + set(MSVC_TOOLCHAIN_ID "142") + elseif(${MSVC_VERSION} GREATER 1909) + set(MSVC_TOOLCHAIN_ID "141") + elseif(${MSVC_VERSION} GREATER 1899) + set(MSVC_TOOLCHAIN_ID "140") else() - message(WARNING "MinGW Boost library names not yet specified") + set(MSVC_TOOLCHAIN_ID "unknown") endif() + list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) + list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) + list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) else() - list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem.a) - list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_system.a) + message(WARNING "MinGW Boost library names not yet specified") endif() set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE) diff --git a/python-package/setup.py b/python-package/setup.py index 6acb7e3ccda6..b15e1aa18bb5 100644 --- a/python-package/setup.py +++ b/python-package/setup.py @@ -58,8 +58,8 @@ def copy_files_helper(folder_name): os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"), verbose=0) if opencl_python_package: - copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeOpenCLPythonPackage.txt"), - os.path.join(CURRENT_DIR, "compile", "CMakeOpenCLPythonPackage.txt"), + copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeOpenCLPythonPackage.cmake"), + os.path.join(CURRENT_DIR, "compile", "CMakeOpenCLPythonPackage.cmake"), verbose=0) if use_gpu: copy_files_helper('compute') From 3013f4db57315c81e719a98b4652928d580ad577 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Thu, 3 Sep 2020 06:16:10 +0000 Subject: [PATCH 11/22] Remove commented code --- CMakeOpenCLPythonPackage.cmake | 2 -- 1 file changed, 2 deletions(-) diff --git a/CMakeOpenCLPythonPackage.cmake b/CMakeOpenCLPythonPackage.cmake index 7f2c4f47351a..d09af60cf4fe 100644 --- a/CMakeOpenCLPythonPackage.cmake +++ b/CMakeOpenCLPythonPackage.cmake @@ -4,11 +4,9 @@ set(BOOST_VERSION_DOT "1.73") string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT}) set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git") -#set(OPENCL_HEADER_TAG "3f7c1f77f8e1a3c9b48765c713ca19013e95b91e") set(OPENCL_HEADER_TAG "1b2a1850f410aaaaeaa56cead5a179b5aea4918e") set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git") -#set(OPENCL_LOADER_TAG "bbdf079426d859fb8a68c332b41b714f9c87d6ad") set(OPENCL_LOADER_TAG "862eebe7ca733c398334a8db8481172a7d3a3c47") set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git") From 9d0614cccd2b9c962c5dd9340542679b9bc55444 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Tue, 8 Sep 2020 15:55:22 -0700 Subject: [PATCH 12/22] Rename integrated OpenCL build option and other cleanups --- ...ckage.cmake => CMakeIntegratedOpenCL.cmake | 28 ++++++------ CMakeLists.txt | 44 +++++++------------ python-package/setup.py | 29 ++++++------ 3 files changed, 42 insertions(+), 59 deletions(-) rename CMakeOpenCLPythonPackage.cmake => CMakeIntegratedOpenCL.cmake (71%) diff --git a/CMakeOpenCLPythonPackage.cmake b/CMakeIntegratedOpenCL.cmake similarity index 71% rename from CMakeOpenCLPythonPackage.cmake rename to CMakeIntegratedOpenCL.cmake index d09af60cf4fe..2febad863bc7 100644 --- a/CMakeOpenCLPythonPackage.cmake +++ b/CMakeIntegratedOpenCL.cmake @@ -1,6 +1,5 @@ - set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) -set(BOOST_VERSION_DOT "1.73") +set(BOOST_VERSION_DOT "1.74") string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT}) set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git") @@ -12,8 +11,8 @@ set(OPENCL_LOADER_TAG "862eebe7ca733c398334a8db8481172a7d3a3c47") set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git") set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0") execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE LIGHTGBM_TAG OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) -string(SHA1 OPENCL_PYTHON_PACKAGE_STAMP "${OPENCL_HEADER_REPOSITORY}@${OPENCL_HEADER_TAG};${OPENCL_LOADER_REPOSITORY}@${OPENCL_LOADER_TAG};${BOOST_REPOSITORY}@${BOOST_TAG};lightgbm@${LIGHTGBM_TAG}") -message(STATUS "OpenCL Python package build stamp: ${OPENCL_PYTHON_PACKAGE_STAMP}") +string(SHA1 INTEGRATED_OPENCL_STAMP "${OPENCL_HEADER_REPOSITORY}@${OPENCL_HEADER_TAG};${OPENCL_LOADER_REPOSITORY}@${OPENCL_LOADER_TAG};${BOOST_REPOSITORY}@${BOOST_TAG};lightgbm@${LIGHTGBM_TAG}") +message(STATUS "Integrated OpenCL build stamp: ${INTEGRATED_OPENCL_STAMP}") # Build Independent OpenCL library include(FetchContent) @@ -34,9 +33,9 @@ if(NOT OpenCL-ICD-Loader_POPULATED) add_subdirectory(${opencl-icd-loader_SOURCE_DIR} ${opencl-icd-loader_BINARY_DIR} EXCLUDE_FROM_ALL) message(STATUS "Populated OpenCL ICD Loader") endif() -list(APPEND OPENCL_PYTHON_PACKAGE_INCLUDES ${OPENCL_ICD_LOADER_HEADERS_DIR}) -list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/Release/OpenCL.lib cfgmgr32.lib runtimeobject.lib) -list(APPEND OPENCL_PYTHON_PACKAGE_DEFINITIONS CL_TARGET_OPENCL_VERSION=120) +list(APPEND INTEGRATED_OPENCL_INCLUDES ${OPENCL_ICD_LOADER_HEADERS_DIR}) +list(APPEND INTEGRATED_OPENCL_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/Release/OpenCL.lib cfgmgr32.lib runtimeobject.lib) +list(APPEND INTEGRATED_OPENCL_DEFINITIONS CL_TARGET_OPENCL_VERSION=120) # Build Independent Boost libraries include(ExternalProject) @@ -66,10 +65,10 @@ ExternalProject_Add(Boost ) set(BOOST_INCLUDE ${BOOST_BASE}/source CACHE PATH "") set(BOOST_LIBRARY ${BOOST_BASE}/source/stage/lib CACHE PATH "") -list(APPEND OPENCL_PYTHON_PACKAGE_INCLUDES ${BOOST_INCLUDE}) +list(APPEND INTEGRATED_OPENCL_INCLUDES ${BOOST_INCLUDE}) if(MSVC) if(${MSVC_VERSION} GREATER 1929) - set(MSVC_TOOLCHAIN_ID "unknown") + message(FATAL_ERROR "Unrecognized MSVC version number") elseif(${MSVC_VERSION} GREATER 1919) set(MSVC_TOOLCHAIN_ID "142") elseif(${MSVC_VERSION} GREATER 1909) @@ -77,14 +76,13 @@ if(MSVC) elseif(${MSVC_VERSION} GREATER 1899) set(MSVC_TOOLCHAIN_ID "140") else() - set(MSVC_TOOLCHAIN_ID "unknown") + message(FATAL_ERROR "Unrecognized MSVC version number") endif() - list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) - list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) - list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) + list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) + list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) + list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) else() - message(WARNING "MinGW Boost library names not yet specified") + message(FATAL_ERROR "MinGW Boost library names not yet specified") endif() set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE) - diff --git a/CMakeLists.txt b/CMakeLists.txt index 930bb811c15f..e01d011bddc1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,4 +1,4 @@ -if(__OPENCL_PYTHON_PACKAGE) +if(__INTEGRATE_OPENCL) cmake_minimum_required(VERSION 3.11) elseif(USE_GPU OR APPLE) cmake_minimum_required(VERSION 3.2) @@ -22,18 +22,10 @@ if(APPLE) OPTION(APPLE_OUTPUT_DYLIB "Output dylib shared library" OFF) endif(APPLE) -if(__OPENCL_PYTHON_PACKAGE) - set(__OPENCL_PYTHON_PACKAGE ON CACHE BOOL "" FORCE) +if(__INTEGRATE_OPENCL) + set(__INTEGRATE_OPENCL ON CACHE BOOL "" FORCE) set(USE_GPU OFF CACHE BOOL "" FORCE) - set(USE_MPI OFF CACHE BOOL "") - set(USE_OPENMP ON CACHE BOOL "") - set(USE_SWIG OFF CACHE BOOL "") - set(USE_HDFS OFF CACHE BOOL "") - set(USE_R35 OFF CACHE BOOL "") - set(USE_TIMETAG OFF CACHE BOOL "") - set(USE_DEBUG OFF CACHE BOOL "") - set(BUILD_FOR_R OFF CACHE BOOL "") - message(WARNING "WARNING: building integrated OpenCL enabled components intended for Python package") + message(WARNING "WARNING: building library with integrated OpenCL components") endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") @@ -139,12 +131,12 @@ if(USE_GPU) ADD_DEFINITIONS(-DUSE_GPU) endif(USE_GPU) -if(__OPENCL_PYTHON_PACKAGE) +if(__INTEGRATE_OPENCL) if (WIN32) - include(CMakeOpenCLPythonPackage.cmake) + include(CMakeIntegratedOpenCL.cmake) ADD_DEFINITIONS(-DUSE_GPU) else() - message(FATAL_ERROR "Integrated OpenCL build available only on WIN32") + message(FATAL_ERROR "Integrated OpenCL build is available only for Windows") endif() endif() @@ -328,17 +320,17 @@ if(USE_GPU) TARGET_LINK_LIBRARIES(_lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES}) endif(USE_GPU) -if(__OPENCL_PYTHON_PACKAGE) - # targets OpenCL and Boost are added in CMakeOpenCLPythonPackage.cmake +if(__INTEGRATE_OPENCL) + # targets OpenCL and Boost are added in CMakeIntegratedOpenCL.cmake add_dependencies(lightgbm OpenCL Boost) add_dependencies(_lightgbm OpenCL Boost) - # variables OPENCL_PYTHON_PACKAGE_* are set in CMakeOpenCLPythonPackage.cmake - target_include_directories(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_INCLUDES}) - target_include_directories(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_INCLUDES}) - target_compile_definitions(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_DEFINITIONS}) - target_compile_definitions(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_DEFINITIONS}) - target_link_libraries(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_LIBRARIES}) - target_link_libraries(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_LIBRARIES}) + # variables INTEGRATED_OPENCL_* are set in CMakeIntegratedOpenCL.cmake + target_include_directories(lightgbm PRIVATE ${INTEGRATED_OPENCL_INCLUDES}) + target_include_directories(_lightgbm PRIVATE ${INTEGRATED_OPENCL_INCLUDES}) + target_compile_definitions(lightgbm PRIVATE ${INTEGRATED_OPENCL_DEFINITIONS}) + target_compile_definitions(_lightgbm PRIVATE ${INTEGRATED_OPENCL_DEFINITIONS}) + target_link_libraries(lightgbm PRIVATE ${INTEGRATED_OPENCL_LIBRARIES}) + target_link_libraries(_lightgbm PRIVATE ${INTEGRATED_OPENCL_LIBRARIES}) endif() if(USE_HDFS) @@ -353,10 +345,6 @@ if(WIN32) TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI) TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI) endif() -else() - if(__OPENCL_PYTHON_PACKAGE) - target_link_libraries(lightgbm PRIVATE pthread dl) - endif() endif() if(BUILD_FOR_R) diff --git a/python-package/setup.py b/python-package/setup.py index b15e1aa18bb5..0cf7e1a76fc6 100644 --- a/python-package/setup.py +++ b/python-package/setup.py @@ -29,7 +29,7 @@ def find_lib(): return LIB_PATH -def copy_files(opencl_python_package=False, use_gpu=False): +def copy_files(integrated_opencl=False, use_gpu=False): def copy_files_helper(folder_name): src = os.path.join(CURRENT_DIR, os.path.pardir, folder_name) @@ -57,9 +57,9 @@ def copy_files_helper(folder_name): copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"), os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"), verbose=0) - if opencl_python_package: - copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeOpenCLPythonPackage.cmake"), - os.path.join(CURRENT_DIR, "compile", "CMakeOpenCLPythonPackage.cmake"), + if integrated_opencl: + copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeIntegratedOpenCL.cmake"), + os.path.join(CURRENT_DIR, "compile", "CMakeIntegratedOpenCL.cmake"), verbose=0) if use_gpu: copy_files_helper('compute') @@ -95,7 +95,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, use_hdfs=False, boost_root=None, boost_dir=None, boost_include_dir=None, boost_librarydir=None, opencl_include_dir=None, opencl_library=None, - nomp=False, bit32=False, opencl_python_package=False): + nomp=False, bit32=False, integrated_opencl=False): if os.path.exists(os.path.join(CURRENT_DIR, "build_cpp")): shutil.rmtree(os.path.join(CURRENT_DIR, "build_cpp")) @@ -105,12 +105,9 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, logger.info("Starting to compile the library.") cmake_cmd = ["cmake", "../compile/"] - if opencl_python_package: + if integrated_opencl: use_gpu = False - use_mpi = False - nomp = False - use_hdfs = False - cmake_cmd.append("-D__OPENCL_PYTHON_PACKAGE=ON") + cmake_cmd.append("-D__INTEGRATE_OPENCL=ON") if use_gpu: cmake_cmd.append("-DUSE_GPU=ON") if boost_root: @@ -144,7 +141,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False, else: status = 1 lib_path = os.path.join(CURRENT_DIR, "compile", "windows", "x64", "DLL", "lib_lightgbm.dll") - if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32, opencl_python_package)): + if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32, integrated_opencl)): logger.info("Starting to compile with MSBuild from existing solution file.") platform_toolsets = ("v142", "v141", "v140") for pt in platform_toolsets: @@ -197,7 +194,7 @@ class CustomInstall(install): user_options = install.user_options + [ ('mingw', 'm', 'Compile with MinGW'), - ('opencl-python-package', None, 'Compile integrated OpenCL version'), + ('integrated-opencl', None, 'Compile integrated OpenCL version'), ('gpu', 'g', 'Compile GPU version'), ('mpi', None, 'Compile MPI version'), ('nomp', None, 'Compile version without OpenMP support'), @@ -215,7 +212,7 @@ class CustomInstall(install): def initialize_options(self): install.initialize_options(self) self.mingw = 0 - self.opencl_python_package = 0 + self.integrated_opencl = 0 self.gpu = 0 self.boost_root = None self.boost_dir = None @@ -239,12 +236,12 @@ def run(self): "please use 64-bit Python instead.") open(LOG_PATH, 'wb').close() if not self.precompile: - copy_files(opencl_python_package=self.opencl_python_package, use_gpu=self.gpu) + copy_files(integrated_opencl=self.integrated_opencl, use_gpu=self.gpu) compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_mpi=self.mpi, use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir, boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir, opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library, - nomp=self.nomp, bit32=self.bit32, opencl_python_package=self.opencl_python_package) + nomp=self.nomp, bit32=self.bit32, integrated_opencl=self.integrated_opencl) install.run(self) if os.path.isfile(LOG_PATH): os.remove(LOG_PATH) @@ -253,7 +250,7 @@ def run(self): class CustomSdist(sdist): def run(self): - copy_files(opencl_python_package=True, use_gpu=True) + copy_files(integrated_opencl=True, use_gpu=True) open(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'), 'w').close() if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')): shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')) From d5fad621f6b5472f999c4cee016791307bb92d1f Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Mon, 14 Sep 2020 11:52:18 -0700 Subject: [PATCH 13/22] Small cleanups --- CMakeIntegratedOpenCL.cmake | 7 ++----- CMakeLists.txt | 1 + 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/CMakeIntegratedOpenCL.cmake b/CMakeIntegratedOpenCL.cmake index 2febad863bc7..d8a4ea065729 100644 --- a/CMakeIntegratedOpenCL.cmake +++ b/CMakeIntegratedOpenCL.cmake @@ -6,13 +6,10 @@ set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git set(OPENCL_HEADER_TAG "1b2a1850f410aaaaeaa56cead5a179b5aea4918e") set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git") -set(OPENCL_LOADER_TAG "862eebe7ca733c398334a8db8481172a7d3a3c47") +set(OPENCL_LOADER_TAG "98ca71fb9f8484f1cd1999f55224bf9e8d18693b") set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git") set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0") -execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE LIGHTGBM_TAG OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET) -string(SHA1 INTEGRATED_OPENCL_STAMP "${OPENCL_HEADER_REPOSITORY}@${OPENCL_HEADER_TAG};${OPENCL_LOADER_REPOSITORY}@${OPENCL_LOADER_TAG};${BOOST_REPOSITORY}@${BOOST_TAG};lightgbm@${LIGHTGBM_TAG}") -message(STATUS "Integrated OpenCL build stamp: ${INTEGRATED_OPENCL_STAMP}") # Build Independent OpenCL library include(FetchContent) @@ -82,7 +79,7 @@ if(MSVC) list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) else() - message(FATAL_ERROR "MinGW Boost library names not yet specified") + message(FATAL_ERROR "Integrated OpenCL build is not yet available for MinGW") endif() set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE) diff --git a/CMakeLists.txt b/CMakeLists.txt index e01d011bddc1..08ad055edd95 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,6 +17,7 @@ OPTION(USE_TIMETAG "Set to ON to output time costs" OFF) OPTION(USE_DEBUG "Set to ON for Debug mode" OFF) OPTION(BUILD_STATIC_LIB "Build static library" OFF) OPTION(BUILD_FOR_R "Set to ON if building lib_lightgbm for use with the R package" OFF) +OPTION(__INTEGRATE_OPENCL "Set to ON if building lib_lightgbm with the OpenCL ICD Loader and its dependencies included" OFF) if(APPLE) OPTION(APPLE_OUTPUT_DYLIB "Output dylib shared library" OFF) From dc5ef753cf35e116a9a172846363817b01f6121c Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Mon, 14 Sep 2020 11:55:36 -0700 Subject: [PATCH 14/22] Update CMakeIntegratedOpenCL.cmake Co-authored-by: Nikita Titov --- CMakeIntegratedOpenCL.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeIntegratedOpenCL.cmake b/CMakeIntegratedOpenCL.cmake index d8a4ea065729..37b57089cda0 100644 --- a/CMakeIntegratedOpenCL.cmake +++ b/CMakeIntegratedOpenCL.cmake @@ -60,7 +60,7 @@ ExternalProject_Add(Boost BUILD_COMMAND ${BOOST_BUILD} -sBOOST_ROOT=${BOOST_BASE}/source -a -q -j ${J} --with-headers --with-chrono --with-filesystem --with-system link=static runtime-link=shared variant=release threading=multi cxxflags="${BOOST_FLAGS}" INSTALL_COMMAND "" ) -set(BOOST_INCLUDE ${BOOST_BASE}/source CACHE PATH "") +set(BOOST_INCLUDE "${BOOST_BASE}/source" CACHE PATH "") set(BOOST_LIBRARY ${BOOST_BASE}/source/stage/lib CACHE PATH "") list(APPEND INTEGRATED_OPENCL_INCLUDES ${BOOST_INCLUDE}) if(MSVC) From 4fd05bc4705cea018423a81a62218078a9e8b538 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Mon, 14 Sep 2020 11:56:12 -0700 Subject: [PATCH 15/22] Update CMakeIntegratedOpenCL.cmake Co-authored-by: Nikita Titov --- CMakeIntegratedOpenCL.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeIntegratedOpenCL.cmake b/CMakeIntegratedOpenCL.cmake index 37b57089cda0..ab9b6fbdfb7d 100644 --- a/CMakeIntegratedOpenCL.cmake +++ b/CMakeIntegratedOpenCL.cmake @@ -61,7 +61,7 @@ ExternalProject_Add(Boost INSTALL_COMMAND "" ) set(BOOST_INCLUDE "${BOOST_BASE}/source" CACHE PATH "") -set(BOOST_LIBRARY ${BOOST_BASE}/source/stage/lib CACHE PATH "") +set(BOOST_LIBRARY "${BOOST_BASE}/source/stage/lib" CACHE PATH "") list(APPEND INTEGRATED_OPENCL_INCLUDES ${BOOST_INCLUDE}) if(MSVC) if(${MSVC_VERSION} GREATER 1929) From 8a239014195914e68ab5efcbfbff12a3b92a8ee7 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Mon, 14 Sep 2020 11:56:41 -0700 Subject: [PATCH 16/22] Update CMakeIntegratedOpenCL.cmake Co-authored-by: Nikita Titov --- CMakeIntegratedOpenCL.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeIntegratedOpenCL.cmake b/CMakeIntegratedOpenCL.cmake index ab9b6fbdfb7d..5898c08ad36c 100644 --- a/CMakeIntegratedOpenCL.cmake +++ b/CMakeIntegratedOpenCL.cmake @@ -65,7 +65,7 @@ set(BOOST_LIBRARY "${BOOST_BASE}/source/stage/lib" CACHE PATH "") list(APPEND INTEGRATED_OPENCL_INCLUDES ${BOOST_INCLUDE}) if(MSVC) if(${MSVC_VERSION} GREATER 1929) - message(FATAL_ERROR "Unrecognized MSVC version number") + message(FATAL_ERROR "Unrecognized MSVC version number: ${MSVC_VERSION}") elseif(${MSVC_VERSION} GREATER 1919) set(MSVC_TOOLCHAIN_ID "142") elseif(${MSVC_VERSION} GREATER 1909) From 8f140dc0177b77bd63fb0ab97f1f0a3d11284a6a Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Mon, 14 Sep 2020 11:57:05 -0700 Subject: [PATCH 17/22] Update CMakeIntegratedOpenCL.cmake Co-authored-by: Nikita Titov --- CMakeIntegratedOpenCL.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeIntegratedOpenCL.cmake b/CMakeIntegratedOpenCL.cmake index 5898c08ad36c..294afe733a74 100644 --- a/CMakeIntegratedOpenCL.cmake +++ b/CMakeIntegratedOpenCL.cmake @@ -73,7 +73,7 @@ if(MSVC) elseif(${MSVC_VERSION} GREATER 1899) set(MSVC_TOOLCHAIN_ID "140") else() - message(FATAL_ERROR "Unrecognized MSVC version number") + message(FATAL_ERROR "Unrecognized MSVC version number: ${MSVC_VERSION}") endif() list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib) From f654d9106680b9528043a9d0e84822b239469cbc Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Mon, 14 Sep 2020 11:59:11 -0700 Subject: [PATCH 18/22] Update CMakeLists.txt Co-authored-by: Nikita Titov --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 08ad055edd95..f35ba371391c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,7 +26,7 @@ endif(APPLE) if(__INTEGRATE_OPENCL) set(__INTEGRATE_OPENCL ON CACHE BOOL "" FORCE) set(USE_GPU OFF CACHE BOOL "" FORCE) - message(WARNING "WARNING: building library with integrated OpenCL components") + message(STATUS "Building library with integrated OpenCL components") endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") From 259754949bd9dedcc7fa44c3298310b5cf511d06 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Mon, 14 Sep 2020 11:59:41 -0700 Subject: [PATCH 19/22] Update CMakeLists.txt Co-authored-by: Nikita Titov --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f35ba371391c..863584efd016 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -133,13 +133,13 @@ if(USE_GPU) endif(USE_GPU) if(__INTEGRATE_OPENCL) - if (WIN32) + if(WIN32) include(CMakeIntegratedOpenCL.cmake) ADD_DEFINITIONS(-DUSE_GPU) else() message(FATAL_ERROR "Integrated OpenCL build is available only for Windows") - endif() -endif() + endif(WIN32) +endif(__INTEGRATE_OPENCL) if(USE_HDFS) find_package(JNI REQUIRED) From db60d59f41ae36cf1cdc08e3f8444afa380e9b62 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Mon, 14 Sep 2020 11:59:58 -0700 Subject: [PATCH 20/22] Update CMakeLists.txt Co-authored-by: Nikita Titov --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 863584efd016..df6861fbc66c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -340,13 +340,13 @@ if(USE_HDFS) endif(USE_HDFS) if(WIN32) - if (MINGW OR CYGWIN) + if(MINGW OR CYGWIN) TARGET_LINK_LIBRARIES(lightgbm Ws2_32) TARGET_LINK_LIBRARIES(_lightgbm Ws2_32) TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI) TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI) - endif() -endif() + endif(MINGW OR CYGWIN) +endif(WIN32) if(BUILD_FOR_R) if(MSVC) From e274c8ef983a257ceb67600fe2ec3d1b4081c2c5 Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Mon, 14 Sep 2020 12:02:47 -0700 Subject: [PATCH 21/22] Update CMakeIntegratedOpenCL.cmake Targeted download of Boost submodules. Co-authored-by: Nikita Titov --- CMakeIntegratedOpenCL.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeIntegratedOpenCL.cmake b/CMakeIntegratedOpenCL.cmake index 294afe733a74..8673f3bdead2 100644 --- a/CMakeIntegratedOpenCL.cmake +++ b/CMakeIntegratedOpenCL.cmake @@ -42,7 +42,7 @@ set(BOOST_BASE "${PROJECT_BINARY_DIR}/Boost") set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.bat") set(BOOST_BUILD "${BOOST_BASE}/source/b2.exe") set(BOOST_FLAGS "") -list(APPEND BOOST_SUBMODULES "libs/*" "tools/*") +list(APPEND BOOST_SUBMODULES "libs/algorithm" "libs/align" "libs/any" "libs/array" "libs/assert" "libs/bind" "libs/chrono" "libs/compute" "libs/concept_check" "libs/config" "libs/container" "libs/container_hash" "libs/core" "libs/detail" "libs/filesystem" "libs/foreach" "libs/format" "libs/function" "libs/function_types" "libs/fusion" "libs/headers" "libs/integer" "libs/io" "libs/iterator" "libs/lexical_cast" "libs/math" "libs/move" "libs/mpl" "libs/multi_index" "libs/numeric/conversion" "libs/optional" "libs/predef" "libs/preprocessor" "libs/property_tree" "libs/range" "libs/ratio" "libs/serialization" "libs/smart_ptr" "libs/static_assert" "libs/system" "libs/throw_exception" "libs/tuple" "libs/typeof" "libs/type_index" "libs/type_traits" "libs/utility" "libs/uuid" "libs/winapi" "tools/boost_install" "tools/build") ExternalProject_Add(Boost TMP_DIR "${BOOST_BASE}/tmp" STAMP_DIR "${BOOST_BASE}/stamp" From 6acfd267207695fad3534a8f4295012c1064aefb Mon Sep 17 00:00:00 2001 From: TP Boudreau Date: Thu, 17 Sep 2020 13:30:29 -0700 Subject: [PATCH 22/22] Update CMakeLists.txt Co-authored-by: Nikita Titov --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d77f6f44a3fd..dddaad4beaf8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -17,7 +17,7 @@ OPTION(USE_TIMETAG "Set to ON to output time costs" OFF) OPTION(USE_DEBUG "Set to ON for Debug mode" OFF) OPTION(BUILD_STATIC_LIB "Build static library" OFF) OPTION(__BUILD_FOR_R "Set to ON if building lib_lightgbm for use with the R package" OFF) -OPTION(__INTEGRATE_OPENCL "Set to ON if building lib_lightgbm with the OpenCL ICD Loader and its dependencies included" OFF) +OPTION(__INTEGRATE_OPENCL "Set to ON if building LightGBM with the OpenCL ICD Loader and its dependencies included" OFF) if(APPLE) OPTION(APPLE_OUTPUT_DYLIB "Output dylib shared library" OFF)