Build integrated Python package library

CMakeLists.txt

@@ -1,4 +1,6 @@
-if(USE_GPU OR APPLE)
+if(__OPENCL_PYTHON_PACKAGE)
+  cmake_minimum_required(VERSION 3.11)
+elseif(USE_GPU OR APPLE)
   cmake_minimum_required(VERSION 3.2)
 else()
   cmake_minimum_required(VERSION 2.8)
@@ -20,6 +22,20 @@ if(APPLE)
     OPTION(APPLE_OUTPUT_DYLIB "Output dylib shared library" OFF)
 endif(APPLE)
 
+if(__OPENCL_PYTHON_PACKAGE)
+  set(__OPENCL_PYTHON_PACKAGE ON CACHE BOOL "" FORCE)
+  set(USE_GPU OFF CACHE BOOL "" FORCE)
+  set(USE_MPI OFF CACHE BOOL "")
+  set(USE_OPENMP ON CACHE BOOL "")
+  set(USE_SWIG OFF CACHE BOOL "")
+  set(USE_HDFS OFF CACHE BOOL "")
+  set(USE_R35 OFF CACHE BOOL "")
+  set(USE_TIMETAG OFF CACHE BOOL "")
+  set(USE_DEBUG OFF CACHE BOOL "")
+  set(BUILD_FOR_R OFF CACHE BOOL "")
+  message(WARNING "WARNING: building integrated OpenCL enabled components intended for Python package")
+endif()
+
 if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
   if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.8.2")
     message(FATAL_ERROR "Insufficient gcc version")
@@ -123,6 +139,15 @@ if(USE_GPU)
     ADD_DEFINITIONS(-DUSE_GPU)
 endif(USE_GPU)
 
+if(__OPENCL_PYTHON_PACKAGE)
+    if (WIN32)
+        include(CMakeOpenCLPythonPackage.cmake)
+        ADD_DEFINITIONS(-DUSE_GPU)
+    else()
+        message(FATAL_ERROR "Integrated OpenCL build available only on WIN32")
+    endif()
+endif()
+
 if(USE_HDFS)
     find_package(JNI REQUIRED)
     find_path(HDFS_INCLUDE_DIR hdfs.h REQUIRED)
@@ -246,7 +271,7 @@ else()
 endif(BUILD_STATIC_LIB)
 
 if(MSVC)
-    set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")
+  set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")
 endif(MSVC)
 
 if(USE_SWIG)
@@ -303,16 +328,35 @@ if(USE_GPU)
   TARGET_LINK_LIBRARIES(_lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
 endif(USE_GPU)
 
+if(__OPENCL_PYTHON_PACKAGE)
+  # targets OpenCL and Boost are added in CMakeOpenCLPythonPackage.cmake
+  add_dependencies(lightgbm OpenCL Boost)
+  add_dependencies(_lightgbm OpenCL Boost)
+  # variables OPENCL_PYTHON_PACKAGE_* are set in CMakeOpenCLPythonPackage.cmake
+  target_include_directories(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_INCLUDES})
+  target_include_directories(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_INCLUDES})
+  target_compile_definitions(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_DEFINITIONS})
+  target_compile_definitions(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_DEFINITIONS})
+  target_link_libraries(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_LIBRARIES})
+  target_link_libraries(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_LIBRARIES})
+endif()
+
 if(USE_HDFS)
   TARGET_LINK_LIBRARIES(lightgbm ${HDFS_CXX_LIBRARIES})
   TARGET_LINK_LIBRARIES(_lightgbm ${HDFS_CXX_LIBRARIES})
 endif(USE_HDFS)
 
-if(WIN32 AND (MINGW OR CYGWIN))
-    TARGET_LINK_LIBRARIES(lightgbm Ws2_32)
-    TARGET_LINK_LIBRARIES(_lightgbm Ws2_32)
-    TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI)
-    TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI)
+if(WIN32)
+    if (MINGW OR CYGWIN)
+      TARGET_LINK_LIBRARIES(lightgbm Ws2_32)
+      TARGET_LINK_LIBRARIES(_lightgbm Ws2_32)
+      TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI)
+      TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI)
+    endif()
+else()
+    if(__OPENCL_PYTHON_PACKAGE)
+      target_link_libraries(lightgbm PRIVATE pthread dl)
+    endif()
 endif()
 
 if(BUILD_FOR_R)

CMakeOpenCLPythonPackage.cmake

@@ -0,0 +1,90 @@
+
+set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
+set(BOOST_VERSION_DOT "1.73")
+string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT})
+
+set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git")
+set(OPENCL_HEADER_TAG "1b2a1850f410aaaaeaa56cead5a179b5aea4918e")
+
+set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git")
+set(OPENCL_LOADER_TAG "862eebe7ca733c398334a8db8481172a7d3a3c47")
+
+set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git")
+set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0")
+execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE LIGHTGBM_TAG OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
+string(SHA1 OPENCL_PYTHON_PACKAGE_STAMP "${OPENCL_HEADER_REPOSITORY}@${OPENCL_HEADER_TAG};${OPENCL_LOADER_REPOSITORY}@${OPENCL_LOADER_TAG};${BOOST_REPOSITORY}@${BOOST_TAG};lightgbm@${LIGHTGBM_TAG}")
+message(STATUS "OpenCL Python package build stamp: ${OPENCL_PYTHON_PACKAGE_STAMP}")
+
+# Build Independent OpenCL library
+include(FetchContent)
+FetchContent_Declare(OpenCL-Headers GIT_REPOSITORY ${OPENCL_HEADER_REPOSITORY} GIT_TAG ${OPENCL_HEADER_TAG})
+FetchContent_GetProperties(OpenCL-Headers)
+if(NOT OpenCL-Headers_POPULATED)
+  FetchContent_Populate(OpenCL-Headers)
+  message(STATUS "Populated OpenCL Headers")
+endif()
+set(OPENCL_ICD_LOADER_HEADERS_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for OpenCL ICD Loader
+set(OpenCL_INCLUDE_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for Boost::Compute
+
+FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG})
+FetchContent_GetProperties(OpenCL-ICD-Loader)
+if(NOT OpenCL-ICD-Loader_POPULATED)
+  FetchContent_Populate(OpenCL-ICD-Loader)
+  set(USE_DYNAMIC_VCXX_RUNTIME ON)
+  add_subdirectory(${opencl-icd-loader_SOURCE_DIR} ${opencl-icd-loader_BINARY_DIR} EXCLUDE_FROM_ALL)
+  message(STATUS "Populated OpenCL ICD Loader")
+endif()
+list(APPEND OPENCL_PYTHON_PACKAGE_INCLUDES ${OPENCL_ICD_LOADER_HEADERS_DIR})
+list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/Release/OpenCL.lib cfgmgr32.lib runtimeobject.lib)
+list(APPEND OPENCL_PYTHON_PACKAGE_DEFINITIONS CL_TARGET_OPENCL_VERSION=120)
+
+# Build Independent Boost libraries
+include(ExternalProject)
+include(ProcessorCount)
+ProcessorCount(J)
+set(BOOST_BASE "${PROJECT_BINARY_DIR}/Boost")
+set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.bat")
+set(BOOST_BUILD "${BOOST_BASE}/source/b2.exe")
+set(BOOST_FLAGS "")
+list(APPEND BOOST_SUBMODULES "libs/*" "tools/*")
+ExternalProject_Add(Boost
+  TMP_DIR "${BOOST_BASE}/tmp"
+  STAMP_DIR "${BOOST_BASE}/stamp"
+  DOWNLOAD_DIR "${BOOST_BASE}/download"
+  SOURCE_DIR "${BOOST_BASE}/source"
+  BINARY_DIR "${BOOST_BASE}/source"
+  INSTALL_DIR "${BOOST_BASE}/install"
+  GIT_REPOSITORY ${BOOST_REPOSITORY}
+  GIT_TAG ${BOOST_TAG}
+  GIT_SUBMODULES ${BOOST_SUBMODULES}
+  GIT_SHALLOW ON
+  UPDATE_COMMAND ""
+  PATCH_COMMAND ""
+  CONFIGURE_COMMAND ${BOOST_BOOTSTRAP}
+  BUILD_COMMAND ${BOOST_BUILD} -sBOOST_ROOT=${BOOST_BASE}/source -a -q -j ${J} --with-headers --with-chrono --with-filesystem --with-system link=static runtime-link=shared variant=release threading=multi cxxflags="${BOOST_FLAGS}"
+  INSTALL_COMMAND ""
+)
+set(BOOST_INCLUDE ${BOOST_BASE}/source CACHE PATH "")
+set(BOOST_LIBRARY ${BOOST_BASE}/source/stage/lib CACHE PATH "")
+list(APPEND OPENCL_PYTHON_PACKAGE_INCLUDES ${BOOST_INCLUDE})
+if(MSVC)
+  if(${MSVC_VERSION} GREATER 1929)
+    set(MSVC_TOOLCHAIN_ID "unknown")
+  elseif(${MSVC_VERSION} GREATER 1919)
+    set(MSVC_TOOLCHAIN_ID "142")
+  elseif(${MSVC_VERSION} GREATER 1909)
+    set(MSVC_TOOLCHAIN_ID "141")
+  elseif(${MSVC_VERSION} GREATER 1899)
+    set(MSVC_TOOLCHAIN_ID "140")
+  else()
+    set(MSVC_TOOLCHAIN_ID "unknown")
+  endif()
+  list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib)
+  list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib)
+  list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib)
+else()
+  message(WARNING "MinGW Boost library names not yet specified")
+endif()
+
+set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE)
+

python-package/setup.py

@@ -29,7 +29,7 @@ def find_lib():
     return LIB_PATH
 
 
-def copy_files(use_gpu=False):
+def copy_files(opencl_python_package=False, use_gpu=False):
 
     def copy_files_helper(folder_name):
         src = os.path.join(CURRENT_DIR, os.path.pardir, folder_name)
@@ -51,14 +51,18 @@ def copy_files_helper(folder_name):
         copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.vcxproj"),
                   os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.vcxproj"),
                   verbose=0)
-        if use_gpu:
-            copy_files_helper('compute')
-        copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"),
-                  os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"),
-                  verbose=0)
         copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "LICENSE"),
                   os.path.join(CURRENT_DIR, "LICENSE"),
                   verbose=0)
+        copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"),
+                  os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"),
+                  verbose=0)
+        if opencl_python_package:
+            copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeOpenCLPythonPackage.cmake"),
+                      os.path.join(CURRENT_DIR, "compile", "CMakeOpenCLPythonPackage.cmake"),
+                      verbose=0)
+        if use_gpu:
+            copy_files_helper('compute')
 
 
 def clear_path(path):
@@ -91,7 +95,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False,
                 use_hdfs=False, boost_root=None, boost_dir=None,
                 boost_include_dir=None, boost_librarydir=None,
                 opencl_include_dir=None, opencl_library=None,
-                nomp=False, bit32=False):
+                nomp=False, bit32=False, opencl_python_package=False):
 
     if os.path.exists(os.path.join(CURRENT_DIR, "build_cpp")):
         shutil.rmtree(os.path.join(CURRENT_DIR, "build_cpp"))
@@ -101,6 +105,12 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False,
     logger.info("Starting to compile the library.")
 
     cmake_cmd = ["cmake", "../compile/"]
+    if opencl_python_package:
+        use_gpu = False
+        use_mpi = False
+        nomp = False
+        use_hdfs = False
+        cmake_cmd.append("-D__OPENCL_PYTHON_PACKAGE=ON")
     if use_gpu:
         cmake_cmd.append("-DUSE_GPU=ON")
         if boost_root:
@@ -134,7 +144,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False,
         else:
             status = 1
             lib_path = os.path.join(CURRENT_DIR, "compile", "windows", "x64", "DLL", "lib_lightgbm.dll")
-            if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32)):
+            if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32, opencl_python_package)):
                 logger.info("Starting to compile with MSBuild from existing solution file.")
                 platform_toolsets = ("v142", "v141", "v140")
                 for pt in platform_toolsets:
@@ -187,6 +197,7 @@ class CustomInstall(install):
 
     user_options = install.user_options + [
         ('mingw', 'm', 'Compile with MinGW'),
+        ('opencl-python-package', None, 'Compile integrated OpenCL version'),
         ('gpu', 'g', 'Compile GPU version'),
         ('mpi', None, 'Compile MPI version'),
         ('nomp', None, 'Compile version without OpenMP support'),
@@ -204,6 +215,7 @@ class CustomInstall(install):
     def initialize_options(self):
         install.initialize_options(self)
         self.mingw = 0
+        self.opencl_python_package = 0
         self.gpu = 0
         self.boost_root = None
         self.boost_dir = None
@@ -227,12 +239,12 @@ def run(self):
                                 "please use 64-bit Python instead.")
         open(LOG_PATH, 'wb').close()
         if not self.precompile:
-            copy_files(use_gpu=self.gpu)
+            copy_files(opencl_python_package=self.opencl_python_package, use_gpu=self.gpu)
             compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_mpi=self.mpi,
                         use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir,
                         boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
                         opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library,
-                        nomp=self.nomp, bit32=self.bit32)
+                        nomp=self.nomp, bit32=self.bit32, opencl_python_package=self.opencl_python_package)
         install.run(self)
         if os.path.isfile(LOG_PATH):
             os.remove(LOG_PATH)
@@ -241,7 +253,7 @@ def run(self):
 class CustomSdist(sdist):
 
     def run(self):
-        copy_files(use_gpu=True)
+        copy_files(opencl_python_package=True, use_gpu=True)
         open(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'), 'w').close()
         if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')):
             shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'Release'))