Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Build integrated Python package library #3144

Merged
merged 31 commits into from
Sep 21, 2020
Merged

Build integrated Python package library #3144

Show file tree
Hide file tree
Changes from 18 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
b1841bd
Add specialized OpenCL/Python package build path
tpboudreau May 27, 2020
27dffcb
Refer to upstream OpenCL repository
tpboudreau Jun 2, 2020
2958084
Reset build job count in setup.py
tpboudreau Jun 3, 2020
519a5b9
TEMPORARY: refer to OpenCL fork to ensure Linux CI builds succeed
tpboudreau Jun 3, 2020
84b6804
Remove intermediate cmake target
tpboudreau Jun 3, 2020
ec4b3f4
Restrict OpenCL headers to documented API version
tpboudreau Jun 4, 2020
1fc7878
Use command line definition to activate integrated build
tpboudreau Jun 8, 2020
8e66b4e
Merge branch 'master' into build_python_package_library
tpboudreau Jun 9, 2020
af176a6
Flag reference to unofficial repo with FIXME
tpboudreau Jun 9, 2020
e0c71cb
Merge branch 'master' into build_python_package_library
tpboudreau Jun 30, 2020
81e6773
Merge branch 'master' into build_python_package_library
tpboudreau Aug 10, 2020
4dac9f5
Merge branch 'master' into build_python_package_library (with conflic…
tpboudreau Aug 27, 2020
180c929
TEMPORARY: update private repo tag for dependency
tpboudreau Aug 27, 2020
80318e0
Merge branch 'master' into build_python_package_library
tpboudreau Sep 3, 2020
1782aed
Remove integrated build for non-Win32 and related cleanup
tpboudreau Sep 3, 2020
3013f4d
Remove commented code
tpboudreau Sep 3, 2020
9d0614c
Rename integrated OpenCL build option and other cleanups
tpboudreau Sep 8, 2020
e567f33
Merge branch 'master' into build_python_package_library
tpboudreau Sep 8, 2020
d5fad62
Small cleanups
tpboudreau Sep 14, 2020
dc5ef75
Update CMakeIntegratedOpenCL.cmake
tpboudreau Sep 14, 2020
4fd05bc
Update CMakeIntegratedOpenCL.cmake
tpboudreau Sep 14, 2020
8a23901
Update CMakeIntegratedOpenCL.cmake
tpboudreau Sep 14, 2020
8f140dc
Update CMakeIntegratedOpenCL.cmake
tpboudreau Sep 14, 2020
f654d91
Update CMakeLists.txt
tpboudreau Sep 14, 2020
2597549
Update CMakeLists.txt
tpboudreau Sep 14, 2020
db60d59
Update CMakeLists.txt
tpboudreau Sep 14, 2020
e274c8e
Update CMakeIntegratedOpenCL.cmake
tpboudreau Sep 14, 2020
e4d2e3a
Merge from master, with conflict resolution
tpboudreau Sep 14, 2020
6acfd26
Update CMakeLists.txt
tpboudreau Sep 17, 2020
ee9f545
Merge branch 'master' into build_python_package_library
tpboudreau Sep 17, 2020
5d85aa1
Merge from master, with conflict resolution
tpboudreau Sep 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 88 additions & 0 deletions CMakeIntegratedOpenCL.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
set(BOOST_VERSION_DOT "1.74")
string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT})

set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git")
set(OPENCL_HEADER_TAG "1b2a1850f410aaaaeaa56cead5a179b5aea4918e")

set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git")
set(OPENCL_LOADER_TAG "862eebe7ca733c398334a8db8481172a7d3a3c47")
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved

set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git")
set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0")
execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE LIGHTGBM_TAG OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
string(SHA1 INTEGRATED_OPENCL_STAMP "${OPENCL_HEADER_REPOSITORY}@${OPENCL_HEADER_TAG};${OPENCL_LOADER_REPOSITORY}@${OPENCL_LOADER_TAG};${BOOST_REPOSITORY}@${BOOST_TAG};lightgbm@${LIGHTGBM_TAG}")
message(STATUS "Integrated OpenCL build stamp: ${INTEGRATED_OPENCL_STAMP}")
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved

# Build Independent OpenCL library
include(FetchContent)
FetchContent_Declare(OpenCL-Headers GIT_REPOSITORY ${OPENCL_HEADER_REPOSITORY} GIT_TAG ${OPENCL_HEADER_TAG})
FetchContent_GetProperties(OpenCL-Headers)
if(NOT OpenCL-Headers_POPULATED)
FetchContent_Populate(OpenCL-Headers)
message(STATUS "Populated OpenCL Headers")
endif()
set(OPENCL_ICD_LOADER_HEADERS_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for OpenCL ICD Loader
set(OpenCL_INCLUDE_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for Boost::Compute

FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG})
FetchContent_GetProperties(OpenCL-ICD-Loader)
if(NOT OpenCL-ICD-Loader_POPULATED)
FetchContent_Populate(OpenCL-ICD-Loader)
set(USE_DYNAMIC_VCXX_RUNTIME ON)
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
add_subdirectory(${opencl-icd-loader_SOURCE_DIR} ${opencl-icd-loader_BINARY_DIR} EXCLUDE_FROM_ALL)
message(STATUS "Populated OpenCL ICD Loader")
endif()
list(APPEND INTEGRATED_OPENCL_INCLUDES ${OPENCL_ICD_LOADER_HEADERS_DIR})
list(APPEND INTEGRATED_OPENCL_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/Release/OpenCL.lib cfgmgr32.lib runtimeobject.lib)
list(APPEND INTEGRATED_OPENCL_DEFINITIONS CL_TARGET_OPENCL_VERSION=120)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you please comment on this from compatibility side? Why 1.2?

Copy link
Contributor Author

@tpboudreau tpboudreau Sep 14, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This appears to be the latest earliest stable version that contains all the required api's (and 2.x seems to have been a long experiment: https://www.extremetech.com/computing/309842-opencl-3-0-kicks-off-with-a-huge-step-backwards).

But the library compiles without specifying the version (with a few extra messages), so we can remove this if you prefer.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the clarification! I'm afraid I have too little knowledge about OpenCL to take a responsibility for this change. I'd better leave it for your decision or more experienced in OpenCL maintainer.

I just want to make it possible that as many as possible users will be able to simply download our wheel and run LightGBM with GPU support.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, including the 1.2 guard when you compile the headers prevents you from inadvertently including a call to a 2.x function, which would limit the library to newer implementations. OpenCL versions are backward compatible, so a 2.x implementation can run a 1.2 application/library. (See item 3 here for example: http://developer.amd.com/wordpress/media/2013/12/AMD_APP_SDK_FAQ1.pdf)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Excellent! Thanks again!


# Build Independent Boost libraries
include(ExternalProject)
include(ProcessorCount)
ProcessorCount(J)
set(BOOST_BASE "${PROJECT_BINARY_DIR}/Boost")
set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.bat")
set(BOOST_BUILD "${BOOST_BASE}/source/b2.exe")
set(BOOST_FLAGS "")
list(APPEND BOOST_SUBMODULES "libs/*" "tools/*")
Copy link
Collaborator

@StrikerRUS StrikerRUS Sep 13, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess we can workaround 1.5Gb+ size problem by specifying only required submodules here.

IIRC the issue is that all boost libraries were being cloned even though only a handful were needed and I couldn't figure out a way to get around this.
#3144 (comment)

Starting from chrono, filesystem, headers, system, tools/* by trials and errors I came up with the following list which allows to compile successfully, but I bet you know better which submodules are needed. Also, I believe we can reduce number of submodules under the tools folder.

Suggested change
list(APPEND BOOST_SUBMODULES "libs/*" "tools/*")
list(APPEND BOOST_SUBMODULES "libs/algorithm" "libs/align" "libs/any" "libs/array" "libs/assert" "libs/bind" "libs/chrono" "libs/compute" "libs/concept_check" "libs/config" "libs/container" "libs/container_hash" "libs/core" "libs/detail" "libs/filesystem" "libs/foreach" "libs/format" "libs/function" "libs/function_types" "libs/fusion" "libs/headers" "libs/integer" "libs/io" "libs/iterator" "libs/lexical_cast" "libs/math" "libs/move" "libs/mpl" "libs/multi_index" "libs/numeric/conversion" "libs/optional" "libs/predef" "libs/preprocessor" "libs/property_tree" "libs/range" "libs/ratio" "libs/serialization" "libs/smart_ptr" "libs/static_assert" "libs/system" "libs/throw_exception" "libs/tuple" "libs/typeof" "libs/type_index" "libs/type_traits" "libs/utility" "libs/uuid" "libs/winapi" "tools/boost_install" "tools/build")

I know, this list is looking super scary, but explicitly listing submodules it is possible to decrease build time and downloading size significantly (1.5Gb+ -> 700Mb)

Copy link
Collaborator

@StrikerRUS StrikerRUS Sep 13, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, I believe we can reduce number of submodules under the tools folder.

OK, I've experimented more and it looks like we do not need submodules from tools/ at all. I'm updating my yesterday original comment with new suggestion.

UPD: Was wrong. tools/boost_install and tools/build are needed.

Copy link
Contributor Author

@tpboudreau tpboudreau Sep 14, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@StrikerRUS , thank you for doing this exploration.

I'm not opposed to this change, and I've committed the suggested patch, but I think there are some considerations the maintainers should be aware of.

One of the challenges in this patch was coaxing three different build systems (Boost's B2, cmake, and MSVC/make) to cooperate. And as you know I was originally targeting more than just the Windows platform. I found that the Boost dependency list generated at build time differed significantly by platform -- on Linux (generally) only the few expected dependent submodules were built, on Windows (generally), many more submodules were fetched. This was a bit surprising, but I didn't spend a lot of time chasing down why, because it seemed to me that downloading all submodules was always safe, and that even if I figured it all out for each platform, it would lead to complicated platform checking code that might turn out to be fragile in the long run (meaning a risk of not working anymore if the Boost libs or any of the three build systems changed or started interacting differently).

Of course I realize that there may be costs associated with fetching and processing code unecessarily in the CI pipeline, and also that non-Windows builds may never be needed, so I'm OK with the suggested change. I just wanted to alert you all that you may have to rethink this targeted download approach if other platforms are added or the build becomes unstable.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OK, I understand! Please correct me: as long as we fix a particular commit in the boost repository and building the library only for Windows users, it is safe to go with my list. In future we can update this list to support building on Linux machines or add conditioning if(WIN32) ... else() ... or fall back to downloading all submodules for all OSes.

Despite that we are using only free CI services (BTW, it will be not true anymore after introducing CUDA support #3160), we build nightly artifacts for each commit in master and in general I believe it is good practice to download things only that are needed.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not related to this PR, but just my curiosity. Don't you know were there any changes in macOS related to OpenCL support? Refer to #1523 (comment) and #667. I believe some macOS users with eGPUs will benefit a lot.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm sorry, but I'm not too familiar with the details of OpenCL on MacOS. But the #667 issue you linked to seems (at first glance) like it could be resolved by an approach similar to this PR. I'll look into it further when I get a bit of time.

ExternalProject_Add(Boost
TMP_DIR "${BOOST_BASE}/tmp"
STAMP_DIR "${BOOST_BASE}/stamp"
DOWNLOAD_DIR "${BOOST_BASE}/download"
SOURCE_DIR "${BOOST_BASE}/source"
BINARY_DIR "${BOOST_BASE}/source"
INSTALL_DIR "${BOOST_BASE}/install"
GIT_REPOSITORY ${BOOST_REPOSITORY}
GIT_TAG ${BOOST_TAG}
GIT_SUBMODULES ${BOOST_SUBMODULES}
GIT_SHALLOW ON
UPDATE_COMMAND ""
PATCH_COMMAND ""
CONFIGURE_COMMAND ${BOOST_BOOTSTRAP}
BUILD_COMMAND ${BOOST_BUILD} -sBOOST_ROOT=${BOOST_BASE}/source -a -q -j ${J} --with-headers --with-chrono --with-filesystem --with-system link=static runtime-link=shared variant=release threading=multi cxxflags="${BOOST_FLAGS}"
INSTALL_COMMAND ""
)
set(BOOST_INCLUDE ${BOOST_BASE}/source CACHE PATH "")
tpboudreau marked this conversation as resolved.
Show resolved Hide resolved
set(BOOST_LIBRARY ${BOOST_BASE}/source/stage/lib CACHE PATH "")
tpboudreau marked this conversation as resolved.
Show resolved Hide resolved
list(APPEND INTEGRATED_OPENCL_INCLUDES ${BOOST_INCLUDE})
if(MSVC)
if(${MSVC_VERSION} GREATER 1929)
message(FATAL_ERROR "Unrecognized MSVC version number")
tpboudreau marked this conversation as resolved.
Show resolved Hide resolved
elseif(${MSVC_VERSION} GREATER 1919)
set(MSVC_TOOLCHAIN_ID "142")
elseif(${MSVC_VERSION} GREATER 1909)
set(MSVC_TOOLCHAIN_ID "141")
elseif(${MSVC_VERSION} GREATER 1899)
set(MSVC_TOOLCHAIN_ID "140")
else()
message(FATAL_ERROR "Unrecognized MSVC version number")
tpboudreau marked this conversation as resolved.
Show resolved Hide resolved
endif()
list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib)
list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib)
list(APPEND INTEGRATED_OPENCL_LIBRARIES ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib)
else()
message(FATAL_ERROR "MinGW Boost library names not yet specified")
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you please comment on this? Is it related to KhronosGroup/OpenCL-ICD-Loader#11 or MinGW support can be just added later in another PR?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a terrible message, I'm sorry for that -- I'm pretty sure it was just a placeholder I forgot to change.

The underlying issue that because MinGW installations are so varied, it's more difficult to predict the name and location of the .lib files. So this should be addressed in a separate PR.

I've changed the message to show that MinGW is not supported yet.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Got it! Thank you!

I'm not sure but it seems that it even doesn't worth to implement MinGW support. Anyway, we will create nightly and release wheels under VS for the performance purposes (https://lightgbm.readthedocs.io/en/latest/FAQ.html#i-am-using-windows-should-i-use-visual-studio-or-mingw-for-compiling-lightgbm). As long as compiled with VS library doesn't conflict with MinGW (have no idea how can it happen) and this option remains "private" (for repo maintainers or very advanced third-party developers), MinGW support is not needed, I believe. Please correct me if I'm wrong.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm far from an expert, but my impression is also that MinGW support is significantly less urgent since MSFT began making the MSVC community edition available for free.

endif()

set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE)
46 changes: 39 additions & 7 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
if(USE_GPU OR APPLE)
if(__INTEGRATE_OPENCL)
cmake_minimum_required(VERSION 3.11)
elseif(USE_GPU OR APPLE)
cmake_minimum_required(VERSION 3.2)
else()
cmake_minimum_required(VERSION 2.8)
Expand All @@ -20,6 +22,12 @@ if(APPLE)
OPTION(APPLE_OUTPUT_DYLIB "Output dylib shared library" OFF)
endif(APPLE)

if(__INTEGRATE_OPENCL)
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
set(__INTEGRATE_OPENCL ON CACHE BOOL "" FORCE)
set(USE_GPU OFF CACHE BOOL "" FORCE)
message(WARNING "WARNING: building library with integrated OpenCL components")
tpboudreau marked this conversation as resolved.
Show resolved Hide resolved
endif()

if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.8.2")
message(FATAL_ERROR "Insufficient gcc version")
Expand Down Expand Up @@ -123,6 +131,15 @@ if(USE_GPU)
ADD_DEFINITIONS(-DUSE_GPU)
endif(USE_GPU)

if(__INTEGRATE_OPENCL)
if (WIN32)
include(CMakeIntegratedOpenCL.cmake)
ADD_DEFINITIONS(-DUSE_GPU)
else()
message(FATAL_ERROR "Integrated OpenCL build is available only for Windows")
endif()
endif()
tpboudreau marked this conversation as resolved.
Show resolved Hide resolved

if(USE_HDFS)
find_package(JNI REQUIRED)
find_path(HDFS_INCLUDE_DIR hdfs.h REQUIRED)
Expand Down Expand Up @@ -246,7 +263,7 @@ else()
endif(BUILD_STATIC_LIB)

if(MSVC)
set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")
set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")
endif(MSVC)

if(USE_SWIG)
Expand Down Expand Up @@ -303,16 +320,31 @@ if(USE_GPU)
TARGET_LINK_LIBRARIES(_lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
endif(USE_GPU)

if(__INTEGRATE_OPENCL)
# targets OpenCL and Boost are added in CMakeIntegratedOpenCL.cmake
add_dependencies(lightgbm OpenCL Boost)
add_dependencies(_lightgbm OpenCL Boost)
# variables INTEGRATED_OPENCL_* are set in CMakeIntegratedOpenCL.cmake
target_include_directories(lightgbm PRIVATE ${INTEGRATED_OPENCL_INCLUDES})
target_include_directories(_lightgbm PRIVATE ${INTEGRATED_OPENCL_INCLUDES})
target_compile_definitions(lightgbm PRIVATE ${INTEGRATED_OPENCL_DEFINITIONS})
target_compile_definitions(_lightgbm PRIVATE ${INTEGRATED_OPENCL_DEFINITIONS})
target_link_libraries(lightgbm PRIVATE ${INTEGRATED_OPENCL_LIBRARIES})
target_link_libraries(_lightgbm PRIVATE ${INTEGRATED_OPENCL_LIBRARIES})
endif()

if(USE_HDFS)
TARGET_LINK_LIBRARIES(lightgbm ${HDFS_CXX_LIBRARIES})
TARGET_LINK_LIBRARIES(_lightgbm ${HDFS_CXX_LIBRARIES})
endif(USE_HDFS)

if(WIN32 AND (MINGW OR CYGWIN))
TARGET_LINK_LIBRARIES(lightgbm Ws2_32)
TARGET_LINK_LIBRARIES(_lightgbm Ws2_32)
TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI)
TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI)
if(WIN32)
if (MINGW OR CYGWIN)
TARGET_LINK_LIBRARIES(lightgbm Ws2_32)
TARGET_LINK_LIBRARIES(_lightgbm Ws2_32)
TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI)
TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI)
endif()
endif()
tpboudreau marked this conversation as resolved.
Show resolved Hide resolved

if(BUILD_FOR_R)
Expand Down
31 changes: 20 additions & 11 deletions python-package/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def find_lib():
return LIB_PATH


def copy_files(use_gpu=False):
def copy_files(integrated_opencl=False, use_gpu=False):

def copy_files_helper(folder_name):
src = os.path.join(CURRENT_DIR, os.path.pardir, folder_name)
Expand All @@ -51,14 +51,18 @@ def copy_files_helper(folder_name):
copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.vcxproj"),
os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.vcxproj"),
verbose=0)
if use_gpu:
copy_files_helper('compute')
copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"),
os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"),
verbose=0)
copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "LICENSE"),
os.path.join(CURRENT_DIR, "LICENSE"),
verbose=0)
copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"),
os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"),
verbose=0)
if integrated_opencl:
copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeIntegratedOpenCL.cmake"),
os.path.join(CURRENT_DIR, "compile", "CMakeIntegratedOpenCL.cmake"),
verbose=0)
if use_gpu:
copy_files_helper('compute')


def clear_path(path):
Expand Down Expand Up @@ -91,7 +95,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False,
use_hdfs=False, boost_root=None, boost_dir=None,
boost_include_dir=None, boost_librarydir=None,
opencl_include_dir=None, opencl_library=None,
nomp=False, bit32=False):
nomp=False, bit32=False, integrated_opencl=False):

if os.path.exists(os.path.join(CURRENT_DIR, "build_cpp")):
shutil.rmtree(os.path.join(CURRENT_DIR, "build_cpp"))
Expand All @@ -101,6 +105,9 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False,
logger.info("Starting to compile the library.")

cmake_cmd = ["cmake", "../compile/"]
if integrated_opencl:
use_gpu = False
cmake_cmd.append("-D__INTEGRATE_OPENCL=ON")
if use_gpu:
cmake_cmd.append("-DUSE_GPU=ON")
if boost_root:
Expand Down Expand Up @@ -134,7 +141,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False,
else:
status = 1
lib_path = os.path.join(CURRENT_DIR, "compile", "windows", "x64", "DLL", "lib_lightgbm.dll")
if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32)):
if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32, integrated_opencl)):
logger.info("Starting to compile with MSBuild from existing solution file.")
platform_toolsets = ("v142", "v141", "v140")
for pt in platform_toolsets:
Expand Down Expand Up @@ -187,6 +194,7 @@ class CustomInstall(install):

user_options = install.user_options + [
('mingw', 'm', 'Compile with MinGW'),
('integrated-opencl', None, 'Compile integrated OpenCL version'),
('gpu', 'g', 'Compile GPU version'),
('mpi', None, 'Compile MPI version'),
('nomp', None, 'Compile version without OpenMP support'),
Expand All @@ -204,6 +212,7 @@ class CustomInstall(install):
def initialize_options(self):
install.initialize_options(self)
self.mingw = 0
self.integrated_opencl = 0
self.gpu = 0
self.boost_root = None
self.boost_dir = None
Expand All @@ -227,12 +236,12 @@ def run(self):
"please use 64-bit Python instead.")
open(LOG_PATH, 'wb').close()
if not self.precompile:
copy_files(use_gpu=self.gpu)
copy_files(integrated_opencl=self.integrated_opencl, use_gpu=self.gpu)
compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_mpi=self.mpi,
use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir,
boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library,
nomp=self.nomp, bit32=self.bit32)
nomp=self.nomp, bit32=self.bit32, integrated_opencl=self.integrated_opencl)
install.run(self)
if os.path.isfile(LOG_PATH):
os.remove(LOG_PATH)
Expand All @@ -241,7 +250,7 @@ def run(self):
class CustomSdist(sdist):

def run(self):
copy_files(use_gpu=True)
copy_files(integrated_opencl=True, use_gpu=True)
open(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'), 'w').close()
if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')):
shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'Release'))
Expand Down