Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Build integrated Python package library #3144

Merged
merged 31 commits into from
Sep 21, 2020
Merged

Build integrated Python package library #3144

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
b1841bd
Add specialized OpenCL/Python package build path
tpboudreau May 27, 2020
27dffcb
Refer to upstream OpenCL repository
tpboudreau Jun 2, 2020
2958084
Reset build job count in setup.py
tpboudreau Jun 3, 2020
519a5b9
TEMPORARY: refer to OpenCL fork to ensure Linux CI builds succeed
tpboudreau Jun 3, 2020
84b6804
Remove intermediate cmake target
tpboudreau Jun 3, 2020
ec4b3f4
Restrict OpenCL headers to documented API version
tpboudreau Jun 4, 2020
1fc7878
Use command line definition to activate integrated build
tpboudreau Jun 8, 2020
8e66b4e
Merge branch 'master' into build_python_package_library
tpboudreau Jun 9, 2020
af176a6
Flag reference to unofficial repo with FIXME
tpboudreau Jun 9, 2020
e0c71cb
Merge branch 'master' into build_python_package_library
tpboudreau Jun 30, 2020
81e6773
Merge branch 'master' into build_python_package_library
tpboudreau Aug 10, 2020
4dac9f5
Merge branch 'master' into build_python_package_library (with conflic…
tpboudreau Aug 27, 2020
180c929
TEMPORARY: update private repo tag for dependency
tpboudreau Aug 27, 2020
80318e0
Merge branch 'master' into build_python_package_library
tpboudreau Sep 3, 2020
1782aed
Remove integrated build for non-Win32 and related cleanup
tpboudreau Sep 3, 2020
3013f4d
Remove commented code
tpboudreau Sep 3, 2020
9d0614c
Rename integrated OpenCL build option and other cleanups
tpboudreau Sep 8, 2020
e567f33
Merge branch 'master' into build_python_package_library
tpboudreau Sep 8, 2020
d5fad62
Small cleanups
tpboudreau Sep 14, 2020
dc5ef75
Update CMakeIntegratedOpenCL.cmake
tpboudreau Sep 14, 2020
4fd05bc
Update CMakeIntegratedOpenCL.cmake
tpboudreau Sep 14, 2020
8a23901
Update CMakeIntegratedOpenCL.cmake
tpboudreau Sep 14, 2020
8f140dc
Update CMakeIntegratedOpenCL.cmake
tpboudreau Sep 14, 2020
f654d91
Update CMakeLists.txt
tpboudreau Sep 14, 2020
2597549
Update CMakeLists.txt
tpboudreau Sep 14, 2020
db60d59
Update CMakeLists.txt
tpboudreau Sep 14, 2020
e274c8e
Update CMakeIntegratedOpenCL.cmake
tpboudreau Sep 14, 2020
e4d2e3a
Merge from master, with conflict resolution
tpboudreau Sep 14, 2020
6acfd26
Update CMakeLists.txt
tpboudreau Sep 17, 2020
ee9f545
Merge branch 'master' into build_python_package_library
tpboudreau Sep 17, 2020
5d85aa1
Merge from master, with conflict resolution
tpboudreau Sep 21, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 44 additions & 7 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
if(USE_GPU OR APPLE)
if(__OPENCL_PYTHON_PACKAGE)
cmake_minimum_required(VERSION 3.11)
elseif(USE_GPU OR APPLE)
cmake_minimum_required(VERSION 3.2)
else()
cmake_minimum_required(VERSION 2.8)
Expand All @@ -20,6 +22,11 @@ if(APPLE)
OPTION(APPLE_OUTPUT_DYLIB "Output dylib shared library" OFF)
endif(APPLE)

if(__OPENCL_PYTHON_PACKAGE)
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
set(USE_GPU OFF CACHE BOOL "" FORCE)
message(WARNING "\nWARNING: building integrated OpenCL enabled components intended for Python package")
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
endif()

if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS "4.8.2")
message(FATAL_ERROR "Insufficient gcc version")
Expand Down Expand Up @@ -127,6 +134,11 @@ if(USE_GPU)
ADD_DEFINITIONS(-DUSE_GPU)
endif(USE_GPU)

if(__OPENCL_PYTHON_PACKAGE)
include(CMakeOpenCLPythonPackage.txt)
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
ADD_DEFINITIONS(-DUSE_GPU)
endif()

if(USE_HDFS)
find_package(JNI REQUIRED)
find_path(HDFS_INCLUDE_DIR hdfs.h REQUIRED)
Expand Down Expand Up @@ -246,7 +258,13 @@ endif(BUILD_FOR_R)
add_library(_lightgbm SHARED ${SOURCES})

if(MSVC)
set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")
set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")
# TODO: use uniquely named dll (in setup.py, lightgbm/libpath.py) to avoid runtime conflicts
#if(__OPENCL_PYTHON_PACKAGE)
# set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm-${OPENCL_PYTHON_PACKAGE_STAMP}")
#else()
# set_target_properties(_lightgbm PROPERTIES OUTPUT_NAME "lib_lightgbm")
#endif()
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
endif(MSVC)

if(USE_SWIG)
Expand Down Expand Up @@ -303,16 +321,35 @@ if(USE_GPU)
TARGET_LINK_LIBRARIES(_lightgbm ${OpenCL_LIBRARY} ${Boost_LIBRARIES})
endif(USE_GPU)

if(__OPENCL_PYTHON_PACKAGE)
# targets OpenCL and Boost are added in CMakeOpenCLPythonPackage.txt
add_dependencies(lightgbm OpenCL Boost)
add_dependencies(_lightgbm OpenCL Boost)
# variables OPENCL_PYTHON_PACKAGE_* are set in CMakeOpenCLPythonPackage.txt
target_include_directories(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_INCLUDES})
target_include_directories(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_INCLUDES})
target_compile_definitions(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_DEFINITIONS})
target_compile_definitions(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_DEFINITIONS})
target_link_libraries(lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_LIBRARIES})
target_link_libraries(_lightgbm PRIVATE ${OPENCL_PYTHON_PACKAGE_LIBRARIES})
endif()

if(USE_HDFS)
TARGET_LINK_LIBRARIES(lightgbm ${HDFS_CXX_LIBRARIES})
TARGET_LINK_LIBRARIES(_lightgbm ${HDFS_CXX_LIBRARIES})
endif(USE_HDFS)

if(WIN32 AND (MINGW OR CYGWIN))
TARGET_LINK_LIBRARIES(lightgbm Ws2_32)
TARGET_LINK_LIBRARIES(_lightgbm Ws2_32)
TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI)
TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI)
if(WIN32)
if (MINGW OR CYGWIN)
TARGET_LINK_LIBRARIES(lightgbm Ws2_32)
TARGET_LINK_LIBRARIES(_lightgbm Ws2_32)
TARGET_LINK_LIBRARIES(lightgbm IPHLPAPI)
TARGET_LINK_LIBRARIES(_lightgbm IPHLPAPI)
endif()
else()
if(__OPENCL_PYTHON_PACKAGE)
target_link_libraries(lightgbm PRIVATE pthread dl)
endif()
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
endif()

if(BUILD_FOR_R)
Expand Down
110 changes: 110 additions & 0 deletions CMakeOpenCLPythonPackage.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@

set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
set(BOOST_VERSION_DOT "1.73")
string(REPLACE "." "_" BOOST_VERSION_UNDERSCORE ${BOOST_VERSION_DOT})

set(OPENCL_HEADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-Headers.git")
set(OPENCL_HEADER_TAG "3f7c1f77f8e1a3c9b48765c713ca19013e95b91e")
#set(OPENCL_LOADER_REPOSITORY "https://github.com/KhronosGroup/OpenCL-ICD-Loader.git")
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
#set(OPENCL_LOADER_TAG "bbdf079426d859fb8a68c332b41b714f9c87d6ad")
set(OPENCL_LOADER_REPOSITORY "https://github.com/tpboudreau/OpenCL-ICD-Loader.git")
set(OPENCL_LOADER_TAG "e42b3464e63f30cb0b818d88b836dd9e02ddbd7b")
set(BOOST_REPOSITORY "https://github.com/boostorg/boost.git")
set(BOOST_TAG "boost-${BOOST_VERSION_DOT}.0")
execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} OUTPUT_VARIABLE LIGHTGBM_TAG OUTPUT_STRIP_TRAILING_WHITESPACE ERROR_QUIET)
string(SHA1 OPENCL_PYTHON_PACKAGE_STAMP "${OPENCL_HEADER_REPOSITORY}@${OPENCL_HEADER_TAG};${OPENCL_LOADER_REPOSITORY}@${OPENCL_LOADER_TAG};${BOOST_REPOSITORY}@${BOOST_TAG};lightgbm@${LIGHTGBM_TAG}")
message(STATUS "OpenCL Python package build stamp: ${OPENCL_PYTHON_PACKAGE_STAMP}")

# Build Independent OpenCL library
include(FetchContent)
FetchContent_Declare(OpenCL-Headers GIT_REPOSITORY ${OPENCL_HEADER_REPOSITORY} GIT_TAG ${OPENCL_HEADER_TAG})
FetchContent_GetProperties(OpenCL-Headers)
if(NOT OpenCL-Headers_POPULATED)
FetchContent_Populate(OpenCL-Headers)
message(STATUS "Populated OpenCL Headers")
endif()
set(OPENCL_ICD_LOADER_HEADERS_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for OpenCL ICD Loader
set(OpenCL_INCLUDE_DIR ${opencl-headers_SOURCE_DIR} CACHE PATH "") # for Boost::Compute

FetchContent_Declare(OpenCL-ICD-Loader GIT_REPOSITORY ${OPENCL_LOADER_REPOSITORY} GIT_TAG ${OPENCL_LOADER_TAG})
FetchContent_GetProperties(OpenCL-ICD-Loader)
if(NOT OpenCL-ICD-Loader_POPULATED)
FetchContent_Populate(OpenCL-ICD-Loader)
if(WIN32)
set(USE_DYNAMIC_VCXX_RUNTIME ON)
else()
set(OPENCL_ICD_LOADER_PIC ON)
endif()
add_subdirectory(${opencl-icd-loader_SOURCE_DIR} ${opencl-icd-loader_BINARY_DIR} EXCLUDE_FROM_ALL)
message(STATUS "Populated OpenCL ICD Loader")
endif()
list(APPEND OPENCL_PYTHON_PACKAGE_INCLUDES ${OPENCL_ICD_LOADER_HEADERS_DIR})
if(WIN32)
list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/Release/OpenCL.lib cfgmgr32.lib runtimeobject.lib)
else()
list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${opencl-icd-loader_BINARY_DIR}/libOpenCL.a)
endif()
list(APPEND OPENCL_PYTHON_PACKAGE_DEFINITIONS CL_TARGET_OPENCL_VERSION=120)

# Build Independent Boost libraries
include(ExternalProject)
include(ProcessorCount)
ProcessorCount(J)
set(BOOST_BASE "${PROJECT_BINARY_DIR}/Boost")
if(WIN32)
set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.bat")
set(BOOST_BUILD "${BOOST_BASE}/source/b2.exe")
set(BOOST_FLAGS "")
list(APPEND BOOST_SUBMODULES "libs/*" "tools/*")
else()
set(BOOST_BOOTSTRAP "${BOOST_BASE}/source/bootstrap.sh")
set(BOOST_BUILD "${BOOST_BASE}/source/b2")
set(BOOST_FLAGS -fPIC)
list(APPEND BOOST_SUBMODULES "libs/compute" "libs/chrono" "libs/config" "libs/filesystem" "libs/headers" "libs/io" "libs/predef" "libs/system" "libs/uuid" "tools/*")
endif()
ExternalProject_Add(Boost
TMP_DIR "${BOOST_BASE}/tmp"
STAMP_DIR "${BOOST_BASE}/stamp"
DOWNLOAD_DIR "${BOOST_BASE}/download"
SOURCE_DIR "${BOOST_BASE}/source"
BINARY_DIR "${BOOST_BASE}/source"
INSTALL_DIR "${BOOST_BASE}/install"
GIT_REPOSITORY ${BOOST_REPOSITORY}
GIT_TAG ${BOOST_TAG}
GIT_SUBMODULES ${BOOST_SUBMODULES}
GIT_SHALLOW ON
UPDATE_COMMAND ""
PATCH_COMMAND ""
CONFIGURE_COMMAND ${BOOST_BOOTSTRAP}
BUILD_COMMAND ${BOOST_BUILD} -sBOOST_ROOT=${BOOST_BASE}/source -a -q -j ${J} --with-headers --with-chrono --with-filesystem --with-system link=static runtime-link=shared variant=release threading=multi cxxflags="${BOOST_FLAGS}"
INSTALL_COMMAND ""
)
set(BOOST_INCLUDE ${BOOST_BASE}/source CACHE PATH "")
set(BOOST_LIBRARY ${BOOST_BASE}/source/stage/lib CACHE PATH "")
list(APPEND OPENCL_PYTHON_PACKAGE_INCLUDES ${BOOST_INCLUDE})
if(WIN32)
if(MSVC)
if(${MSVC_VERSION} GREATER 1929)
set(MSVC_TOOLCHAIN_ID "unknown")
elseif(${MSVC_VERSION} GREATER 1919)
set(MSVC_TOOLCHAIN_ID "142")
elseif(${MSVC_VERSION} GREATER 1909)
set(MSVC_TOOLCHAIN_ID "141")
elseif(${MSVC_VERSION} GREATER 1899)
set(MSVC_TOOLCHAIN_ID "140")
else()
set(MSVC_TOOLCHAIN_ID "unknown")
endif()
list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib)
list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_system-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib)
list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_chrono-vc${MSVC_TOOLCHAIN_ID}-mt-x64-${BOOST_VERSION_UNDERSCORE}.lib)
else()
message(WARNING "MinGW Boost library names not yet specified")
endif()
else()
list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_filesystem.a)
list(APPEND OPENCL_PYTHON_PACKAGE_LIBRARIES ${BOOST_LIBRARY}/libboost_system.a)
endif()

set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE)

13 changes: 13 additions & 0 deletions CMakeOpenCLPythonPackageCache.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@

set(__OPENCL_PYTHON_PACKAGE ON CACHE BOOL "" FORCE)

set(USE_MPI OFF CACHE BOOL "")
set(USE_OPENMP ON CACHE BOOL "")
set(USE_GPU OFF CACHE BOOL "")
set(USE_SWIG OFF CACHE BOOL "")
set(USE_HDFS OFF CACHE BOOL "")
set(USE_R35 OFF CACHE BOOL "")
set(USE_TIMETAG OFF CACHE BOOL "")
set(USE_DEBUG OFF CACHE BOOL "")
set(BUILD_FOR_R OFF CACHE BOOL "")

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't have other comments to offer yet, but can you please be sure that CMake cache files are not checked into the repo?

Maybe add a *Cache.txt entry to .gitignore, I don't think that would conflict with anything we have intentionally checked in.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the early feedback.

The file was meant as a cmake preload file, but since it's likely to cause further confusion, I've moved its contents into the main CMakeLists.txt file and removed the offending file.

37 changes: 26 additions & 11 deletions python-package/setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def find_lib():
return LIB_PATH


def copy_files(use_gpu=False):
def copy_files(opencl_python_package=False, use_gpu=False):

def copy_files_helper(folder_name):
src = os.path.join(CURRENT_DIR, os.path.pardir, folder_name)
Expand All @@ -50,14 +50,21 @@ def copy_files_helper(folder_name):
distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "windows", "LightGBM.vcxproj"),
os.path.join(CURRENT_DIR, "compile", "windows", "LightGBM.vcxproj"),
verbose=0)
if use_gpu:
copy_files_helper('compute')
distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"),
os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"),
verbose=0)
distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "LICENSE"),
os.path.join(CURRENT_DIR, "LICENSE"),
verbose=0)
distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeLists.txt"),
os.path.join(CURRENT_DIR, "compile", "CMakeLists.txt"),
verbose=0)
if opencl_python_package:
distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeOpenCLPythonPackage.txt"),
os.path.join(CURRENT_DIR, "compile", "CMakeOpenCLPythonPackage.txt"),
verbose=0)
distutils.file_util.copy_file(os.path.join(CURRENT_DIR, os.path.pardir, "CMakeOpenCLPythonPackageCache.txt"),
os.path.join(CURRENT_DIR, "compile", "CMakeOpenCLPythonPackageCache.txt"),
verbose=0)
if use_gpu:
copy_files_helper('compute')


def clear_path(path):
Expand Down Expand Up @@ -90,7 +97,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False,
use_hdfs=False, boost_root=None, boost_dir=None,
boost_include_dir=None, boost_librarydir=None,
opencl_include_dir=None, opencl_library=None,
nomp=False, bit32=False):
nomp=False, bit32=False, opencl_python_package=False):

if os.path.exists(os.path.join(CURRENT_DIR, "build_cpp")):
shutil.rmtree(os.path.join(CURRENT_DIR, "build_cpp"))
Expand All @@ -100,6 +107,12 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False,
logger.info("Starting to compile the library.")

cmake_cmd = ["cmake", "../compile/"]
if opencl_python_package:
use_gpu = False
use_mpi = False
nomp = False
use_hdfs = False
StrikerRUS marked this conversation as resolved.
Show resolved Hide resolved
cmake_cmd.append("-C../compile/CMakeOpenCLPythonPackageCache.txt")
if use_gpu:
cmake_cmd.append("-DUSE_GPU=ON")
if boost_root:
Expand Down Expand Up @@ -133,7 +146,7 @@ def compile_cpp(use_mingw=False, use_gpu=False, use_mpi=False,
else:
status = 1
lib_path = os.path.join(CURRENT_DIR, "compile", "windows", "x64", "DLL", "lib_lightgbm.dll")
if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32)):
if not any((use_gpu, use_mpi, use_hdfs, nomp, bit32, opencl_python_package)):
logger.info("Starting to compile with MSBuild from existing solution file.")
platform_toolsets = ("v142", "v141", "v140")
for pt in platform_toolsets:
Expand Down Expand Up @@ -186,6 +199,7 @@ class CustomInstall(install):

user_options = install.user_options + [
('mingw', 'm', 'Compile with MinGW'),
('opencl-python-package', None, 'Compile integrated OpenCL version'),
('gpu', 'g', 'Compile GPU version'),
('mpi', None, 'Compile MPI version'),
('nomp', None, 'Compile version without OpenMP support'),
Expand All @@ -203,6 +217,7 @@ class CustomInstall(install):
def initialize_options(self):
install.initialize_options(self)
self.mingw = 0
self.opencl_python_package = 0
self.gpu = 0
self.boost_root = None
self.boost_dir = None
Expand All @@ -226,12 +241,12 @@ def run(self):
"please use 64-bit Python instead.")
open(LOG_PATH, 'wb').close()
if not self.precompile:
copy_files(use_gpu=self.gpu)
copy_files(opencl_python_package=self.opencl_python_package, use_gpu=self.gpu)
compile_cpp(use_mingw=self.mingw, use_gpu=self.gpu, use_mpi=self.mpi,
use_hdfs=self.hdfs, boost_root=self.boost_root, boost_dir=self.boost_dir,
boost_include_dir=self.boost_include_dir, boost_librarydir=self.boost_librarydir,
opencl_include_dir=self.opencl_include_dir, opencl_library=self.opencl_library,
nomp=self.nomp, bit32=self.bit32)
nomp=self.nomp, bit32=self.bit32, opencl_python_package=self.opencl_python_package)
install.run(self)
if os.path.isfile(LOG_PATH):
os.remove(LOG_PATH)
Expand All @@ -240,7 +255,7 @@ def run(self):
class CustomSdist(sdist):

def run(self):
copy_files(use_gpu=True)
copy_files(opencl_python_package=True, use_gpu=True)
open(os.path.join(CURRENT_DIR, '_IS_SOURCE_PACKAGE.txt'), 'w').close()
if os.path.exists(os.path.join(CURRENT_DIR, 'lightgbm', 'Release')):
shutil.rmtree(os.path.join(CURRENT_DIR, 'lightgbm', 'Release'))
Expand Down