From 4ce77a0bab330e8b83c52a083cf2c72cbf2c67d6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 15 Aug 2021 15:06:49 -0500 Subject: [PATCH 1/7] [R-package] parallelize compilation in CMake-based builds --- R-package/src/install.libs.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R-package/src/install.libs.R b/R-package/src/install.libs.R index 9f839e2c1d73..6fbcfbd373db 100644 --- a/R-package/src/install.libs.R +++ b/R-package/src/install.libs.R @@ -132,7 +132,7 @@ if (WINDOWS && use_visual_studio) { # Prepare installation steps cmake_args <- NULL build_cmd <- "make" -build_args <- "_lightgbm" +build_args <- "_lightgbm -j4" lib_folder <- file.path(source_dir, fsep = "/") # add in command-line arguments @@ -194,7 +194,7 @@ if (WINDOWS) { cmake_args <- c(cmake_args, "-G", shQuote(windows_makefile_generator)) .run_shell_command("cmake", c(cmake_args, ".."), strict = FALSE) build_cmd <- windows_build_tool - build_args <- "_lightgbm" + build_args <- "_lightgbm -j4" } else { visual_studio_succeeded <- .generate_vs_makefiles(cmake_args) if (!isTRUE(visual_studio_succeeded)) { @@ -203,7 +203,7 @@ if (WINDOWS) { cmake_args <- c(cmake_args, "-G", shQuote(windows_makefile_generator)) .run_shell_command("cmake", c(cmake_args, ".."), strict = FALSE) build_cmd <- windows_build_tool - build_args <- "_lightgbm" + build_args <- "_lightgbm -j4" } else { build_cmd <- "cmake" build_args <- c("--build", ".", "--target", "_lightgbm", "--config", "Release") From 31bb22cdc5cb810aee1d18ebfbe5b78c46d10830 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 5 Nov 2021 23:41:35 -0500 Subject: [PATCH 2/7] Apply suggestions from code review Co-authored-by: Nikita Titov --- R-package/src/install.libs.R | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/R-package/src/install.libs.R b/R-package/src/install.libs.R index 6fbcfbd373db..4de9ee9be7b1 100644 --- a/R-package/src/install.libs.R +++ b/R-package/src/install.libs.R @@ -132,7 +132,7 @@ if (WINDOWS && use_visual_studio) { # Prepare installation steps cmake_args <- NULL build_cmd <- "make" -build_args <- "_lightgbm -j4" +build_args <- c("_lightgbm", "-j4") lib_folder <- file.path(source_dir, fsep = "/") # add in command-line arguments @@ -194,7 +194,7 @@ if (WINDOWS) { cmake_args <- c(cmake_args, "-G", shQuote(windows_makefile_generator)) .run_shell_command("cmake", c(cmake_args, ".."), strict = FALSE) build_cmd <- windows_build_tool - build_args <- "_lightgbm -j4" + build_args <- c("_lightgbm", "-j4") } else { visual_studio_succeeded <- .generate_vs_makefiles(cmake_args) if (!isTRUE(visual_studio_succeeded)) { @@ -203,7 +203,7 @@ if (WINDOWS) { cmake_args <- c(cmake_args, "-G", shQuote(windows_makefile_generator)) .run_shell_command("cmake", c(cmake_args, ".."), strict = FALSE) build_cmd <- windows_build_tool - build_args <- "_lightgbm -j4" + build_args <- c("_lightgbm", "-j4") } else { build_cmd <- "cmake" build_args <- c("--build", ".", "--target", "_lightgbm", "--config", "Release") From 188f157ac9db604f2e7d7e40a6b6c9c5f4a3d42e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 6 Nov 2021 09:15:06 -0500 Subject: [PATCH 3/7] working on adding -j --- build_r.R | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/build_r.R b/build_r.R index aa16c6fcdc6c..bb860c3edd58 100644 --- a/build_r.R +++ b/build_r.R @@ -21,9 +21,12 @@ TEMP_SOURCE_DIR <- file.path(TEMP_R_DIR, "src") out_list <- list( "flags" = character(0L) , "keyword_args" = character(0L) + , "make_args" = character(0L) ) for (arg in args) { - if (any(grepl("=", arg))) { + if (any(grepl("^\\-j[0-9]+", arg))) { + out_list[["make_args"]] <- arg + } else if (any(grepl("=", arg))) { split_arg <- strsplit(arg, "=")[[1L]] arg_name <- split_arg[[1L]] arg_value <- split_arg[[2L]] From d4a88541efd98a8728357840b45afd8ab5c031ae Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 6 Nov 2021 21:04:16 -0500 Subject: [PATCH 4/7] pass -j through to install.libs.R --- .ci/test_r_package.sh | 2 +- .ci/test_r_package_windows.ps1 | 4 ++-- R-package/src/install.libs.R | 7 ++++--- build_r.R | 14 ++++++++++++++ 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index a6656dfb83e6..6693a04a7fe4 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -112,7 +112,7 @@ cd ${BUILD_DIRECTORY} PKG_TARBALL="lightgbm_*.tar.gz" LOG_FILE_NAME="lightgbm.Rcheck/00check.log" if [[ $R_BUILD_TYPE == "cmake" ]]; then - Rscript build_r.R --skip-install || exit -1 + Rscript build_r.R -j2 --skip-install || exit -1 elif [[ $R_BUILD_TYPE == "cran" ]]; then # on Linux, we recreate configure in CI to test if diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 3c0f11ca58c6..2d676034154c 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -153,10 +153,10 @@ if ($env:COMPILER -ne "MSVC") { if ($env:R_BUILD_TYPE -eq "cmake") { if ($env:TOOLCHAIN -eq "MINGW") { Write-Output "Telling R to use MinGW" - $env:BUILD_R_FLAGS = "c('--skip-install', '--use-mingw')" + $env:BUILD_R_FLAGS = "c('--skip-install', '--use-mingw', '-j2')" } elseif ($env:TOOLCHAIN -eq "MSYS") { Write-Output "Telling R to use MSYS" - $env:BUILD_R_FLAGS = "c('--skip-install', '--use-msys2')" + $env:BUILD_R_FLAGS = "c('--skip-install', '--use-msys2', '-j2')" } elseif ($env:TOOLCHAIN -eq "MSVC") { $env:BUILD_R_FLAGS = "'--skip-install'" } else { diff --git a/R-package/src/install.libs.R b/R-package/src/install.libs.R index 4de9ee9be7b1..f292b43e318c 100644 --- a/R-package/src/install.libs.R +++ b/R-package/src/install.libs.R @@ -1,5 +1,6 @@ # User options use_gpu <- FALSE +make_args_from_build_script <- character(0L) # For Windows, the package will be built with Visual Studio # unless you set one of these to TRUE @@ -132,7 +133,7 @@ if (WINDOWS && use_visual_studio) { # Prepare installation steps cmake_args <- NULL build_cmd <- "make" -build_args <- c("_lightgbm", "-j4") +build_args <- c("_lightgbm", make_args_from_build_script) lib_folder <- file.path(source_dir, fsep = "/") # add in command-line arguments @@ -194,7 +195,7 @@ if (WINDOWS) { cmake_args <- c(cmake_args, "-G", shQuote(windows_makefile_generator)) .run_shell_command("cmake", c(cmake_args, ".."), strict = FALSE) build_cmd <- windows_build_tool - build_args <- c("_lightgbm", "-j4") + build_args <- c("_lightgbm", make_args_from_build_script) } else { visual_studio_succeeded <- .generate_vs_makefiles(cmake_args) if (!isTRUE(visual_studio_succeeded)) { @@ -203,7 +204,7 @@ if (WINDOWS) { cmake_args <- c(cmake_args, "-G", shQuote(windows_makefile_generator)) .run_shell_command("cmake", c(cmake_args, ".."), strict = FALSE) build_cmd <- windows_build_tool - build_args <- c("_lightgbm", "-j4") + build_args <- c("_lightgbm", make_args_from_build_script) } else { build_cmd <- "cmake" build_args <- c("--build", ".", "--target", "_lightgbm", "--config", "Release") diff --git a/build_r.R b/build_r.R index bb860c3edd58..b4c610197692 100644 --- a/build_r.R +++ b/build_r.R @@ -112,6 +112,20 @@ if (length(keyword_args) > 0L) { ) } +# if provided, set '-j' in 'make' commands in install.libs.R +if (length(parsed_args[["make_args"]]) > 0L) { + install_libs_content <- gsub( + pattern = "make_args_from_build_script <- character(0L)" + , replacement = paste0( + "make_args_from_build_script <- c(\"" + , paste0(parsed_args[["make_args"]], collapse = "\", \"") + , "\")" + ) + , x = install_libs_content + , fixed = TRUE + ) +} + # R returns FALSE (not a non-zero exit code) if a file copy operation # breaks. Let's fix that .handle_result <- function(res) { From a2f589804156dad061e4db3d0c779496389f6adc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 6 Nov 2021 21:16:33 -0500 Subject: [PATCH 5/7] add docs on -j --- R-package/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/R-package/README.md b/R-package/README.md index 352071f0ad4b..b836ba62e9a1 100644 --- a/R-package/README.md +++ b/R-package/README.md @@ -152,6 +152,9 @@ Rscript build_r.R The `build_r.R` script builds the package in a temporary directory called `lightgbm_r`. It will destroy and recreate that directory each time you run the script. That script supports the following command-line options: +- `-j[jobs]`: number of threads to use when compiling LightGBM. e.g. `-j4` will try to compile 4 objects at a time. + - by default, this script uses single-thread compilation + - for best results, set `-j` to the number of physical CPUs - `--skip-install`: Build the package tarball, but do not install it. - `--use-gpu`: Build a GPU-enabled version of the library. - `--use-mingw`: Force the use of MinGW toolchain, regardless of R version. From 8cb11aafecf0de2cb239528e7fc5e3529c608e53 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 7 Nov 2021 12:26:09 -0600 Subject: [PATCH 6/7] use -j4 --- .ci/test_r_package.sh | 2 +- .ci/test_r_package_windows.ps1 | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index 6693a04a7fe4..522e622b51f0 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -112,7 +112,7 @@ cd ${BUILD_DIRECTORY} PKG_TARBALL="lightgbm_*.tar.gz" LOG_FILE_NAME="lightgbm.Rcheck/00check.log" if [[ $R_BUILD_TYPE == "cmake" ]]; then - Rscript build_r.R -j2 --skip-install || exit -1 + Rscript build_r.R -j4 --skip-install || exit -1 elif [[ $R_BUILD_TYPE == "cran" ]]; then # on Linux, we recreate configure in CI to test if diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 2d676034154c..b9845148e7b7 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -153,10 +153,10 @@ if ($env:COMPILER -ne "MSVC") { if ($env:R_BUILD_TYPE -eq "cmake") { if ($env:TOOLCHAIN -eq "MINGW") { Write-Output "Telling R to use MinGW" - $env:BUILD_R_FLAGS = "c('--skip-install', '--use-mingw', '-j2')" + $env:BUILD_R_FLAGS = "c('--skip-install', '--use-mingw', '-j4')" } elseif ($env:TOOLCHAIN -eq "MSYS") { Write-Output "Telling R to use MSYS" - $env:BUILD_R_FLAGS = "c('--skip-install', '--use-msys2', '-j2')" + $env:BUILD_R_FLAGS = "c('--skip-install', '--use-msys2', '-j4')" } elseif ($env:TOOLCHAIN -eq "MSVC") { $env:BUILD_R_FLAGS = "'--skip-install'" } else { From cd20d60bf68b5da1c0f233581143be89a6afd048 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 8 Nov 2021 17:31:07 -0600 Subject: [PATCH 7/7] Update R-package/README.md Co-authored-by: Nikita Titov --- R-package/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R-package/README.md b/R-package/README.md index b836ba62e9a1..73659d2b48bf 100644 --- a/R-package/README.md +++ b/R-package/README.md @@ -152,7 +152,7 @@ Rscript build_r.R The `build_r.R` script builds the package in a temporary directory called `lightgbm_r`. It will destroy and recreate that directory each time you run the script. That script supports the following command-line options: -- `-j[jobs]`: number of threads to use when compiling LightGBM. e.g. `-j4` will try to compile 4 objects at a time. +- `-j[jobs]`: number of threads to use when compiling LightGBM. E.g., `-j4` will try to compile 4 objects at a time. - by default, this script uses single-thread compilation - for best results, set `-j` to the number of physical CPUs - `--skip-install`: Build the package tarball, but do not install it.