From 7cb47d0af151ece4c4bc367ada0e59f345ab34bd Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 2 Apr 2021 04:41:38 +0100 Subject: [PATCH 1/8] [R-package] prevent symbol lookup conflicts --- R-package/src/lightgbm_R.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/R-package/src/lightgbm_R.cpp b/R-package/src/lightgbm_R.cpp index a5e94f011ac5..c7abf73ffc61 100644 --- a/R-package/src/lightgbm_R.cpp +++ b/R-package/src/lightgbm_R.cpp @@ -720,6 +720,8 @@ static const R_CallMethodDef CallEntries[] = { {NULL, NULL, 0} }; +LIGHTGBM_C_EXPORT void R_init_lightgbm(DllInfo *dll); + void R_init_lightgbm(DllInfo *dll) { R_registerRoutines(dll, NULL, CallEntries, NULL, NULL); R_useDynamicSymbols(dll, FALSE); From 791cf6cb3e6140231f008ff7702d7f6fbc1bcf9a Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 2 Apr 2021 05:09:40 +0100 Subject: [PATCH 2/8] add unit tests --- R-package/tests/testthat/test_registration.R | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 R-package/tests/testthat/test_registration.R diff --git a/R-package/tests/testthat/test_registration.R b/R-package/tests/testthat/test_registration.R new file mode 100644 index 000000000000..ea44f563348e --- /dev/null +++ b/R-package/tests/testthat/test_registration.R @@ -0,0 +1,13 @@ +# this test is used to catch silent errors in routine registration, +# like the one documented in +# https://github.com/microsoft/LightGBM/issues/4045#issuecomment-812289182 +test_that("lightgbm routine registration worked", { + dll_info <- getLoadedDLLs()[["lightgbm"]] + + # check that dynamic lookup has been disabled + expect_false(dll_info[["dynamicLookup"]]) + + # check that all the expected symbols show up + registered_routines <- getDLLRegisteredRoutines(dll_info[["path"]])[[".Call"]] + expect_gt(length(registered_routines), 20L) +}) From f2a584ed99bdd572bcce975a75b1238e204d76ff Mon Sep 17 00:00:00 2001 From: James Lamb Date: Fri, 2 Apr 2021 05:23:53 +0100 Subject: [PATCH 3/8] only run test on Windows --- R-package/tests/testthat/test_registration.R | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/R-package/tests/testthat/test_registration.R b/R-package/tests/testthat/test_registration.R index ea44f563348e..e22b8a5e23e2 100644 --- a/R-package/tests/testthat/test_registration.R +++ b/R-package/tests/testthat/test_registration.R @@ -1,7 +1,11 @@ +ON_WINDOWS <- .Platform$OS.type == "windows" + # this test is used to catch silent errors in routine registration, # like the one documented in # https://github.com/microsoft/LightGBM/issues/4045#issuecomment-812289182 test_that("lightgbm routine registration worked", { + testthat::skip_if_not(ON_WINDOWS) + dll_info <- getLoadedDLLs()[["lightgbm"]] # check that dynamic lookup has been disabled From 6668390e954193a7ab9eca76dad52aef0c2dbb64 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 24 Apr 2021 22:10:42 -0500 Subject: [PATCH 4/8] move to .Call() calls --- .appveyor.yml | 72 +-- .github/workflows/cuda.yml | 186 +++--- .github/workflows/linkchecker.yml | 60 +- .github/workflows/optional_checks.yml | 56 +- .github/workflows/python_package.yml | 152 ++--- .github/workflows/static_analysis.yml | 164 ++--- .vsts-ci.yml | 762 ++++++++++++------------ R-package/R/lgb.Booster.R | 236 ++++++-- R-package/R/lgb.Dataset.R | 120 ++-- R-package/R/lgb.Predictor.R | 54 +- R-package/R/utils.R | 53 +- R-package/tests/testthat/test_dataset.R | 10 +- 12 files changed, 1020 insertions(+), 905 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index b7d9fe589176..ccb08022792a 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,43 +1,43 @@ -version: 3.2.1.99.{build} +# version: 3.2.1.99.{build} -image: Visual Studio 2015 -platform: x64 -configuration: # a trick to construct a build matrix with multiple Python versions - - 3.7 +# image: Visual Studio 2015 +# platform: x64 +# configuration: # a trick to construct a build matrix with multiple Python versions +# - 3.7 -# only build pull requests and -# commits to 'master' -branches: - only: - - master +# # only build pull requests and +# # commits to 'master' +# branches: +# only: +# - master -environment: - matrix: - - COMPILER: MSVC - TASK: python - - COMPILER: MINGW - TASK: python +# environment: +# matrix: +# - COMPILER: MSVC +# TASK: python +# - COMPILER: MINGW +# TASK: python -clone_depth: 5 +# clone_depth: 5 -install: - - git submodule update --init --recursive # get `external_libs` folder - - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) - - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - set PYTHON_VERSION=%CONFIGURATION% - - set CONDA_ENV="test-env" - - ps: | - switch ($env:PYTHON_VERSION) { - "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} - "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} - default {$env:MINICONDA = "C:\Miniconda37-x64"} - } - $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" - $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" - $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() +# install: +# - git submodule update --init --recursive # get `external_libs` folder +# - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) +# - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% +# - set PYTHON_VERSION=%CONFIGURATION% +# - set CONDA_ENV="test-env" +# - ps: | +# switch ($env:PYTHON_VERSION) { +# "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} +# "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} +# default {$env:MINICONDA = "C:\Miniconda37-x64"} +# } +# $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" +# $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" +# $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() -build: false +# build: false -test_script: - - conda init powershell - - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 +# test_script: +# - conda init powershell +# - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 959a2c717edf..b93e155b934e 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -1,96 +1,96 @@ -name: CUDA Version +# name: CUDA Version -on: - push: - branches: - - master - pull_request: - branches: - - master +# on: +# push: +# branches: +# - master +# pull_request: +# branches: +# - master -env: - github_actions: 'true' - os_name: linux - task: cuda - conda_env: test-env +# env: +# github_actions: 'true' +# os_name: linux +# task: cuda +# conda_env: test-env -jobs: - test: - name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - - method: source - compiler: gcc - python_version: 3.7 - cuda_version: "11.2.2" - - method: pip - compiler: clang - python_version: 3.8 - cuda_version: "10.0" - - method: wheel - compiler: gcc - python_version: 3.9 - cuda_version: "9.0" - steps: - - name: Setup or update software on host machine - run: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - git \ - gnupg-agent \ - software-properties-common - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: Remove old folder with repository - run: sudo rm -rf $GITHUB_WORKSPACE - - name: Checkout repository - uses: actions/checkout@v1 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export ROOT_DOCKER_FOLDER=/LightGBM - cat > docker.env < docker-script.sh < docker.env < docker-script.sh < docker.env < docker-script.sh < docker.env < docker-script.sh < buf_len) { + buf_len <- act_len + buf <- raw(buf_len) + call_state <- 0L + buf <- .Call( + LGBM_BoosterSaveModelToString_R , private$handle , as.integer(num_iteration) , as.integer(feature_importance_type) + , buf_len + , act_len + , buf + , call_state ) + } + + return( + lgb.encode.char(arr = buf, len = act_len) ) }, @@ -494,13 +550,39 @@ Booster <- R6::R6Class( num_iteration <- self$best_iter } - return( - lgb.call.return.str( - fun_name = "LGBM_BoosterDumpModel_R" + buf_len <- as.integer(1024L * 1024L) + act_len <- 0L + buf <- raw(buf_len) + call_state <- 0L + buf <- .Call( + LGBM_BoosterDumpModel_R + , private$handle + , as.integer(num_iteration) + , as.integer(feature_importance_type) + , buf_len + , act_len + , buf + , call_state + ) + + if (act_len > buf_len) { + buf_len <- act_len + buf <- raw(buf_len) + call_state <- 0L + buf <- .Call( + LGBM_BoosterDumpModel_R , private$handle , as.integer(num_iteration) , as.integer(feature_importance_type) + , buf_len + , act_len + , buf + , call_state ) + } + + return( + lgb.encode.char(arr = buf, len = act_len) ) }, @@ -595,12 +677,14 @@ Booster <- R6::R6Class( if (is.null(private$predict_buffer[[data_name]])) { # Store predictions + call_state <- 0L npred <- 0L - npred <- lgb.call( - fun_name = "LGBM_BoosterGetNumPredict_R" - , ret = npred + npred <- .Call( + LGBM_BoosterGetNumPredict_R , private$handle , as.integer(idx - 1L) + , npred + , call_state ) private$predict_buffer[[data_name]] <- numeric(npred) @@ -610,11 +694,13 @@ Booster <- R6::R6Class( if (!private$is_predicted_cur_iter[[idx]]) { # Use buffer - private$predict_buffer[[data_name]] <- lgb.call( - fun_name = "LGBM_BoosterGetPredict_R" - , ret = private$predict_buffer[[data_name]] + call_state <- 0L + private$predict_buffer[[data_name]] <- .Call( + LGBM_BoosterGetPredict_R , private$handle , as.integer(idx - 1L) + , private$predict_buffer[[data_name]] + , call_state ) private$is_predicted_cur_iter[[idx]] <- TRUE } @@ -629,10 +715,32 @@ Booster <- R6::R6Class( if (is.null(private$eval_names)) { # Get evaluation names - names <- lgb.call.return.str( - fun_name = "LGBM_BoosterGetEvalNames_R" + buf_len <- as.integer(1024L * 1024L) + act_len <- 0L + buf <- raw(buf_len) + call_state <- 0L + buf <- .Call( + LGBM_BoosterGetEvalNames_R , private$handle + , buf_len + , act_len + , buf + , call_state ) + if (act_len > buf_len) { + buf_len <- act_len + buf <- raw(buf_len) + call_state <- 0L + buf <- .Call( + LGBM_BoosterGetEvalNames_R + , private$handle + , buf_len + , act_len + , buf + , call_state + ) + } + names <- lgb.encode.char(arr = buf, len = act_len) # Check names' length if (nchar(names) > 0L) { @@ -673,11 +781,11 @@ Booster <- R6::R6Class( # Create evaluation values tmp_vals <- numeric(length(private$eval_names)) - tmp_vals <- lgb.call( - fun_name = "LGBM_BoosterGetEval_R" - , ret = tmp_vals + tmp_vals <- .Call( + LGBM_BoosterGetEval_R , private$handle , as.integer(data_idx - 1L) + , tmp_vals ) # Loop through all evaluation names diff --git a/R-package/R/lgb.Dataset.R b/R-package/R/lgb.Dataset.R index cfc1c9391139..4ab18604cef8 100644 --- a/R-package/R/lgb.Dataset.R +++ b/R-package/R/lgb.Dataset.R @@ -13,7 +13,11 @@ Dataset <- R6::R6Class( if (!lgb.is.null.handle(x = private$handle)) { # Freeing up handle - lgb.call(fun_name = "LGBM_DatasetFree_R", ret = NULL, private$handle) + call_state <- 0L + .Call( + LGBM_DatasetFree_R + , private$handle + ) private$handle <- NULL } @@ -197,25 +201,29 @@ Dataset <- R6::R6Class( # Are we using a data file? if (is.character(private$raw_data)) { - handle <- lgb.call( - fun_name = "LGBM_DatasetCreateFromFile_R" - , ret = handle + call_state <- 0L + handle <- .Call( + LGBM_DatasetCreateFromFile_R , lgb.c_str(x = private$raw_data) , params_str , ref_handle + , handle + , call_state ) } else if (is.matrix(private$raw_data)) { # Are we using a matrix? - handle <- lgb.call( - fun_name = "LGBM_DatasetCreateFromMat_R" - , ret = handle + call_state <- 0L + handle <- .Call( + LGBM_DatasetCreateFromMat_R , private$raw_data , nrow(private$raw_data) , ncol(private$raw_data) , params_str , ref_handle + , handle + , call_state ) } else if (methods::is(private$raw_data, "dgCMatrix")) { @@ -223,9 +231,9 @@ Dataset <- R6::R6Class( stop("Cannot support large CSC matrix") } # Are we using a dgCMatrix (sparsed matrix column compressed) - handle <- lgb.call( - fun_name = "LGBM_DatasetCreateFromCSC_R" - , ret = handle + call_state <- 0L + handle <- .Call( + LGBM_DatasetCreateFromCSC_R , private$raw_data@p , private$raw_data@i , private$raw_data@x @@ -234,6 +242,8 @@ Dataset <- R6::R6Class( , nrow(private$raw_data) , params_str , ref_handle + , handle + , call_state ) } else { @@ -254,13 +264,15 @@ Dataset <- R6::R6Class( } # Construct subset - handle <- lgb.call( - fun_name = "LGBM_DatasetGetSubset_R" - , ret = handle + call_state <- 0L + handle <- .Call( + LGBM_DatasetGetSubset_R , ref_handle , c(private$used_indices) # Adding c() fixes issue in R v3.5 , length(private$used_indices) , params_str + , handle + , call_state ) } @@ -329,17 +341,20 @@ Dataset <- R6::R6Class( num_col <- 0L # Get numeric data and numeric features + call_state <- 0L return( c( - lgb.call( - fun_name = "LGBM_DatasetGetNumData_R" - , ret = num_row + .Call( + LGBM_DatasetGetNumData_R , private$handle + , num_row + , call_state ), - lgb.call( - fun_name = "LGBM_DatasetGetNumFeature_R" - , ret = num_col + .Call( + LGBM_DatasetGetNumFeature_R , private$handle + , num_col + , call_state ) ) ) @@ -369,10 +384,31 @@ Dataset <- R6::R6Class( if (!lgb.is.null.handle(x = private$handle)) { # Get feature names and write them - cnames <- lgb.call.return.str( - fun_name = "LGBM_DatasetGetFeatureNames_R" - , private$handle + buf_len <- as.integer(1024L * 1024L) + act_len <- 0L + buf <- raw(buf_len) + call_state <- 0L + buf <- .Call( + LGBM_DatasetGetFeatureNames_R + , private$handle + , buf_len + , act_len + , buf + , call_state ) + if (act_len > buf_len) { + buf_len <- act_len + buf <- raw(buf_len) + buf <- .Call( + LGBM_DatasetGetFeatureNames_R + , private$handle + , buf_len + , act_len + , buf + , call_state + ) + } + cnames <- lgb.encode.char(arr = buf, len = act_len) private$colnames <- as.character(base::strsplit(cnames, "\t")[[1L]]) return(private$colnames) @@ -413,11 +449,12 @@ Dataset <- R6::R6Class( # Merge names with tab separation merged_name <- paste0(as.list(private$colnames), collapse = "\t") - lgb.call( - fun_name = "LGBM_DatasetSetFeatureNames_R" - , ret = NULL + call_state <- 0L + .Call( + LGBM_DatasetSetFeatureNames_R , private$handle , lgb.c_str(x = merged_name) + , call_state ) } @@ -446,11 +483,13 @@ Dataset <- R6::R6Class( # Get field size of info info_len <- 0L - info_len <- lgb.call( - fun_name = "LGBM_DatasetGetFieldSize_R" - , ret = info_len + call_state <- 0L + info_len <- .Call( + LGBM_DatasetGetFieldSize_R , private$handle , lgb.c_str(x = name) + , info_len + , call_state ) # Check if info is not empty @@ -464,11 +503,13 @@ Dataset <- R6::R6Class( numeric(info_len) # Numeric } - ret <- lgb.call( - fun_name = "LGBM_DatasetGetField_R" - , ret = ret + call_state <- 0L + ret <- .Call( + LGBM_DatasetGetField_R , private$handle , lgb.c_str(x = name) + , ret + , call_state ) private$info[[name]] <- ret @@ -505,13 +546,14 @@ Dataset <- R6::R6Class( if (length(info) > 0L) { - lgb.call( - fun_name = "LGBM_DatasetSetField_R" - , ret = NULL + call_state <- 0L + .Call( + LGBM_DatasetSetField_R , private$handle , lgb.c_str(x = name) , info , length(info) + , call_state ) private$version <- private$version + 1L @@ -558,11 +600,10 @@ Dataset <- R6::R6Class( tryCatch({ call_state <- 0L .Call( - "LGBM_DatasetUpdateParamChecking_R" + LGBM_DatasetUpdateParamChecking_R , lgb.params2str(params = private$params) , lgb.params2str(params = params) , call_state - , PACKAGE = "lib_lightgbm" ) }, error = function(e) { # If updating failed but raw data is not available, raise an error because @@ -660,11 +701,12 @@ Dataset <- R6::R6Class( # Store binary data self$construct() - lgb.call( - fun_name = "LGBM_DatasetSaveBinary_R" - , ret = NULL + call_state <- 0L + .Call( + LGBM_DatasetSaveBinary_R , private$handle , lgb.c_str(x = fname) + , call_state ) return(invisible(self)) } diff --git a/R-package/R/lgb.Predictor.R b/R-package/R/lgb.Predictor.R index ced408e410dd..9f8b30f43831 100644 --- a/R-package/R/lgb.Predictor.R +++ b/R-package/R/lgb.Predictor.R @@ -14,10 +14,11 @@ Predictor <- R6::R6Class( if (private$need_free_handle && !lgb.is.null.handle(x = private$handle)) { # Freeing up handle - lgb.call( - fun_name = "LGBM_BoosterFree_R" - , ret = NULL + call_state <- 0L + .Call( + LGBM_BoosterFree_R , private$handle + , call_state ) private$handle <- NULL @@ -38,10 +39,12 @@ Predictor <- R6::R6Class( if (is.character(modelfile)) { # Create handle on it - handle <- lgb.call( - fun_name = "LGBM_BoosterCreateFromModelfile_R" - , ret = handle + call_state <- 0L + handle <- .Call( + LGBM_BoosterCreateFromModelfile_R , lgb.c_str(x = modelfile) + , handle + , call_state ) private$need_free_handle <- TRUE @@ -69,11 +72,13 @@ Predictor <- R6::R6Class( current_iter = function() { cur_iter <- 0L + call_state <- 0L return( - lgb.call( - fun_name = "LGBM_BoosterGetCurrentIteration_R" - , ret = cur_iter + .Call( + LGBM_BoosterGetCurrentIteration_R , private$handle + , cur_iter + , call_state ) ) @@ -108,9 +113,9 @@ Predictor <- R6::R6Class( on.exit(unlink(tmp_filename), add = TRUE) # Predict from temporary file - lgb.call( - fun_name = "LGBM_BoosterPredictForFile_R" - , ret = NULL + call_state <- 0L + .Call( + LGBM_BoosterPredictForFile_R , private$handle , data , as.integer(header) @@ -121,6 +126,7 @@ Predictor <- R6::R6Class( , as.integer(num_iteration) , private$params , lgb.c_str(x = tmp_filename) + , call_state ) # Get predictions from file @@ -136,9 +142,9 @@ Predictor <- R6::R6Class( npred <- 0L # Check number of predictions to do - npred <- lgb.call( - fun_name = "LGBM_BoosterCalcNumPredict_R" - , ret = npred + call_state <- 0L + npred <- .Call( + LGBM_BoosterCalcNumPredict_R , private$handle , as.integer(num_row) , as.integer(rawscore) @@ -146,6 +152,8 @@ Predictor <- R6::R6Class( , as.integer(predcontrib) , as.integer(start_iteration) , as.integer(num_iteration) + , npred + , call_state ) # Pre-allocate empty vector @@ -158,9 +166,9 @@ Predictor <- R6::R6Class( if (storage.mode(data) != "double") { storage.mode(data) <- "double" } - preds <- lgb.call( - fun_name = "LGBM_BoosterPredictForMat_R" - , ret = preds + call_state <- 0L + preds <- .Call( + LGBM_BoosterPredictForMat_R , private$handle , data , as.integer(nrow(data)) @@ -171,6 +179,8 @@ Predictor <- R6::R6Class( , as.integer(start_iteration) , as.integer(num_iteration) , private$params + , preds + , call_state ) } else if (methods::is(data, "dgCMatrix")) { @@ -178,9 +188,9 @@ Predictor <- R6::R6Class( stop("Cannot support large CSC matrix") } # Check if data is a dgCMatrix (sparse matrix, column compressed format) - preds <- lgb.call( - fun_name = "LGBM_BoosterPredictForCSC_R" - , ret = preds + call_state <- 0L + preds <- .Call( + LGBM_BoosterPredictForCSC_R , private$handle , data@p , data@i @@ -194,6 +204,8 @@ Predictor <- R6::R6Class( , as.integer(start_iteration) , as.integer(num_iteration) , private$params + , preds + , call_state ) } else { diff --git a/R-package/R/utils.R b/R-package/R/utils.R index 0b43779f7536..5afd409fda14 100644 --- a/R-package/R/utils.R +++ b/R-package/R/utils.R @@ -33,11 +33,10 @@ lgb.last_error <- function() { act_len <- 0L err_msg <- raw(buf_len) err_msg <- .Call( - "LGBM_GetLastError_R" + LGBM_GetLastError_R , buf_len , act_len , err_msg - , PACKAGE = "lib_lightgbm" ) # Check error buffer @@ -45,11 +44,10 @@ lgb.last_error <- function() { buf_len <- act_len err_msg <- raw(buf_len) err_msg <- .Call( - "LGBM_GetLastError_R" + LGBM_GetLastError_R , buf_len , act_len , err_msg - , PACKAGE = "lib_lightgbm" ) } @@ -59,53 +57,6 @@ lgb.last_error <- function() { } -lgb.call <- function(fun_name, ret, ...) { - # Set call state to a zero value - call_state <- 0L - - # Check for a ret call - if (!is.null(ret)) { - call_state <- .Call( - fun_name - , ... - , ret - , call_state - , PACKAGE = "lib_lightgbm" - ) - } else { - call_state <- .Call( - fun_name - , ... - , call_state - , PACKAGE = "lib_lightgbm" - ) - } - - return(ret) - -} - -lgb.call.return.str <- function(fun_name, ...) { - - # Create buffer - buf_len <- as.integer(1024L * 1024L) - act_len <- 0L - buf <- raw(buf_len) - - # Call buffer - buf <- lgb.call(fun_name = fun_name, ret = buf, ..., buf_len, act_len) - - # Check for buffer content - if (act_len > buf_len) { - buf_len <- act_len - buf <- raw(buf_len) - buf <- lgb.call(fun_name = fun_name, ret = buf, ..., buf_len, act_len) - } - - return(lgb.encode.char(arr = buf, len = act_len)) - -} - lgb.params2str <- function(params, ...) { # Check for a list as input diff --git a/R-package/tests/testthat/test_dataset.R b/R-package/tests/testthat/test_dataset.R index f864785fa0a0..7e861b733cd4 100644 --- a/R-package/tests/testthat/test_dataset.R +++ b/R-package/tests/testthat/test_dataset.R @@ -75,17 +75,19 @@ test_that("lgb.Dataset: Dataset should be able to construct from matrix and retu rawData <- matrix(runif(1000L), ncol = 10L) handle <- lgb.null.handle() ref_handle <- NULL - handle <- lightgbm:::lgb.call( - "LGBM_DatasetCreateFromMat_R" - , ret = handle + call_state <- 0L + handle <- .Call( + LGBM_DatasetCreateFromMat_R , rawData , nrow(rawData) , ncol(rawData) , lightgbm:::lgb.params2str(params = list()) , ref_handle + , handle + , call_state ) expect_false(is.na(handle)) - lgb.call("LGBM_DatasetFree_R", ret = NULL, handle) + .Call(LGBM_DatasetFree_R, handle, call_state) handle <- NULL }) From a2ccac5ddcd274753236cb8a6d5eb6ba5bf6069f Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sat, 24 Apr 2021 22:37:04 -0500 Subject: [PATCH 5/8] fix references --- R-package/R/lgb.Booster.R | 73 ++++++++++++------------- R-package/R/lgb.Dataset.R | 46 ++++++++-------- R-package/R/lgb.Predictor.R | 21 ++++--- R-package/tests/testthat/test_dataset.R | 3 +- 4 files changed, 72 insertions(+), 71 deletions(-) diff --git a/R-package/R/lgb.Booster.R b/R-package/R/lgb.Booster.R index 5520ecd30efb..5aa02599606f 100644 --- a/R-package/R/lgb.Booster.R +++ b/R-package/R/lgb.Booster.R @@ -55,7 +55,7 @@ Booster <- R6::R6Class( params_str <- lgb.params2str(params = params) # Store booster handle call_state <- 0L - handle <- .Call( + .Call( LGBM_BoosterCreate_R , train_set_handle , params_str @@ -95,7 +95,7 @@ Booster <- R6::R6Class( # Create booster from model call_state <- 0L - handle <- .Call( + .Call( LGBM_BoosterCreateFromModelfile_R , lgb.c_str(x = modelfile) , handle @@ -111,7 +111,7 @@ Booster <- R6::R6Class( # Create booster from model call_state <- 0L - handle <- .Call( + .Call( LGBM_BoosterLoadModelFromString_R , lgb.c_str(x = model_str) , handle @@ -142,7 +142,7 @@ Booster <- R6::R6Class( private$handle <- handle private$num_class <- 1L call_state <- 0L - private$num_class <- .Call( + .Call( LGBM_BoosterGetNumClasses_R , private$handle , private$num_class @@ -273,7 +273,7 @@ Booster <- R6::R6Class( } # Boost iteration from known objective call_state <- 0L - ret <- .Call( + .Call( LGBM_BoosterUpdateOneIter_R , private$handle , call_state @@ -300,7 +300,7 @@ Booster <- R6::R6Class( # Return custom boosting gradient/hessian call_state <- 0L - ret <- .Call( + .Call( LGBM_BoosterUpdateOneIterCustom_R , private$handle , gpair$grad @@ -316,7 +316,7 @@ Booster <- R6::R6Class( private$is_predicted_cur_iter[[i]] <- FALSE } - return(ret) + return(invisible(self)) }, @@ -345,14 +345,13 @@ Booster <- R6::R6Class( cur_iter <- 0L call_state <- 0L - return( - .Call( - LGBM_BoosterGetCurrentIteration_R - , private$handle - , cur_iter - , call_state - ) + .Call( + LGBM_BoosterGetCurrentIteration_R + , private$handle + , cur_iter + , call_state ) + return(cur_iter) }, @@ -361,14 +360,13 @@ Booster <- R6::R6Class( upper_bound <- 0.0 call_state <- 0L - return( - .Call( - LGBM_BoosterGetUpperBoundValue_R - , private$handle - , upper_bound - , call_state - ) + .Call( + LGBM_BoosterGetUpperBoundValue_R + , private$handle + , upper_bound + , call_state ) + return(upper_bound) }, @@ -377,14 +375,13 @@ Booster <- R6::R6Class( lower_bound <- 0.0 call_state <- 0L - return( - .Call( - LGBM_BoosterGetLowerBoundValue_R - , private$handle - , lower_bound - , call_state - ) + .Call( + LGBM_BoosterGetLowerBoundValue_R + , private$handle + , lower_bound + , call_state ) + return(lower_bound) }, @@ -508,7 +505,7 @@ Booster <- R6::R6Class( # Call buffer call_state <- 0L - buf <- .Call( + .Call( LGBM_BoosterSaveModelToString_R , private$handle , as.integer(num_iteration) @@ -524,7 +521,7 @@ Booster <- R6::R6Class( buf_len <- act_len buf <- raw(buf_len) call_state <- 0L - buf <- .Call( + .Call( LGBM_BoosterSaveModelToString_R , private$handle , as.integer(num_iteration) @@ -554,7 +551,7 @@ Booster <- R6::R6Class( act_len <- 0L buf <- raw(buf_len) call_state <- 0L - buf <- .Call( + .Call( LGBM_BoosterDumpModel_R , private$handle , as.integer(num_iteration) @@ -569,7 +566,7 @@ Booster <- R6::R6Class( buf_len <- act_len buf <- raw(buf_len) call_state <- 0L - buf <- .Call( + .Call( LGBM_BoosterDumpModel_R , private$handle , as.integer(num_iteration) @@ -679,7 +676,7 @@ Booster <- R6::R6Class( # Store predictions call_state <- 0L npred <- 0L - npred <- .Call( + .Call( LGBM_BoosterGetNumPredict_R , private$handle , as.integer(idx - 1L) @@ -695,7 +692,7 @@ Booster <- R6::R6Class( # Use buffer call_state <- 0L - private$predict_buffer[[data_name]] <- .Call( + .Call( LGBM_BoosterGetPredict_R , private$handle , as.integer(idx - 1L) @@ -719,7 +716,7 @@ Booster <- R6::R6Class( act_len <- 0L buf <- raw(buf_len) call_state <- 0L - buf <- .Call( + .Call( LGBM_BoosterGetEvalNames_R , private$handle , buf_len @@ -731,7 +728,7 @@ Booster <- R6::R6Class( buf_len <- act_len buf <- raw(buf_len) call_state <- 0L - buf <- .Call( + .Call( LGBM_BoosterGetEvalNames_R , private$handle , buf_len @@ -781,11 +778,13 @@ Booster <- R6::R6Class( # Create evaluation values tmp_vals <- numeric(length(private$eval_names)) - tmp_vals <- .Call( + call_state <- 0L + .Call( LGBM_BoosterGetEval_R , private$handle , as.integer(data_idx - 1L) , tmp_vals + , call_state ) # Loop through all evaluation names diff --git a/R-package/R/lgb.Dataset.R b/R-package/R/lgb.Dataset.R index 4ab18604cef8..07eae1b5ae8d 100644 --- a/R-package/R/lgb.Dataset.R +++ b/R-package/R/lgb.Dataset.R @@ -17,6 +17,7 @@ Dataset <- R6::R6Class( .Call( LGBM_DatasetFree_R , private$handle + , call_state ) private$handle <- NULL @@ -202,7 +203,7 @@ Dataset <- R6::R6Class( if (is.character(private$raw_data)) { call_state <- 0L - handle <- .Call( + .Call( LGBM_DatasetCreateFromFile_R , lgb.c_str(x = private$raw_data) , params_str @@ -215,7 +216,7 @@ Dataset <- R6::R6Class( # Are we using a matrix? call_state <- 0L - handle <- .Call( + .Call( LGBM_DatasetCreateFromMat_R , private$raw_data , nrow(private$raw_data) @@ -232,7 +233,7 @@ Dataset <- R6::R6Class( } # Are we using a dgCMatrix (sparsed matrix column compressed) call_state <- 0L - handle <- .Call( + .Call( LGBM_DatasetCreateFromCSC_R , private$raw_data@p , private$raw_data@i @@ -265,7 +266,7 @@ Dataset <- R6::R6Class( # Construct subset call_state <- 0L - handle <- .Call( + .Call( LGBM_DatasetGetSubset_R , ref_handle , c(private$used_indices) # Adding c() fixes issue in R v3.5 @@ -342,21 +343,21 @@ Dataset <- R6::R6Class( # Get numeric data and numeric features call_state <- 0L + .Call( + LGBM_DatasetGetNumData_R + , private$handle + , num_row + , call_state + ) + call_state <- 0L + .Call( + LGBM_DatasetGetNumFeature_R + , private$handle + , num_col + , call_state + ) return( - c( - .Call( - LGBM_DatasetGetNumData_R - , private$handle - , num_row - , call_state - ), - .Call( - LGBM_DatasetGetNumFeature_R - , private$handle - , num_col - , call_state - ) - ) + c(num_row, num_col) ) } else if (is.matrix(private$raw_data) || methods::is(private$raw_data, "dgCMatrix")) { @@ -388,7 +389,7 @@ Dataset <- R6::R6Class( act_len <- 0L buf <- raw(buf_len) call_state <- 0L - buf <- .Call( + .Call( LGBM_DatasetGetFeatureNames_R , private$handle , buf_len @@ -399,7 +400,8 @@ Dataset <- R6::R6Class( if (act_len > buf_len) { buf_len <- act_len buf <- raw(buf_len) - buf <- .Call( + call_state <- 0L + .Call( LGBM_DatasetGetFeatureNames_R , private$handle , buf_len @@ -484,7 +486,7 @@ Dataset <- R6::R6Class( # Get field size of info info_len <- 0L call_state <- 0L - info_len <- .Call( + .Call( LGBM_DatasetGetFieldSize_R , private$handle , lgb.c_str(x = name) @@ -504,7 +506,7 @@ Dataset <- R6::R6Class( } call_state <- 0L - ret <- .Call( + .Call( LGBM_DatasetGetField_R , private$handle , lgb.c_str(x = name) diff --git a/R-package/R/lgb.Predictor.R b/R-package/R/lgb.Predictor.R index 9f8b30f43831..0ca21f5eb51a 100644 --- a/R-package/R/lgb.Predictor.R +++ b/R-package/R/lgb.Predictor.R @@ -40,7 +40,7 @@ Predictor <- R6::R6Class( # Create handle on it call_state <- 0L - handle <- .Call( + .Call( LGBM_BoosterCreateFromModelfile_R , lgb.c_str(x = modelfile) , handle @@ -73,14 +73,13 @@ Predictor <- R6::R6Class( cur_iter <- 0L call_state <- 0L - return( - .Call( - LGBM_BoosterGetCurrentIteration_R - , private$handle - , cur_iter - , call_state - ) + .Call( + LGBM_BoosterGetCurrentIteration_R + , private$handle + , cur_iter + , call_state ) + return(cur_iter) }, @@ -143,7 +142,7 @@ Predictor <- R6::R6Class( # Check number of predictions to do call_state <- 0L - npred <- .Call( + .Call( LGBM_BoosterCalcNumPredict_R , private$handle , as.integer(num_row) @@ -167,7 +166,7 @@ Predictor <- R6::R6Class( storage.mode(data) <- "double" } call_state <- 0L - preds <- .Call( + .Call( LGBM_BoosterPredictForMat_R , private$handle , data @@ -189,7 +188,7 @@ Predictor <- R6::R6Class( } # Check if data is a dgCMatrix (sparse matrix, column compressed format) call_state <- 0L - preds <- .Call( + .Call( LGBM_BoosterPredictForCSC_R , private$handle , data@p diff --git a/R-package/tests/testthat/test_dataset.R b/R-package/tests/testthat/test_dataset.R index 7e861b733cd4..4e291074cefe 100644 --- a/R-package/tests/testthat/test_dataset.R +++ b/R-package/tests/testthat/test_dataset.R @@ -76,7 +76,7 @@ test_that("lgb.Dataset: Dataset should be able to construct from matrix and retu handle <- lgb.null.handle() ref_handle <- NULL call_state <- 0L - handle <- .Call( + .Call( LGBM_DatasetCreateFromMat_R , rawData , nrow(rawData) @@ -87,6 +87,7 @@ test_that("lgb.Dataset: Dataset should be able to construct from matrix and retu , call_state ) expect_false(is.na(handle)) + call_state <- 0L .Call(LGBM_DatasetFree_R, handle, call_state) handle <- NULL }) From f89aa2c8eb5141725ff77aebbefc71ea49af07c5 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 25 Apr 2021 11:57:58 -0500 Subject: [PATCH 6/8] testing registration for CMake builds --- R-package/NAMESPACE | 2 +- R-package/tests/testthat/test_registration.R | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/R-package/NAMESPACE b/R-package/NAMESPACE index 92812c50dd73..aba100ab2a68 100644 --- a/R-package/NAMESPACE +++ b/R-package/NAMESPACE @@ -49,4 +49,4 @@ importFrom(jsonlite,fromJSON) importFrom(methods,is) importFrom(stats,quantile) importFrom(utils,read.delim) -useDynLib(lib_lightgbm , .registration = TRUE) +useDynLib(lib_lightgbm , LGBM_GetLastError_R, LGBM_DatasetCreateFromFile_R, LGBM_DatasetCreateFromCSC_R, LGBM_DatasetCreateFromMat_R, LGBM_DatasetGetSubset_R, LGBM_DatasetSetFeatureNames_R, LGBM_DatasetGetFeatureNames_R, LGBM_DatasetSaveBinary_R, LGBM_DatasetFree_R, LGBM_DatasetSetField_R, LGBM_DatasetGetFieldSize_R, LGBM_DatasetGetField_R, LGBM_DatasetUpdateParamChecking_R, LGBM_DatasetGetNumData_R, LGBM_DatasetGetNumFeature_R, LGBM_BoosterCreate_R, LGBM_BoosterFree_R, LGBM_BoosterCreateFromModelfile_R, LGBM_BoosterLoadModelFromString_R, LGBM_BoosterMerge_R, LGBM_BoosterAddValidData_R, LGBM_BoosterResetTrainingData_R, LGBM_BoosterResetParameter_R, LGBM_BoosterGetNumClasses_R, LGBM_BoosterUpdateOneIter_R, LGBM_BoosterUpdateOneIterCustom_R, LGBM_BoosterRollbackOneIter_R, LGBM_BoosterGetCurrentIteration_R, LGBM_BoosterGetUpperBoundValue_R, LGBM_BoosterGetLowerBoundValue_R, LGBM_BoosterGetEvalNames_R, LGBM_BoosterGetEval_R, LGBM_BoosterGetNumPredict_R, LGBM_BoosterGetPredict_R, LGBM_BoosterPredictForFile_R, LGBM_BoosterCalcNumPredict_R, LGBM_BoosterPredictForCSC_R, LGBM_BoosterPredictForMat_R, LGBM_BoosterSaveModel_R, LGBM_BoosterSaveModelToString_R, LGBM_BoosterDumpModel_R) diff --git a/R-package/tests/testthat/test_registration.R b/R-package/tests/testthat/test_registration.R index e22b8a5e23e2..0db927b32544 100644 --- a/R-package/tests/testthat/test_registration.R +++ b/R-package/tests/testthat/test_registration.R @@ -1,10 +1,7 @@ -ON_WINDOWS <- .Platform$OS.type == "windows" - # this test is used to catch silent errors in routine registration, # like the one documented in # https://github.com/microsoft/LightGBM/issues/4045#issuecomment-812289182 test_that("lightgbm routine registration worked", { - testthat::skip_if_not(ON_WINDOWS) dll_info <- getLoadedDLLs()[["lightgbm"]] From 2d8de84cbba0514f47c266b1e6703f51acae8583 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 25 Apr 2021 13:04:28 -0500 Subject: [PATCH 7/8] revert NAMESPACE changes --- R-package/NAMESPACE | 2 +- R-package/tests/testthat/test_registration.R | 14 -------- build_r.R | 37 ++++++++++++++++++++ 3 files changed, 38 insertions(+), 15 deletions(-) delete mode 100644 R-package/tests/testthat/test_registration.R diff --git a/R-package/NAMESPACE b/R-package/NAMESPACE index aba100ab2a68..92812c50dd73 100644 --- a/R-package/NAMESPACE +++ b/R-package/NAMESPACE @@ -49,4 +49,4 @@ importFrom(jsonlite,fromJSON) importFrom(methods,is) importFrom(stats,quantile) importFrom(utils,read.delim) -useDynLib(lib_lightgbm , LGBM_GetLastError_R, LGBM_DatasetCreateFromFile_R, LGBM_DatasetCreateFromCSC_R, LGBM_DatasetCreateFromMat_R, LGBM_DatasetGetSubset_R, LGBM_DatasetSetFeatureNames_R, LGBM_DatasetGetFeatureNames_R, LGBM_DatasetSaveBinary_R, LGBM_DatasetFree_R, LGBM_DatasetSetField_R, LGBM_DatasetGetFieldSize_R, LGBM_DatasetGetField_R, LGBM_DatasetUpdateParamChecking_R, LGBM_DatasetGetNumData_R, LGBM_DatasetGetNumFeature_R, LGBM_BoosterCreate_R, LGBM_BoosterFree_R, LGBM_BoosterCreateFromModelfile_R, LGBM_BoosterLoadModelFromString_R, LGBM_BoosterMerge_R, LGBM_BoosterAddValidData_R, LGBM_BoosterResetTrainingData_R, LGBM_BoosterResetParameter_R, LGBM_BoosterGetNumClasses_R, LGBM_BoosterUpdateOneIter_R, LGBM_BoosterUpdateOneIterCustom_R, LGBM_BoosterRollbackOneIter_R, LGBM_BoosterGetCurrentIteration_R, LGBM_BoosterGetUpperBoundValue_R, LGBM_BoosterGetLowerBoundValue_R, LGBM_BoosterGetEvalNames_R, LGBM_BoosterGetEval_R, LGBM_BoosterGetNumPredict_R, LGBM_BoosterGetPredict_R, LGBM_BoosterPredictForFile_R, LGBM_BoosterCalcNumPredict_R, LGBM_BoosterPredictForCSC_R, LGBM_BoosterPredictForMat_R, LGBM_BoosterSaveModel_R, LGBM_BoosterSaveModelToString_R, LGBM_BoosterDumpModel_R) +useDynLib(lib_lightgbm , .registration = TRUE) diff --git a/R-package/tests/testthat/test_registration.R b/R-package/tests/testthat/test_registration.R deleted file mode 100644 index 0db927b32544..000000000000 --- a/R-package/tests/testthat/test_registration.R +++ /dev/null @@ -1,14 +0,0 @@ -# this test is used to catch silent errors in routine registration, -# like the one documented in -# https://github.com/microsoft/LightGBM/issues/4045#issuecomment-812289182 -test_that("lightgbm routine registration worked", { - - dll_info <- getLoadedDLLs()[["lightgbm"]] - - # check that dynamic lookup has been disabled - expect_false(dll_info[["dynamicLookup"]]) - - # check that all the expected symbols show up - registered_routines <- getDLLRegisteredRoutines(dll_info[["path"]])[[".Call"]] - expect_gt(length(registered_routines), 20L) -}) diff --git a/build_r.R b/build_r.R index a4eeee228c6b..649aef05298c 100644 --- a/build_r.R +++ b/build_r.R @@ -10,6 +10,8 @@ INSTALL_AFTER_BUILD <- !("--skip-install" %in% args) TEMP_R_DIR <- file.path(getwd(), "lightgbm_r") TEMP_SOURCE_DIR <- file.path(TEMP_R_DIR, "src") +args <- c() + # [description] # Parse the content of commandArgs() into a structured # list. This returns a list with two sections. @@ -369,6 +371,41 @@ description_contents <- gsub( ) writeLines(description_contents, DESCRIPTION_FILE) +# CMake-based builds can't currently use R's builtin routine registration, +# so have to update NAMESPACE manually, with a statement like this: +# +# useDynLib(lib_lightgbm, LGBM_GetLastError_R, LGBM_DatasetCreateFromFile_R, ...) +# +# See https://cran.r-project.org/doc/manuals/r-release/R-exts.html#useDynLib for +# documentation of this approach, where the NAMESPACE file uses a statement like +# useDynLib(foo, myRoutine, myOtherRoutine) +NAMESPACE_FILE <- file.path(TEMP_R_DIR, "NAMESPACE") +namespace_contents <- readLines(NAMESPACE_FILE) +dynlib_line <- grep( + pattern = "^useDynLib" + , x = namespace_contents +) + +c_api_contents <- readLines(file.path(TEMP_SOURCE_DIR, "src", "lightgbm_R.h")) +c_api_contents <- c_api_contents[grepl("^LIGHTGBM_C_EXPORT", c_api_contents)] +c_api_contents <- gsub( + pattern = "LIGHTGBM_C_EXPORT LGBM_SE " + , replacement = "" + , x = c_api_contents +) +c_api_symbols <- gsub( + pattern = "\\(" + , replacement = "" + , x = c_api_contents +) +dynlib_statement <- paste0( + "useDynLib(lib_lightgbm, " + , paste0(c_api_symbols, collapse = ", ") + , ")" +) +namespace_contents[dynlib_line] <- dynlib_statement +writeLines(namespace_contents, NAMESPACE_FILE) + # NOTE: --keep-empty-dirs is necessary to keep the deep paths expected # by CMake while also meeting the CRAN req to create object files # on demand From 42d01c844783c216b18fdda8b574609a802c4d4b Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 25 Apr 2021 13:30:19 -0500 Subject: [PATCH 8/8] revert testing changes --- .appveyor.yml | 72 +-- .github/workflows/cuda.yml | 186 +++---- .github/workflows/linkchecker.yml | 60 +- .github/workflows/optional_checks.yml | 56 +- .github/workflows/python_package.yml | 152 ++--- .github/workflows/static_analysis.yml | 164 +++--- .vsts-ci.yml | 762 +++++++++++++------------- build_r.R | 2 - 8 files changed, 726 insertions(+), 728 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index ccb08022792a..b7d9fe589176 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -1,43 +1,43 @@ -# version: 3.2.1.99.{build} +version: 3.2.1.99.{build} -# image: Visual Studio 2015 -# platform: x64 -# configuration: # a trick to construct a build matrix with multiple Python versions -# - 3.7 +image: Visual Studio 2015 +platform: x64 +configuration: # a trick to construct a build matrix with multiple Python versions + - 3.7 -# # only build pull requests and -# # commits to 'master' -# branches: -# only: -# - master +# only build pull requests and +# commits to 'master' +branches: + only: + - master -# environment: -# matrix: -# - COMPILER: MSVC -# TASK: python -# - COMPILER: MINGW -# TASK: python +environment: + matrix: + - COMPILER: MSVC + TASK: python + - COMPILER: MINGW + TASK: python -# clone_depth: 5 +clone_depth: 5 -# install: -# - git submodule update --init --recursive # get `external_libs` folder -# - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) -# - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% -# - set PYTHON_VERSION=%CONFIGURATION% -# - set CONDA_ENV="test-env" -# - ps: | -# switch ($env:PYTHON_VERSION) { -# "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} -# "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} -# default {$env:MINICONDA = "C:\Miniconda37-x64"} -# } -# $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" -# $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" -# $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() +install: + - git submodule update --init --recursive # get `external_libs` folder + - set PATH=%PATH:C:\Program Files\Git\usr\bin;=% # delete sh.exe from PATH (mingw32-make fix) + - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + - set PYTHON_VERSION=%CONFIGURATION% + - set CONDA_ENV="test-env" + - ps: | + switch ($env:PYTHON_VERSION) { + "3.6" {$env:MINICONDA = "C:\Miniconda36-x64"} + "3.7" {$env:MINICONDA = "C:\Miniconda37-x64"} + default {$env:MINICONDA = "C:\Miniconda37-x64"} + } + $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" + $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" + $env:LGB_VER = (Get-Content $env:APPVEYOR_BUILD_FOLDER\VERSION.txt).trim() -# build: false +build: false -# test_script: -# - conda init powershell -# - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 +test_script: + - conda init powershell + - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test_windows.ps1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index b93e155b934e..959a2c717edf 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -1,96 +1,96 @@ -# name: CUDA Version +name: CUDA Version -# on: -# push: -# branches: -# - master -# pull_request: -# branches: -# - master +on: + push: + branches: + - master + pull_request: + branches: + - master -# env: -# github_actions: 'true' -# os_name: linux -# task: cuda -# conda_env: test-env +env: + github_actions: 'true' + os_name: linux + task: cuda + conda_env: test-env -# jobs: -# test: -# name: cuda ${{ matrix.cuda_version }} ${{ matrix.method }} (linux, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) -# runs-on: [self-hosted, linux] -# timeout-minutes: 60 -# strategy: -# fail-fast: false -# matrix: -# include: -# - method: source -# compiler: gcc -# python_version: 3.7 -# cuda_version: "11.2.2" -# - method: pip -# compiler: clang -# python_version: 3.8 -# cuda_version: "10.0" -# - method: wheel -# compiler: gcc -# python_version: 3.9 -# cuda_version: "9.0" -# steps: -# - name: Setup or update software on host machine -# run: | -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# apt-transport-https \ -# ca-certificates \ -# curl \ -# git \ -# gnupg-agent \ -# software-properties-common -# curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - -# sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y -# curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - -# curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list -# sudo apt-get update -# sudo apt-get install --no-install-recommends -y \ -# containerd.io \ -# docker-ce \ -# docker-ce-cli \ -# nvidia-docker2 -# sudo chmod a+rw /var/run/docker.sock -# sudo systemctl restart docker -# - name: Remove old folder with repository -# run: sudo rm -rf $GITHUB_WORKSPACE -# - name: Checkout repository -# uses: actions/checkout@v1 -# with: -# fetch-depth: 5 -# submodules: true -# - name: Setup and run tests -# run: | -# export ROOT_DOCKER_FOLDER=/LightGBM -# cat > docker.env < docker-script.sh < docker.env < docker-script.sh < docker.env < docker-script.sh < docker.env < docker-script.sh <