Merge branch 'master' into 6021-fix_single_row_contention

.appveyor.yml

@@ -1,4 +1,4 @@
-version: 4.0.0.99.{build}
+version: 4.1.0.99.{build}
 
 image: Visual Studio 2015
 platform: x64

.ci/check_python_dists.sh

@@ -25,7 +25,7 @@ if [ $PY_MINOR_VER -gt 7 ]; then
         pydistcheck \
             --inspect \
             --ignore 'compiled-objects-have-debug-symbols,distro-too-large-compressed' \
-            --max-allowed-size-uncompressed '60M' \
+            --max-allowed-size-uncompressed '100M' \
             --max-allowed-files 800 \
             ${DIST_DIR}/* || exit -1
     elif { test $(uname -m) = "aarch64"; }; then

.ci/lint-cpp.sh

@@ -18,3 +18,28 @@ cmakelint \
     ${cmake_files} \
 || exit -1
 echo "done running cmakelint"
+
+echo "checking that all OpenMP pragmas specify num_threads()"
+get_omp_pragmas_without_num_threads() {
+    grep \
+        -n \
+        -R \
+        --include='*.c' \
+        --include='*.cc' \
+        --include='*.cpp' \
+        --include='*.h' \
+        --include='*.hpp' \
+        'pragma omp parallel' \
+    | grep -v ' num_threads' \
+    | grep -v 'openmp_wrapper.h'
+}
+PROBLEMATIC_LINES=$(
+    get_omp_pragmas_without_num_threads
+)
+if test "${PROBLEMATIC_LINES}" != ""; then
+    get_omp_pragmas_without_num_threads
+    echo "Found '#pragma omp parallel' not using explicit num_threads() configuration. Fix those."
+    echo "For details, see https://www.openmp.org/spec-html/5.0/openmpse14.html#x54-800002.6"
+    exit -1
+fi
+echo "done checking OpenMP pragmas"

.ci/lint_r_code.R

@@ -1,5 +1,5 @@
 
-library(lintr)
+loadNamespace("lintr")
 
 args <- commandArgs(
     trailingOnly = TRUE
@@ -33,30 +33,34 @@ LINTERS_TO_USE <- list(
     , "any_duplicated"       = lintr::any_duplicated_linter()
     , "any_is_na"            = lintr::any_is_na_linter()
     , "assignment"           = lintr::assignment_linter()
+    , "backport"             = lintr::backport_linter()
     , "boolean_arithmetic"   = lintr::boolean_arithmetic_linter()
     , "braces"               = lintr::brace_linter()
     , "class_equals"         = lintr::class_equals_linter()
     , "commas"               = lintr::commas_linter()
+    , "conjunct_test"        = lintr::conjunct_test_linter()
     , "duplicate_argument"   = lintr::duplicate_argument_linter()
     , "empty_assignment"     = lintr::empty_assignment_linter()
     , "equals_na"            = lintr::equals_na_linter()
+    , "fixed_regex"          = lintr::fixed_regex_linter()
     , "for_loop_index"       = lintr::for_loop_index_linter()
     , "function_left"        = lintr::function_left_parentheses_linter()
+    , "function_return"      = lintr::function_return_linter()
+    , "implicit_assignment"  = lintr::implicit_assignment_linter()
     , "implicit_integers"    = lintr::implicit_integer_linter()
     , "infix_spaces"         = lintr::infix_spaces_linter()
     , "inner_combine"        = lintr::inner_combine_linter()
     , "is_numeric"           = lintr::is_numeric_linter()
-    , "fixed_regex"          = lintr::fixed_regex_linter()
-    , "function_return"      = lintr::function_return_linter()
     , "lengths"              = lintr::lengths_linter()
+    , "line_length"          = lintr::line_length_linter(length = 120L)
     , "literal_coercion"     = lintr::literal_coercion_linter()
-    , "long_lines"           = lintr::line_length_linter(length = 120L)
     , "matrix"               = lintr::matrix_apply_linter()
     , "missing_argument"     = lintr::missing_argument_linter()
     , "non_portable_path"    = lintr::nonportable_path_linter()
     , "numeric_leading_zero" = lintr::numeric_leading_zero_linter()
     , "outer_negation"       = lintr::outer_negation_linter()
     , "package_hooks"        = lintr::package_hooks_linter()
+    , "paren_body"           = lintr::paren_body_linter()
     , "paste"                = lintr::paste_linter()
     , "quotes"               = lintr::quotes_linter()
     , "redundant_equals"     = lintr::redundant_equals_linter()
@@ -100,13 +104,15 @@ LINTERS_TO_USE <- list(
             "%>%" = pipe_text
             , "%.%" = pipe_text
             , "%..%" = pipe_text
+            , "|>" = pipe_text
             , "?" = interactive_text
             , "??" = interactive_text
         )
     )
     , "unnecessary_concatenation" = lintr::unnecessary_concatenation_linter()
     , "unnecessary_lambda"        = lintr::unnecessary_lambda_linter()
     , "unreachable_code"          = lintr::unreachable_code_linter()
+    , "unused_import"             = lintr::unused_import_linter()
     , "vector_logic"              = lintr::vector_logic_linter()
     , "whitespace"                = lintr::whitespace_linter()
 )

.ci/test.sh

@@ -37,7 +37,7 @@ fi
 CONDA_PYTHON_REQUIREMENT="python=$PYTHON_VERSION[build=*cpython]"
 
 if [[ $TASK == "if-else" ]]; then
-    conda create -q -y -n $CONDA_ENV ${CONDA_PYTHON_REQUIREMENT} numpy
+    mamba create -q -y -n $CONDA_ENV ${CONDA_PYTHON_REQUIREMENT} numpy
     source activate $CONDA_ENV
     mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build && cmake .. && make lightgbm -j4 || exit -1
     cd $BUILD_DIRECTORY/tests/cpp_tests && ../../lightgbm config=train.conf convert_model_language=cpp convert_model=../../src/boosting/gbdt_prediction.cpp && ../../lightgbm config=predict.conf output_result=origin.pred || exit -1
@@ -67,7 +67,7 @@ fi
 
 if [[ $TASK == "lint" ]]; then
     cd ${BUILD_DIRECTORY}
-    conda create -q -y -n $CONDA_ENV \
+    mamba create -q -y -n $CONDA_ENV \
         ${CONDA_PYTHON_REQUIREMENT} \
         cmakelint \
         cpplint \
@@ -87,10 +87,10 @@ fi
 
 if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
     cd $BUILD_DIRECTORY/docs
-    conda env create \
+    mamba env create \
         -n $CONDA_ENV \
         --file ./env.yml || exit -1
-    conda install \
+    mamba install \
         -q \
         -y \
         -n $CONDA_ENV \
@@ -124,11 +124,11 @@ fi
 # ref: https://github.com/microsoft/LightGBM/issues/6030
 CONSTRAINED_DEPENDENCIES="'dask-core>=2023.5.0' 'distributed>=2023.5.0' 'pandas>=2.0'"
 if [[ $PYTHON_VERSION == "3.7" ]]; then
-    CONSTRAINED_DEPENDENCIES="'dask-core' 'distributed 'pandas<2.0'"
+    CONSTRAINED_DEPENDENCIES="'dask-core' 'distributed' 'pandas<2.0'"
 fi
 
 # including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy
-conda create -q -y -n $CONDA_ENV \
+mamba create -q -y -n $CONDA_ENV \
     ${CONSTRAINED_DEPENDENCIES} \
     cloudpickle \
     joblib \
@@ -304,7 +304,7 @@ matplotlib.use\(\"Agg\"\)\
 ' plot_example.py  # prevent interactive window mode
     sed -i'.bak' 's/graph.render(view=True)/graph.render(view=False)/' plot_example.py
     # requirements for examples
-    conda install -q -y -n $CONDA_ENV \
+    mamba install -q -y -n $CONDA_ENV \
         h5py \
         ipywidgets \
         notebook

.ci/test_r_package_windows.ps1

@@ -55,12 +55,14 @@ Remove-From-Path ".*\\R\\.*"
 Remove-From-Path ".*R Client.*"
 Remove-From-Path ".*rtools40.*"
 Remove-From-Path ".*rtools42.*"
+Remove-From-Path ".*rtools43.*"
 Remove-From-Path ".*shells.*"
 Remove-From-Path ".*Strawberry.*"
 Remove-From-Path ".*tools.*"
 
 Remove-Item C:\rtools40 -Force -Recurse -ErrorAction Ignore
 Remove-Item C:\rtools42 -Force -Recurse -ErrorAction Ignore
+Remove-Item C:\rtools43 -Force -Recurse -ErrorAction Ignore
 
 # Get details needed for installing R components
 #
@@ -76,11 +78,11 @@ if ($env:R_MAJOR_VERSION -eq "3") {
   $env:RTOOLS_EXE_FILE = "rtools35-x86_64.exe"
   $env:R_WINDOWS_VERSION = "3.6.3"
 } elseif ($env:R_MAJOR_VERSION -eq "4") {
-  $RTOOLS_INSTALL_PATH = "C:\rtools42"
+  $RTOOLS_INSTALL_PATH = "C:\rtools43"
   $env:RTOOLS_BIN = "$RTOOLS_INSTALL_PATH\usr\bin"
   $env:RTOOLS_MINGW_BIN = "$RTOOLS_INSTALL_PATH\x86_64-w64-mingw32.static.posix\bin"
-  $env:RTOOLS_EXE_FILE = "rtools42-5253-5107.exe"
-  $env:R_WINDOWS_VERSION = "4.2.2"
+  $env:RTOOLS_EXE_FILE = "rtools43-5550-5548.exe"
+  $env:R_WINDOWS_VERSION = "4.3.1"
 } else {
   Write-Output "[ERROR] Unrecognized R version: $env:R_VERSION"
   Check-Output $false
@@ -203,6 +205,19 @@ if ($env:COMPILER -ne "MSVC") {
   }
 }
 
+# Checking that the correct R version was used
+if ($env:TOOLCHAIN -ne "MSVC") {
+  $checks = Select-String -Path "${LOG_FILE_NAME}" -Pattern "using R version $env:R_WINDOWS_VERSION"
+  $checks_cnt = $checks.Matches.length
+} else {
+  $checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "R version passed into FindLibR.* $env:R_WINDOWS_VERSION"
+  $checks_cnt = $checks.Matches.length
+}
+if ($checks_cnt -eq 0) {
+  Write-Output "Wrong R version was found (expected '$env:R_WINDOWS_VERSION'). Check the build logs."
+  Check-Output $False
+}
+
 # Checking that we actually got the expected compiler. The R package has some logic
 # to fail back to MinGW if MSVC fails, but for CI builds we need to check that the correct
 # compiler was used.

.github/workflows/cuda.yml

@@ -31,17 +31,17 @@ jobs:
           - method: wheel
             compiler: gcc
             python_version: "3.11"
-            cuda_version: "11.7.1"
+            cuda_version: "11.8.0"
             task: cuda
           - method: source
             compiler: gcc
             python_version: "3.9"
-            cuda_version: "10.0"
+            cuda_version: "12.2.0"
             task: cuda
           - method: pip
             compiler: clang
             python_version: "3.10"
-            cuda_version: "11.7.1"
+            cuda_version: "11.8.0"
             task: cuda
     steps:
       - name: Setup or update software on host machine
@@ -98,8 +98,10 @@ jobs:
             cuda_version="${{ matrix.cuda_version }}"
             cuda_major=${cuda_version%%.*}
             docker_img="nvcr.io/nvidia/cuda:${cuda_version}-devel"
-            if [[ ${cuda_major} -gt 10 ]]; then
-                docker_img="${docker_img}-ubuntu$(lsb_release -rs)" 
+            if [[ ${cuda_major} -eq 11 ]]; then
+                docker_img="${docker_img}-ubuntu18.04"
+            elif [[ ${cuda_major} -ge 12 ]]; then
+                docker_img="${docker_img}-ubuntu20.04"
             fi
             docker run --env-file docker.env -v "$GITHUB_WORKSPACE":"$ROOT_DOCKER_FOLDER" --rm --gpus all "$docker_img" /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
   all-cuda-jobs-successful:

.github/workflows/r_package.yml

@@ -86,7 +86,7 @@ jobs:
             task: r-package
             compiler: MINGW
             toolchain: MSYS
-            r_version: 4.2
+            r_version: 4.3
             build_type: cmake
             container: null
           # Visual Studio 2019
@@ -102,7 +102,7 @@ jobs:
             task: r-package
             compiler: MSVC
             toolchain: MSVC
-            r_version: 4.2
+            r_version: 4.3
             build_type: cmake
             container: null
           ###############
@@ -119,7 +119,7 @@ jobs:
             task: r-package
             compiler: MINGW
             toolchain: MSYS
-            r_version: 4.2
+            r_version: 4.3
             build_type: cran
             container: null
           - os: ubuntu-latest

.gitignore

@@ -139,8 +139,6 @@ publish/
 # Publish Web Output
 *.[Pp]ublish.xml
 *.azurePubxml
-# TODO: Comment the next line if you want to checkin your web deploy settings 
-# but database connection strings (with potential passwords) will be unencrypted
 *.pubxml
 *.publishproj
 

CMakeLists.txt

@@ -203,20 +203,25 @@ if(__INTEGRATE_OPENCL)
 endif()
 
 if(USE_CUDA)
-    find_package(CUDA 10.0 REQUIRED)
+    find_package(CUDA 11.0 REQUIRED)
     include_directories(${CUDA_INCLUDE_DIRS})
     set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler=${OpenMP_CXX_FLAGS} -Xcompiler=-fPIC -Xcompiler=-Wall")
 
-    set(CUDA_ARCHS "6.0" "6.1" "6.2" "7.0")
-    if(CUDA_VERSION VERSION_GREATER_EQUAL "10.0")
-        list(APPEND CUDA_ARCHS "7.5")
-    endif()
+    # reference for mapping of CUDA toolkit component versions to supported architectures ("compute capabilities"):
+    # https://en.wikipedia.org/wiki/CUDA#GPUs_supported
+    set(CUDA_ARCHS "6.0" "6.1" "6.2" "7.0" "7.5")
     if(CUDA_VERSION VERSION_GREATER_EQUAL "11.0")
         list(APPEND CUDA_ARCHS "8.0")
     endif()
     if(CUDA_VERSION VERSION_GREATER_EQUAL "11.1")
         list(APPEND CUDA_ARCHS "8.6")
     endif()
+    if(CUDA_VERSION VERSION_GREATER_EQUAL "11.5")
+        list(APPEND CUDA_ARCHS "8.7")
+    endif()
+    if(CUDA_VERSION VERSION_GREATER_EQUAL "11.8")
+        list(APPEND CUDA_ARCHS "9.0")
+    endif()
     list(POP_BACK CUDA_ARCHS CUDA_LAST_SUPPORTED_ARCH)
     list(APPEND CUDA_ARCHS "${CUDA_LAST_SUPPORTED_ARCH}+PTX")
     cuda_select_nvcc_arch_flags(CUDA_ARCH_FLAGS ${CUDA_ARCHS})
@@ -326,6 +331,13 @@ if(UNIX OR MINGW OR CYGWIN)
     CMAKE_CXX_FLAGS
     "${CMAKE_CXX_FLAGS} -std=c++11 -pthread -Wextra -Wall -Wno-ignored-attributes -Wno-unknown-pragmas -Wno-return-type"
   )
+  if(MINGW)
+    # ignore this warning: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=95353
+    set(
+      CMAKE_CXX_FLAGS
+      "${CMAKE_CXX_FLAGS} -Wno-stringop-overflow"
+    )
+  endif()
   if(USE_DEBUG)
       set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g -O0")
   else()
@@ -371,6 +383,13 @@ if(MSVC)
         CMAKE_CXX_FLAGS_RELWITHDEBINFO
     )
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /W4 /MP")
+    if(__BUILD_FOR_R)
+        # MSVC does not like this commit:
+        # https://github.com/wch/r-source/commit/fb52ac1a610571fcb8ac92d886b9fefcffaa7d48
+        #
+        # and raises "error C3646: 'private_data_c': unknown override specifier"
+        add_definitions(-DR_LEGACY_RCOMPLEX)
+    endif()
     if(USE_DEBUG)
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Od")
     else()

R-package/README.md

@@ -97,17 +97,18 @@ After installing `Rtools` and `CMake`, be sure the following paths are added to
     - If you have `Rtools` 4.0, example:
         - `C:\rtools40\mingw64\bin`
         - `C:\rtools40\usr\bin`
-    - If you have `Rtools` 4.2, example:
+    - If you have `Rtools` 4.2+, example:
         - `C:\rtools42\x86_64-w64-mingw32.static.posix\bin`
         - `C:\rtools42\usr\bin`
+        - **NOTE**: this is e.g. `rtools43\` for R 4.3
 * `CMake`
     - example: `C:\Program Files\CMake\bin`
 * `R`
     - example: `C:\Program Files\R\R-3.6.1\bin`
 
 NOTE: Two `Rtools` paths are required from `Rtools` 4.0 onwards because paths and the list of included software was changed in `Rtools` 4.0.
 
-NOTE: `Rtools42` takes a very different approach to the compiler toolchain than previous releases, and how you install it changes what is required to build packages. See ["Howto: Building R 4.2 and packages on Windows"](https://cran.r-project.org/bin/windows/base/howto-R-4.2.html).
+NOTE: `Rtools42` and later take a very different approach to the compiler toolchain than previous releases, and how you install it changes what is required to build packages. See ["Howto: Building R 4.2 and packages on Windows"](https://cran.r-project.org/bin/windows/base/howto-R-4.2.html).
 
 #### Windows Toolchain Options