Merge branch 'master' into docs/install-docs

.ci/test_r_package.sh

@@ -104,6 +104,18 @@ if [[ $OS_NAME == "macos" ]]; then
     sudo installer \
         -pkg $(pwd)/R.pkg \
         -target / || exit 1
+
+    # install tidy v5.8.0
+    # ref: https://groups.google.com/g/r-sig-mac/c/7u_ivEj4zhM
+    TIDY_URL=https://github.com/htacg/tidy-html5/releases/download/5.8.0/tidy-5.8.0-macos-x86_64+arm64.pkg
+    curl -sL ${TIDY_URL} -o tidy.pkg
+    sudo installer \
+        -pkg $(pwd)/tidy.pkg \
+        -target /
+
+    # ensure that this newer version of 'tidy' is used by 'R CMD check'
+    # ref: https://cran.r-project.org/doc/manuals/R-exts.html#Checking-packages
+    export R_TIDYCMD=/usr/local/bin/tidy
 fi
 
 # fix for issue where CRAN was not returning {lattice} and {evaluate} when using R 3.6
@@ -263,20 +275,25 @@ fi
 
 # this check makes sure that CI builds of the package
 # actually use MM_PREFETCH preprocessor definition
-if [[ $R_BUILD_TYPE == "cran" ]]; then
-    mm_prefetch_working=$(
-        cat $BUILD_LOG_FILE \
-        | grep --count -E "checking whether MM_PREFETCH work.*yes"
-    )
-else
-    mm_prefetch_working=$(
-        cat $BUILD_LOG_FILE \
-        | grep --count -E ".*Performing Test MM_PREFETCH - Success"
-    )
-fi
-if [[ $mm_prefetch_working -ne 1 ]]; then
-    echo "MM_PREFETCH test was not passed"
-    exit 1
+#
+# _mm_prefetch will not work on arm64 architecture
+# ref: https://github.com/microsoft/LightGBM/issues/4124
+if [[ $ARCH != "arm64" ]]; then
+    if [[ $R_BUILD_TYPE == "cran" ]]; then
+        mm_prefetch_working=$(
+            cat $BUILD_LOG_FILE \
+            | grep --count -E "checking whether MM_PREFETCH work.*yes"
+        )
+    else
+        mm_prefetch_working=$(
+            cat $BUILD_LOG_FILE \
+            | grep --count -E ".*Performing Test MM_PREFETCH - Success"
+        )
+    fi
+    if [[ $mm_prefetch_working -ne 1 ]]; then
+        echo "MM_PREFETCH test was not passed"
+        exit 1
+    fi
 fi
 
 # this check makes sure that CI builds of the package

.github/workflows/r_package.yml

@@ -55,12 +55,6 @@ jobs:
             r_version: 4.3
             build_type: cmake
             container: 'ubuntu:22.04'
-          - os: ubuntu-latest
-            task: r-package
-            compiler: clang
-            r_version: 3.6
-            build_type: cmake
-            container: 'ubuntu:18.04'
           - os: ubuntu-latest
             task: r-package
             compiler: clang
@@ -138,6 +132,13 @@ jobs:
             r_version: 4.3
             build_type: cran
             container: null
+          # macos-14 = arm64
+          - os: macos-14
+            task: r-package
+            compiler: clang
+            r_version: 4.3
+            build_type: cran
+            container: null
     steps:
       - name: Prevent conversion of line endings on Windows
         if: startsWith(matrix.os, 'windows')
@@ -188,12 +189,12 @@ jobs:
           CTAN_MIRROR: https://ctan.math.illinois.edu/systems/win32/miktex
           TINYTEX_INSTALLER: TinyTeX
       - name: Setup and run tests on Linux and macOS
-        if: matrix.os == 'macos-13' || matrix.os == 'ubuntu-latest'
+        if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu')
         shell: bash
         run: |
           export TASK="${{ matrix.task }}"
           export COMPILER="${{ matrix.compiler }}"
-          if [[ "${{ matrix.os }}" == "macos-13" ]]; then
+          if [[ "${{ matrix.os }}" =~ ^macos ]]; then
               export OS_NAME="macos"
           elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
               export OS_NAME="linux"

CMakeLists.txt

@@ -2,7 +2,6 @@ option(USE_MPI "Enable MPI-based distributed learning" OFF)
 option(USE_OPENMP "Enable OpenMP" ON)
 option(USE_GPU "Enable GPU-accelerated training" OFF)
 option(USE_SWIG "Enable SWIG to generate Java API" OFF)
-option(USE_HDFS "Enable HDFS support (EXPERIMENTAL)" OFF)
 option(USE_TIMETAG "Set to ON to output time costs" OFF)
 option(USE_CUDA "Enable CUDA-accelerated training " OFF)
 option(USE_DEBUG "Set to ON for Debug mode" OFF)
@@ -294,21 +293,6 @@ if(USE_CUDA)
     endforeach()
 endif()
 
-if(USE_HDFS)
-    message(
-      DEPRECATION
-      "HDFS support in LightGBM is deprecated, and will be removed in a future release.\
-      See https://github.com/microsoft/LightGBM/issues/6436.
-      "
-    )
-    find_package(JNI REQUIRED)
-    find_path(HDFS_INCLUDE_DIR hdfs.h REQUIRED)
-    find_library(HDFS_LIB NAMES hdfs REQUIRED)
-    include_directories(${HDFS_INCLUDE_DIR})
-    add_definitions(-DUSE_HDFS)
-    set(HDFS_CXX_LIBRARIES ${HDFS_LIB} ${JAVA_JVM_LIBRARY})
-endif()
-
 include(CheckCXXSourceCompiles)
 check_cxx_source_compiles("
 #include <xmmintrin.h>
@@ -647,10 +631,6 @@ if(USE_CUDA)
   target_link_libraries(_lightgbm PRIVATE ${histograms})
 endif()
 
-if(USE_HDFS)
-  target_link_libraries(lightgbm_objs PUBLIC ${HDFS_CXX_LIBRARIES})
-endif()
-
 if(WIN32)
     if(MINGW OR CYGWIN)
       target_link_libraries(lightgbm_objs PUBLIC ws2_32 iphlpapi)

build-python.sh

@@ -40,8 +40,6 @@
 #                                   Compile CUDA version.
 #     --gpu
 #                                   Compile GPU version.
-#     --hdfs
-#                                   Compile HDFS version.
 #     --integrated-opencl
 #                                   Compile integrated OpenCL version.
 #     --mingw
@@ -148,9 +146,6 @@ while [ $# -gt 0 ]; do
     --gpu)
         BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_GPU=ON"
         ;;
-    --hdfs)
-        BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.USE_HDFS=ON"
-        ;;
     --integrated-opencl)
         BUILD_ARGS="${BUILD_ARGS} --config-setting=cmake.define.__INTEGRATE_OPENCL=ON"
         ;;

docs/Installation-Guide.rst

@@ -618,39 +618,6 @@ Windows
 The CUDA version is not supported on Windows.
 Use the GPU version (``device_type=gpu``) for GPU acceleration on Windows.
 
-Build HDFS Version
-~~~~~~~~~~~~~~~~~~
-
-.. warning::
-   HDFS support in LightGBM is deprecated, and will be removed in a future release.
-   See https://github.com/microsoft/LightGBM/issues/6436.
-
-The HDFS version of LightGBM was tested on CDH-5.14.4 cluster.
-
-Linux
-^^^^^
-
-On Linux a HDFS version of LightGBM can be built using **CMake** and **gcc**.
-
-1. Install `CMake`_.
-
-2. Run the following commands:
-
-   .. code:: sh
-
-     git clone --recursive https://github.com/microsoft/LightGBM
-     cd LightGBM
-     cmake -B build -S . -DUSE_HDFS=ON
-     # if you have installed HDFS to a customized location, you should specify paths to HDFS headers (hdfs.h) and library (libhdfs.so) like the following:
-     # cmake \
-     #   -DUSE_HDFS=ON \
-     #   -DHDFS_LIB="/opt/cloudera/parcels/CDH-5.14.4-1.cdh5.14.4.p0.3/lib64/libhdfs.so" \
-     #   -DHDFS_INCLUDE_DIR="/opt/cloudera/parcels/CDH-5.14.4-1.cdh5.14.4.p0.3/include/" \
-     #   ..
-     cmake --build build -j4
-
-**Note**: In some rare cases you may need to install OpenMP runtime library separately (use your package manager and search for ``lib[g|i]omp`` for doing this).
-
 Build Java Wrapper
 ~~~~~~~~~~~~~~~~~~
 

docs/_static/js/script.js

@@ -29,7 +29,6 @@ $(function() {
             '#build-mpi-version',
             '#build-gpu-version',
             '#build-cuda-version',
-            '#build-hdfs-version',
             '#build-java-wrapper',
             '#build-c-unit-tests'
         ];

python-package/README.rst

@@ -151,23 +151,6 @@ All requirements from `Build from Sources section <#build-from-sources>`__ apply
 
 To use the CUDA version within Python, pass ``{"device": "cuda"}`` respectively in parameters.
 
-Build HDFS Version
-~~~~~~~~~~~~~~~~~~
-
-.. warning::
-   HDFS support in LightGBM is deprecated, and will be removed in a future release.
-   See https://github.com/microsoft/LightGBM/issues/6436.
-
-.. code:: sh
-
-    pip install lightgbm --config-settings=cmake.define.USE_HDFS=ON
-
-All requirements from `Build from Sources section <#build-from-sources>`__ apply for this installation option as well.
-
-**HDFS** library is needed: details for installation can be found in `Installation Guide <https://github.com/microsoft/LightGBM/blob/master/docs/Installation-Guide.rst#build-hdfs-version>`__.
-
-Note that the installation process of HDFS version was tested only on **Linux**.
-
 Build with MinGW-w64 on Windows
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -253,8 +236,6 @@ Run ``sh ./build-python.sh install --gpu`` to enable GPU support. All requiremen
 
 Run ``sh ./build-python.sh install --cuda`` to enable CUDA support. All requirements from `Build CUDA Version section <#build-cuda-version>`__ apply for this installation option as well.
 
-Run ``sh ./build-python.sh install --hdfs`` to enable HDFS support. All requirements from `Build HDFS Version section <#build-hdfs-version>`__ apply for this installation option as well.
-
 Run ``sh ./build-python.sh install --bit32``, if you want to use 32-bit version. All requirements from `Build 32-bit Version with 32-bit Python section <#build-32-bit-version-with-32-bit-python>`__ apply for this installation option as well.
 
 Run ``sh ./build-python.sh install --time-costs``, if you want to output time costs for different internal routines. All requirements from `Build with Time Costs Output section <#build-with-time-costs-output>`__ apply for this installation option as well.

src/io/config.cpp

@@ -40,9 +40,24 @@ void GetFirstValueAsInt(const std::unordered_map<std::string, std::vector<std::s
 }
 
 void Config::SetVerbosity(const std::unordered_map<std::string, std::vector<std::string>>& params) {
-  int verbosity = Config().verbosity;
-  GetFirstValueAsInt(params, "verbose", &verbosity);
-  GetFirstValueAsInt(params, "verbosity", &verbosity);
+  int verbosity = 1;
+
+  // if "verbosity" was found in params, prefer that to any other aliases
+  const auto verbosity_iter = params.find("verbosity");
+  if (verbosity_iter != params.end()) {
+    GetFirstValueAsInt(params, "verbosity", &verbosity);
+  } else {
+    // if "verbose" was found in params and "verbosity" was not, use that value
+    const auto verbose_iter = params.find("verbose");
+    if (verbose_iter != params.end()) {
+      GetFirstValueAsInt(params, "verbose", &verbosity);
+    } else {
+      // if "verbosity" and "verbose" were both missing from params, don't modify LightGBM's log level
+      return;
+    }
+  }
+
+  // otherwise, update LightGBM's log level based on the passed-in value
   if (verbosity < 0) {
     LightGBM::Log::ResetLogLevel(LightGBM::LogLevel::Fatal);
   } else if (verbosity == 0) {