diff --git a/.ci/conda-envs/ci-core.txt b/.ci/conda-envs/ci-core.txt index 46d20963ed98..ec6dddd41a36 100644 --- a/.ci/conda-envs/ci-core.txt +++ b/.ci/conda-envs/ci-core.txt @@ -23,7 +23,7 @@ joblib>=1.3.2 matplotlib-base>=3.7.3 numpy>=1.24.4 pandas>2.0 -pyarrow-core>=6.0 +pyarrow-core>=16.0 python-graphviz>=0.20.3 scikit-learn>=1.3.2 scipy>=1.1 diff --git a/.ci/test-windows.ps1 b/.ci/test-windows.ps1 index 264c13961aff..49fd76b7a3f1 100644 --- a/.ci/test-windows.ps1 +++ b/.ci/test-windows.ps1 @@ -66,7 +66,7 @@ if ($env:TASK -eq "swig") { conda init powershell conda activate conda config --set always_yes yes --set changeps1 no -conda update -q -y conda "python=$env:PYTHON_VERSION[build=*cpython]" +conda update -q -y conda "python=$env:PYTHON_VERSION=*_cp*" if ($env:PYTHON_VERSION -eq "3.7") { $env:CONDA_REQUIREMENT_FILE = "$env:BUILD_SOURCESDIRECTORY/.ci/conda-envs/ci-core-py37.txt" @@ -80,7 +80,7 @@ $condaParams = @( "-y", "-n", "$env:CONDA_ENV", "--file", "$env:CONDA_REQUIREMENT_FILE", - "python=$env:PYTHON_VERSION[build=*cpython]" + "python=$env:PYTHON_VERSION[build=*_cp*]" ) conda create @condaParams ; Assert-Output $? diff --git a/.ci/test.sh b/.ci/test.sh index 82c159064a33..e9d97b42293a 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -2,6 +2,9 @@ set -e -E -o -u pipefail +# TODO(jameslamb): revert before merging +set -x + # defaults CONDA_ENV="test-env" IN_UBUNTU_BASE_CONTAINER=${IN_UBUNTU_BASE_CONTAINER:-"false"} @@ -64,8 +67,9 @@ if [[ "$TASK" == "cpp-tests" ]]; then exit 0 fi -# including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy -CONDA_PYTHON_REQUIREMENT="python=${PYTHON_VERSION}[build=*cpython]" +# including python=version=*_cp* to ensure that conda prefers CPython and doesn't fall back to +# other implementations like pypy +CONDA_PYTHON_REQUIREMENT="python=${PYTHON_VERSION}=*_cp*" if [[ $TASK == "if-else" ]]; then conda create -q -y -n "${CONDA_ENV}" "${CONDA_PYTHON_REQUIREMENT}" numpy diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index f6bb354bc5a9..9e6968ebcf96 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -81,19 +81,19 @@ jobs: include: - method: wheel compiler: gcc - python_version: "3.10" + python_version: "3.11" cuda_version: "12.6.1" linux_version: "ubuntu22.04" task: cuda - method: source compiler: gcc - python_version: "3.12" + python_version: "3.13" cuda_version: "12.2.2" linux_version: "ubuntu22.04" task: cuda - method: pip compiler: clang - python_version: "3.11" + python_version: "3.12" cuda_version: "11.8.0" linux_version: "ubuntu20.04" task: cuda diff --git a/.github/workflows/linkchecker.yml b/.github/workflows/linkchecker.yml index 42391bb31da0..4f9dc6dfb559 100644 --- a/.github/workflows/linkchecker.yml +++ b/.github/workflows/linkchecker.yml @@ -10,7 +10,7 @@ on: env: COMPILER: gcc OS_NAME: 'linux' - PYTHON_VERSION: '3.12' + PYTHON_VERSION: '3.13' TASK: 'check-links' jobs: diff --git a/.github/workflows/python_package.yml b/.github/workflows/python_package.yml index acc40026f9b8..1963fb01237c 100644 --- a/.github/workflows/python_package.yml +++ b/.github/workflows/python_package.yml @@ -28,32 +28,32 @@ jobs: include: - os: macos-13 task: regular - python_version: '3.10' + python_version: '3.11' - os: macos-13 task: sdist - python_version: '3.11' + python_version: '3.12' - os: macos-13 task: bdist - python_version: '3.8' + python_version: '3.9' - os: macos-13 task: if-else - python_version: '3.9' + python_version: '3.10' - os: macos-14 task: bdist method: wheel - python_version: '3.10' + python_version: '3.11' - os: macos-13 task: mpi method: source - python_version: '3.11' + python_version: '3.12' - os: macos-13 task: mpi method: pip - python_version: '3.12' + python_version: '3.13' - os: macos-13 task: mpi method: wheel - python_version: '3.9' + python_version: '3.10' steps: - name: Checkout repository uses: actions/checkout@v4 @@ -113,12 +113,19 @@ jobs: --rm \ -v $(pwd):/opt/lgb-build \ -w /opt/lgb-build \ - python:3.12 \ + python:3.13 \ /bin/bash ./.ci/test-python-latest.sh - test-oldest-versions: + test-old-versions: name: Python - oldest supported versions (ubuntu-latest) runs-on: ubuntu-latest timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + # end-of-life Python versions + python_version: + - 3.7 + - 3.8 steps: - name: Checkout repository uses: actions/checkout@v4 @@ -140,12 +147,12 @@ jobs: --rm \ -v $(pwd):/opt/lgb-build \ -w /opt/lgb-build \ - python:3.7 \ + python:${{ matrix.python_version }} \ /bin/bash ./.ci/test-python-oldest.sh all-python-package-jobs-successful: if: always() runs-on: ubuntu-latest - needs: [test, test-latest-versions, test-oldest-versions] + needs: [test, test-latest-versions, test-old-versions] steps: - name: Note that all tests succeeded uses: re-actors/alls-green@v1.2.2 diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml deleted file mode 100644 index 66e05a18ba1f..000000000000 --- a/.github/workflows/r_package.yml +++ /dev/null @@ -1,350 +0,0 @@ -name: R-package - -on: - push: - branches: - - master - pull_request: - branches: - - master - -# automatically cancel in-progress builds if another commit is pushed -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - # in CMake-driven builds, parallelize compilation - CMAKE_BUILD_PARALLEL_LEVEL: 4 - # on Debian-based images, avoid interactive prompts - DEBIAN_FRONTEND: noninteractive - # parallelize compilation (extra important for Linux, where CRAN doesn't supply pre-compiled binaries) - MAKEFLAGS: "-j4" - # hack to get around this: - # https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html - _R_CHECK_SYSTEM_CLOCK_: 0 - # ignore R CMD CHECK NOTE checking how long it has - # been since the last submission - _R_CHECK_CRAN_INCOMING_REMOTE_: 0 - # CRAN ignores the "installed size is too large" NOTE, - # so our CI can too. Setting to a large value here just - # to catch extreme problems - _R_CHECK_PKG_SIZES_THRESHOLD_: 100 - -jobs: - test: - name: ${{ matrix.task }} (${{ matrix.os }}, ${{ matrix.compiler }}, R ${{ matrix.r_version }}, ${{ matrix.build_type }}) - runs-on: ${{ matrix.os }} - container: ${{ matrix.container }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - ################ - # CMake builds # - ################ - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cmake - container: 'ubuntu:22.04' - - os: ubuntu-latest - task: r-package - compiler: clang - r_version: 4.3 - build_type: cmake - container: 'ubuntu:22.04' - - os: macos-13 - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cmake - container: null - - os: macos-13 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cmake - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MINGW - r_version: 3.6 - build_type: cmake - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MSYS - r_version: 4.3 - build_type: cmake - container: null - # Visual Studio 2019 - - os: windows-2019 - task: r-package - compiler: MSVC - toolchain: MSVC - r_version: 3.6 - build_type: cmake - container: null - # Visual Studio 2022 - - os: windows-2022 - task: r-package - compiler: MSVC - toolchain: MSVC - r_version: 4.3 - build_type: cmake - container: null - ############### - # CRAN builds # - ############### - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MINGW - r_version: 3.6 - build_type: cran - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MSYS - r_version: 4.3 - build_type: cran - container: null - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cran - container: 'ubuntu:22.04' - - os: macos-13 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cran - container: null - # macos-14 = arm64 - - os: macos-14 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cran - container: null - steps: - - name: Prevent conversion of line endings on Windows - if: startsWith(matrix.os, 'windows') - shell: pwsh - run: git config --global core.autocrlf false - - name: Install packages used by third-party actions - if: startsWith(matrix.os, 'ubuntu') - shell: bash - run: | - apt-get update -y - apt-get install --no-install-recommends -y \ - ca-certificates \ - dirmngr \ - gpg \ - gpg-agent \ - software-properties-common \ - sudo - # install newest version of git - # ref: - # - https://unix.stackexchange.com/a/170831/550004 - # - https://git-scm.com/download/linux - add-apt-repository ppa:git-core/ppa -y - apt-get update -y - apt-get install --no-install-recommends -y \ - git - - name: Trust git cloning LightGBM - if: startsWith(matrix.os, 'ubuntu') - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Install pandoc - uses: r-lib/actions/setup-pandoc@v2 - - name: Install tinytex - if: startsWith(matrix.os, 'windows') - uses: r-lib/actions/setup-tinytex@v2 - env: - CTAN_MIRROR: https://ctan.math.illinois.edu/systems/win32/miktex - TINYTEX_INSTALLER: TinyTeX - - name: Setup and run tests on Linux and macOS - if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu') - shell: bash - run: | - export TASK="${{ matrix.task }}" - export COMPILER="${{ matrix.compiler }}" - if [[ "${{ matrix.os }}" =~ ^macos ]]; then - export OS_NAME="macos" - elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then - export OS_NAME="linux" - export IN_UBUNTU_BASE_CONTAINER="true" - fi - export BUILD_DIRECTORY="$GITHUB_WORKSPACE" - export R_VERSION="${{ matrix.r_version }}" - export R_BUILD_TYPE="${{ matrix.build_type }}" - $GITHUB_WORKSPACE/.ci/setup.sh - $GITHUB_WORKSPACE/.ci/test.sh - - name: Setup and run tests on Windows - if: startsWith(matrix.os, 'windows') - shell: pwsh -command ". {0}" - run: | - $env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE - $env:LGB_VER = (Get-Content -TotalCount 1 $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim().replace('rc', '-') - $env:TOOLCHAIN = "${{ matrix.toolchain }}" - $env:R_VERSION = "${{ matrix.r_version }}" - $env:R_BUILD_TYPE = "${{ matrix.build_type }}" - $env:COMPILER = "${{ matrix.compiler }}" - $env:TASK = "${{ matrix.task }}" - & "$env:GITHUB_WORKSPACE/.ci/test-windows.ps1" - test-r-sanitizers: - name: r-sanitizers (ubuntu-latest, R-devel, ${{ matrix.compiler }} ASAN/UBSAN) - timeout-minutes: 60 - runs-on: ubuntu-latest - container: wch1/r-debug - strategy: - fail-fast: false - matrix: - include: - - r_customization: san - compiler: gcc - - r_customization: csan - compiler: clang - steps: - - name: Trust git cloning LightGBM - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Install packages - shell: bash - run: | - RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }} - RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit 1 - - name: Run tests with sanitizers - shell: bash - run: | - cd R-package/tests - exit_code=0 - RDscript${{ matrix.r_customization }} testthat.R >> tests.log 2>&1 || exit_code=-1 - cat ./tests.log - exit ${exit_code} - test-r-extra-checks: - name: r-package (${{ matrix.image }}, R-devel) - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - # references: - # * CRAN "additional checks": https://cran.r-project.org/web/checks/check_issue_kinds.html - # * images: https://r-hub.github.io/containers/containers.html - image: - - clang16 - - clang17 - - clang18 - - clang19 - - gcc14 - - intel - - rchk - runs-on: ubuntu-latest - container: ghcr.io/r-hub/containers/${{ matrix.image }}:latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Install pandoc - uses: r-lib/actions/setup-pandoc@v2 - - name: Install LaTeX - shell: bash - run: | - if type -f apt 2>&1 > /dev/null; then - apt-get update - apt-get install --no-install-recommends -y \ - devscripts \ - texinfo \ - texlive-latex-extra \ - texlive-latex-recommended \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - tidy \ - qpdf - else - yum update -y - yum install -y \ - devscripts \ - qpdf \ - texinfo \ - texinfo-tex \ - texlive-latex \ - tidy - fi - - name: Install packages and run tests - shell: bash - run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - sh build-cran-package.sh - - # 'rchk' isn't run through 'R CMD check', use the approach documented at - # https://r-hub.github.io/containers/local.html - if [[ "${{ matrix.image }}" =~ "rchk" ]]; then - r-check "$(pwd)" \ - | tee ./rchk-logs.txt 2>&1 - - # the '-v' exceptions below are from R/rchk itself and not LightGBM: - # https://github.com/kalibera/rchk/issues/22#issuecomment-656036156 - if grep -E '\[PB\]|ERROR' ./rchk-logs.txt \ - | grep -v 'too many states' \ - > /dev/null; \ - then - echo "rchk found issues" - exit 1 - else - echo "rchk did not find any issues" - exit 0 - fi - fi - - # 'testthat' is not needed by 'rchk', so avoid installing it until here - Rscript -e "install.packages('testthat', repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - - if [[ "${{ matrix.image }}" =~ "clang" ]]; then - # allowing the following NOTEs (produced by default in the clang images): - # - # * checking compilation flags used ... NOTE - # Compilation used the following non-portable flag(s): - # ‘-Wp,-D_FORTIFY_SOURCE=3’ - # - # even though CRAN itself sets that: - # https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang - # - declare -i allowed_notes=1 - else - declare -i allowed_notes=0 - fi - - bash .ci/run-r-cmd-check.sh \ - "$(echo lightgbm_$(head -1 VERSION.txt).tar.gz)" \ - "${allowed_notes}" - all-r-package-jobs-successful: - if: always() - runs-on: ubuntu-latest - needs: [test, test-r-sanitizers, test-r-extra-checks] - steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/static_analysis.yml b/.github/workflows/static_analysis.yml index 34e573e0eea6..50c65ad0ce56 100644 --- a/.github/workflows/static_analysis.yml +++ b/.github/workflows/static_analysis.yml @@ -19,7 +19,7 @@ env: COMPILER: 'gcc' MAKEFLAGS: '-j4' OS_NAME: 'linux' - PYTHON_VERSION: '3.12' + PYTHON_VERSION: '3.13' jobs: test: diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 40424840c82d..51527c29661d 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -94,19 +94,19 @@ jobs: matrix: regular: TASK: regular - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.11' sdist: TASK: sdist - PYTHON_VERSION: '3.8' + PYTHON_VERSION: '3.9' bdist: TASK: bdist - PYTHON_VERSION: '3.9' + PYTHON_VERSION: '3.10' inference: TASK: if-else mpi_source: TASK: mpi METHOD: source - PYTHON_VERSION: '3.9' + PYTHON_VERSION: '3.10' gpu_source: TASK: gpu METHOD: source @@ -159,7 +159,7 @@ jobs: TASK: sdist bdist: TASK: bdist - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.11' inference: TASK: if-else mpi_source: @@ -168,23 +168,23 @@ jobs: mpi_pip: TASK: mpi METHOD: pip - PYTHON_VERSION: '3.11' + PYTHON_VERSION: '3.12' mpi_wheel: TASK: mpi METHOD: wheel - PYTHON_VERSION: '3.9' + PYTHON_VERSION: '3.10' gpu_source: TASK: gpu METHOD: source - PYTHON_VERSION: '3.11' + PYTHON_VERSION: '3.12' gpu_pip: TASK: gpu METHOD: pip - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.11' gpu_wheel: TASK: gpu METHOD: wheel - PYTHON_VERSION: '3.9' + PYTHON_VERSION: '3.10' cpp_tests: TASK: cpp-tests METHOD: with-sanitizers @@ -293,10 +293,10 @@ jobs: matrix: regular: TASK: regular - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.11' sdist: TASK: sdist - PYTHON_VERSION: '3.9' + PYTHON_VERSION: '3.10' bdist: TASK: bdist swig: @@ -341,10 +341,10 @@ jobs: matrix: regular: TASK: regular - PYTHON_VERSION: '3.10' + PYTHON_VERSION: '3.11' sdist: TASK: sdist - PYTHON_VERSION: '3.9' + PYTHON_VERSION: '3.10' bdist: TASK: bdist swig: diff --git a/docs/env.yml b/docs/env.yml index 5c9c3dd73853..7e3db252faed 100644 --- a/docs/env.yml +++ b/docs/env.yml @@ -4,7 +4,7 @@ channels: - conda-forge dependencies: - breathe>=4.35 - - python=3.10 + - python=3.11 - r-base=4.3.2 - r-data.table=1.14.10 - r-jsonlite=1.8.8 diff --git a/python-package/pyproject.toml b/python-package/pyproject.toml index 8fcc85814db5..1e93fc61f679 100644 --- a/python-package/pyproject.toml +++ b/python-package/pyproject.toml @@ -16,6 +16,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering :: Artificial Intelligence" ] dependencies = [