From 05cd10fa3c73db01299634dfe8f6bbb2932fe4a9 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 11 Nov 2024 21:00:00 -0600 Subject: [PATCH 01/27] [python-package] adapt to scikit-learn 1.6 testing changes --- .appveyor.yml | 42 -- .github/workflows/cuda.yml | 136 ------ .github/workflows/r_package.yml | 367 ----------------- .vsts-ci.yml | 481 ---------------------- python-package/lightgbm/sklearn.py | 1 - tests/python_package_test/test_sklearn.py | 38 +- 6 files changed, 34 insertions(+), 1031 deletions(-) delete mode 100644 .appveyor.yml delete mode 100644 .github/workflows/cuda.yml delete mode 100644 .github/workflows/r_package.yml delete mode 100644 .vsts-ci.yml diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 3819447db96b..000000000000 --- a/.appveyor.yml +++ /dev/null @@ -1,42 +0,0 @@ -version: 4.5.0.99.{build} - -image: Visual Studio 2015 -platform: x64 -configuration: - - '3.8' - -# only build on 'master' and pull requests targeting it -branches: - only: - - master - -environment: - matrix: - - COMPILER: MSVC - TASK: python - - COMPILER: MINGW - TASK: python - -clone_depth: 5 - -install: - - git submodule update --init --recursive # get `external_libs` folder - - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% - - set PYTHON_VERSION=%CONFIGURATION% - - ps: | - $env:ALLOW_SKIP_ARROW_TESTS = "1" - $env:APPVEYOR = "true" - $env:CMAKE_BUILD_PARALLEL_LEVEL = 4 - $env:MINICONDA = "C:\Miniconda3-x64" - $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" - $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" - -build: false - -test_script: - - conda config --remove channels defaults - - conda config --add channels nodefaults - - conda config --add channels conda-forge - - conda config --set channel_priority strict - - conda init powershell - - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test-windows.ps1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml deleted file mode 100644 index f6bb354bc5a9..000000000000 --- a/.github/workflows/cuda.yml +++ /dev/null @@ -1,136 +0,0 @@ -name: CUDA Version - -on: - push: - branches: - - master - pull_request: - branches: - - master - # Run manually by clicking a button in the UI - workflow_dispatch: - inputs: - restart_docker: - description: 'Restart nvidia-docker on the runner before building?' - required: true - type: boolean - default: false - -# automatically cancel in-progress builds if another commit is pushed -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - # Optionally reinstall + restart docker on the runner before building. - # This is safe as long as only 1 of these jobs runs at a time. - restart-docker: - name: set up docker - runs-on: [self-hosted, linux] - timeout-minutes: 30 - steps: - - name: Setup or update software on host machine - if: ${{ inputs.restart_docker }} - run: | - # install core packages - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - apt-transport-https \ - ca-certificates \ - curl \ - gnupg-agent \ - lsb-release \ - software-properties-common - # set up nvidia-docker - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y - curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - - curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - containerd.io \ - docker-ce \ - docker-ce-cli \ - nvidia-docker2 - sudo chmod a+rw /var/run/docker.sock - sudo systemctl restart docker - - name: mark job successful - run: | - exit 0 - test: - name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (${{ matrix.linux_version }}, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) - runs-on: [self-hosted, linux] - needs: [restart-docker] - container: - image: nvcr.io/nvidia/cuda:${{ matrix.cuda_version }}-devel-${{ matrix.linux_version }} - env: - CMAKE_BUILD_PARALLEL_LEVEL: 4 - COMPILER: ${{ matrix.compiler }} - CONDA: /tmp/miniforge - DEBIAN_FRONTEND: noninteractive - METHOD: ${{ matrix.method }} - OS_NAME: linux - PYTHON_VERSION: ${{ matrix.python_version }} - TASK: ${{ matrix.task }} - SKBUILD_STRICT_CONFIG: true - options: --gpus all - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - include: - - method: wheel - compiler: gcc - python_version: "3.10" - cuda_version: "12.6.1" - linux_version: "ubuntu22.04" - task: cuda - - method: source - compiler: gcc - python_version: "3.12" - cuda_version: "12.2.2" - linux_version: "ubuntu22.04" - task: cuda - - method: pip - compiler: clang - python_version: "3.11" - cuda_version: "11.8.0" - linux_version: "ubuntu20.04" - task: cuda - steps: - - name: Install latest git and sudo - run: | - apt-get update - apt-get install --no-install-recommends -y \ - ca-certificates \ - software-properties-common - add-apt-repository ppa:git-core/ppa -y - apt-get update - apt-get install --no-install-recommends -y \ - git \ - sudo - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Setup and run tests - run: | - export BUILD_DIRECTORY="$GITHUB_WORKSPACE" - export PATH=$CONDA/bin:$PATH - - # check GPU usage - nvidia-smi - - # build and test - $GITHUB_WORKSPACE/.ci/setup.sh - $GITHUB_WORKSPACE/.ci/test.sh - all-cuda-jobs-successful: - if: always() - runs-on: ubuntu-latest - needs: [test] - steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml deleted file mode 100644 index 8811f53b61c0..000000000000 --- a/.github/workflows/r_package.yml +++ /dev/null @@ -1,367 +0,0 @@ -name: R-package - -on: - push: - branches: - - master - pull_request: - branches: - - master - -# automatically cancel in-progress builds if another commit is pushed -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -env: - # https://github.com/actions/checkout/issues/1590#issuecomment-2207052044 - # - # this could be removed (hopefully) when R 3.6 support is removed - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - # in CMake-driven builds, parallelize compilation - CMAKE_BUILD_PARALLEL_LEVEL: 4 - # on Debian-based images, avoid interactive prompts - DEBIAN_FRONTEND: noninteractive - # parallelize compilation (extra important for Linux, where CRAN doesn't supply pre-compiled binaries) - MAKEFLAGS: "-j4" - # hack to get around this: - # https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html - _R_CHECK_SYSTEM_CLOCK_: 0 - # ignore R CMD CHECK NOTE checking how long it has - # been since the last submission - _R_CHECK_CRAN_INCOMING_REMOTE_: 0 - # CRAN ignores the "installed size is too large" NOTE, - # so our CI can too. Setting to a large value here just - # to catch extreme problems - _R_CHECK_PKG_SIZES_THRESHOLD_: 100 - -jobs: - test: - name: ${{ matrix.task }} (${{ matrix.os }}, ${{ matrix.compiler }}, R ${{ matrix.r_version }}, ${{ matrix.build_type }}) - runs-on: ${{ matrix.os }} - container: ${{ matrix.container }} - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - include: - ################ - # CMake builds # - ################ - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 3.6 - build_type: cmake - container: 'ubuntu:18.04' - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cmake - container: 'ubuntu:22.04' - - os: ubuntu-latest - task: r-package - compiler: clang - r_version: 4.3 - build_type: cmake - container: 'ubuntu:22.04' - - os: macos-13 - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cmake - container: null - - os: macos-13 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cmake - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MINGW - r_version: 3.6 - build_type: cmake - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MSYS - r_version: 4.3 - build_type: cmake - container: null - # Visual Studio 2019 - - os: windows-2019 - task: r-package - compiler: MSVC - toolchain: MSVC - r_version: 3.6 - build_type: cmake - container: null - # Visual Studio 2022 - - os: windows-2022 - task: r-package - compiler: MSVC - toolchain: MSVC - r_version: 4.3 - build_type: cmake - container: null - ############### - # CRAN builds # - ############### - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MINGW - r_version: 3.6 - build_type: cran - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MSYS - r_version: 4.3 - build_type: cran - container: null - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cran - container: 'ubuntu:22.04' - - os: macos-13 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cran - container: null - # macos-14 = arm64 - - os: macos-14 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cran - container: null - steps: - - name: Prevent conversion of line endings on Windows - if: startsWith(matrix.os, 'windows') - shell: pwsh - run: git config --global core.autocrlf false - - name: Install packages used by third-party actions - if: startsWith(matrix.os, 'ubuntu') - shell: bash - run: | - apt-get update -y - apt-get install --no-install-recommends -y \ - ca-certificates \ - dirmngr \ - gpg \ - gpg-agent \ - software-properties-common \ - sudo - # install newest version of git - # ref: - # - https://unix.stackexchange.com/a/170831/550004 - # - https://git-scm.com/download/linux - add-apt-repository ppa:git-core/ppa -y - apt-get update -y - apt-get install --no-install-recommends -y \ - git - - name: Trust git cloning LightGBM - if: startsWith(matrix.os, 'ubuntu') - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Checkout repository - uses: actions/checkout@v3 - with: - fetch-depth: 5 - submodules: true - - name: Install pandoc - uses: r-lib/actions/setup-pandoc@v2 - if: matrix.container != 'ubuntu:18.04' - # R 3.6 binary isn't easily available on Ubuntu 18.04, - # but setup-pandoc>=2.7.1 is uses a too-new glibc for it. - # ref: https://github.com/microsoft/LightGBM/issues/6298 - - name: Install pandoc - uses: r-lib/actions/setup-pandoc@v2.6.0 - if: matrix.container == 'ubuntu:18.04' - - name: Install tinytex - if: startsWith(matrix.os, 'windows') - uses: r-lib/actions/setup-tinytex@v2 - env: - CTAN_MIRROR: https://ctan.math.illinois.edu/systems/win32/miktex - TINYTEX_INSTALLER: TinyTeX - - name: Setup and run tests on Linux and macOS - if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu') - shell: bash - run: | - export TASK="${{ matrix.task }}" - export COMPILER="${{ matrix.compiler }}" - if [[ "${{ matrix.os }}" =~ ^macos ]]; then - export OS_NAME="macos" - elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then - export OS_NAME="linux" - export IN_UBUNTU_BASE_CONTAINER="true" - fi - export BUILD_DIRECTORY="$GITHUB_WORKSPACE" - export R_VERSION="${{ matrix.r_version }}" - export R_BUILD_TYPE="${{ matrix.build_type }}" - $GITHUB_WORKSPACE/.ci/setup.sh - $GITHUB_WORKSPACE/.ci/test.sh - - name: Setup and run tests on Windows - if: startsWith(matrix.os, 'windows') - shell: pwsh -command ". {0}" - run: | - $env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE - $env:LGB_VER = (Get-Content -TotalCount 1 $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim().replace('rc', '-') - $env:TOOLCHAIN = "${{ matrix.toolchain }}" - $env:R_VERSION = "${{ matrix.r_version }}" - $env:R_BUILD_TYPE = "${{ matrix.build_type }}" - $env:COMPILER = "${{ matrix.compiler }}" - $env:TASK = "${{ matrix.task }}" - & "$env:GITHUB_WORKSPACE/.ci/test-windows.ps1" - test-r-sanitizers: - name: r-sanitizers (ubuntu-latest, R-devel, ${{ matrix.compiler }} ASAN/UBSAN) - timeout-minutes: 60 - runs-on: ubuntu-latest - container: wch1/r-debug - strategy: - fail-fast: false - matrix: - include: - - r_customization: san - compiler: gcc - - r_customization: csan - compiler: clang - steps: - - name: Trust git cloning LightGBM - run: | - git config --global --add safe.directory "${GITHUB_WORKSPACE}" - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Install packages - shell: bash - run: | - RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }} - RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit 1 - - name: Run tests with sanitizers - shell: bash - run: | - cd R-package/tests - exit_code=0 - RDscript${{ matrix.r_customization }} testthat.R >> tests.log 2>&1 || exit_code=-1 - cat ./tests.log - exit ${exit_code} - test-r-extra-checks: - name: r-package (${{ matrix.image }}, R-devel) - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - # references: - # * CRAN "additional checks": https://cran.r-project.org/web/checks/check_issue_kinds.html - # * images: https://r-hub.github.io/containers/containers.html - image: - - clang16 - - clang17 - - clang18 - - clang19 - - gcc14 - - intel - - rchk - runs-on: ubuntu-latest - container: ghcr.io/r-hub/containers/${{ matrix.image }}:latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 5 - submodules: true - - name: Install pandoc - uses: r-lib/actions/setup-pandoc@v2 - - name: Install LaTeX - shell: bash - run: | - if type -f apt 2>&1 > /dev/null; then - apt-get update - apt-get install --no-install-recommends -y \ - devscripts \ - texinfo \ - texlive-latex-extra \ - texlive-latex-recommended \ - texlive-fonts-recommended \ - texlive-fonts-extra \ - tidy \ - qpdf - else - yum update -y - yum install -y \ - devscripts \ - qpdf \ - texinfo \ - texinfo-tex \ - texlive-latex \ - tidy - fi - - name: Install packages and run tests - shell: bash - run: | - Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - sh build-cran-package.sh - - # 'rchk' isn't run through 'R CMD check', use the approach documented at - # https://r-hub.github.io/containers/local.html - if [[ "${{ matrix.image }}" =~ "rchk" ]]; then - r-check "$(pwd)" \ - | tee ./rchk-logs.txt 2>&1 - - # the '-v' exceptions below are from R/rchk itself and not LightGBM: - # https://github.com/kalibera/rchk/issues/22#issuecomment-656036156 - if grep -E '\[PB\]|ERROR' ./rchk-logs.txt \ - | grep -v 'too many states' \ - > /dev/null; \ - then - echo "rchk found issues" - exit 1 - else - echo "rchk did not find any issues" - exit 0 - fi - fi - - # 'testthat' is not needed by 'rchk', so avoid installing it until here - Rscript -e "install.packages('testthat', repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" - - if [[ "${{ matrix.image }}" =~ "clang" ]]; then - # allowing the following NOTEs (produced by default in the clang images): - # - # * checking compilation flags used ... NOTE - # Compilation used the following non-portable flag(s): - # ‘-Wp,-D_FORTIFY_SOURCE=3’ - # - # even though CRAN itself sets that: - # https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang - # - declare -i allowed_notes=1 - else - declare -i allowed_notes=0 - fi - - bash .ci/run-r-cmd-check.sh \ - "$(echo lightgbm_$(head -1 VERSION.txt).tar.gz)" \ - "${allowed_notes}" - all-r-package-jobs-successful: - if: always() - runs-on: ubuntu-latest - needs: [test, test-r-sanitizers, test-r-extra-checks] - steps: - - name: Note that all tests succeeded - uses: re-actors/alls-green@v1.2.2 - with: - jobs: ${{ toJSON(needs) }} diff --git a/.vsts-ci.yml b/.vsts-ci.yml deleted file mode 100644 index 3a111e10898e..000000000000 --- a/.vsts-ci.yml +++ /dev/null @@ -1,481 +0,0 @@ -trigger: - branches: - include: - - master - tags: - include: - - v* -pr: -- master -variables: - AZURE: 'true' - CMAKE_BUILD_PARALLEL_LEVEL: 4 - PYTHON_VERSION: '3.12' - runCodesignValidationInjection: false - skipComponentGovernanceDetection: true - Codeql.Enabled: false - Codeql.SkipTaskAutoInjection: true - DOTNET_CLI_TELEMETRY_OPTOUT: true - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - SKBUILD_STRICT_CONFIG: true -resources: - # The __work/ directory, where Azure DevOps writes the source files, needs to be read-write because - # LightGBM's CI jobs write files in the source directory. - # - # For all the containers included here, all other directories that Azure mounts in are mounted as read-only - # to minimize the risk of side effects from one run affecting future runs. - # ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container - containers: - - container: linux-artifact-builder - image: lightgbm/vsts-agent:manylinux_2_28_x86_64 - mountReadOnly: - work: false - externals: true - tools: true - tasks: true - - container: ubuntu-latest - image: 'ubuntu:22.04' - options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" - mountReadOnly: - work: false - externals: true - tools: true - tasks: true - - container: rbase - image: wch1/r-debug - mountReadOnly: - work: false - externals: true - tools: true - tasks: true -jobs: -########################################### -- job: Maintenance -########################################### - pool: mariner-20240410-0 - container: ubuntu-latest - # routine maintenance (like periodically deleting old files), - # to be run on 1 random CI runner in the self-hosted pool each runner - steps: - - script: | - print-diagnostics(){ - echo "---- df -h -m ----" - df -h -m - echo "---- docker system df ----" - /tmp/docker system df - echo "---- docker images ----" - /tmp/docker images - } - # check disk usage - print-diagnostics - # remove old containers, container images, volumes - # ref: https://stackoverflow.com/a/32723127/3986677) - echo "---- running 'docker system prune' ----" - /tmp/docker system prune \ - --all \ - --force \ - --filter until=720h - # check disk usage again - print-diagnostics - displayName: clean -########################################### -- job: Linux -########################################### - variables: - COMPILER: gcc - SETUP_CONDA: 'false' - OS_NAME: 'linux' - PRODUCES_ARTIFACTS: 'true' - pool: mariner-20240410-0 - container: linux-artifact-builder - strategy: - matrix: - regular: - TASK: regular - PYTHON_VERSION: '3.10' - sdist: - TASK: sdist - PYTHON_VERSION: '3.8' - bdist: - TASK: bdist - PYTHON_VERSION: '3.9' - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - PYTHON_VERSION: '3.9' - gpu_source: - TASK: gpu - METHOD: source - swig: - TASK: swig - steps: - - script: | - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.prependpath]/usr/lib64/openmpi/bin" - echo "##vso[task.prependpath]$CONDA/bin" - displayName: 'Set variables' - - script: | - git clean -d -f -x - displayName: 'Clean source directory' - - script: | - echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt' - displayName: 'Add commit hash to artifacts archive' - - task: Bash@3 - displayName: Setup - inputs: - filePath: $(Build.SourcesDirectory)/.ci/setup.sh - targetType: filePath - - task: Bash@3 - displayName: Test - inputs: - filePath: $(Build.SourcesDirectory)/.ci/test.sh - targetType: filePath - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container -########################################### -- job: Linux_latest -########################################### - variables: - COMPILER: clang-17 - DEBIAN_FRONTEND: 'noninteractive' - IN_UBUNTU_BASE_CONTAINER: 'true' - OS_NAME: 'linux' - SETUP_CONDA: 'true' - pool: mariner-20240410-0 - container: ubuntu-latest - strategy: - matrix: - regular: - TASK: regular - sdist: - TASK: sdist - bdist: - TASK: bdist - PYTHON_VERSION: '3.10' - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - mpi_pip: - TASK: mpi - METHOD: pip - PYTHON_VERSION: '3.11' - mpi_wheel: - TASK: mpi - METHOD: wheel - PYTHON_VERSION: '3.9' - gpu_source: - TASK: gpu - METHOD: source - PYTHON_VERSION: '3.11' - gpu_pip: - TASK: gpu - METHOD: pip - PYTHON_VERSION: '3.10' - gpu_wheel: - TASK: gpu - METHOD: wheel - PYTHON_VERSION: '3.9' - cpp_tests: - TASK: cpp-tests - METHOD: with-sanitizers - steps: - - script: | - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - CONDA=$HOME/miniforge - echo "##vso[task.setvariable variable=CONDA]$CONDA" - echo "##vso[task.prependpath]$CONDA/bin" - displayName: 'Set variables' - # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 - - script: | - /tmp/docker exec -t -u 0 ci-container \ - sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" - displayName: 'Install sudo' - - script: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends git - git clean -d -f -x - displayName: 'Clean source directory' - - task: Bash@3 - displayName: Setup - inputs: - filePath: $(Build.SourcesDirectory)/.ci/setup.sh - targetType: 'filePath' - - task: Bash@3 - displayName: Test - inputs: - filePath: $(Build.SourcesDirectory)/.ci/test.sh - targetType: 'filePath' -########################################### -- job: QEMU_multiarch -########################################### - variables: - BUILD_DIRECTORY: /LightGBM - COMPILER: gcc - PRODUCES_ARTIFACTS: 'true' - pool: - vmImage: ubuntu-22.04 - timeoutInMinutes: 180 - strategy: - matrix: - bdist: - TASK: bdist - ARCH: aarch64 - steps: - - script: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y \ - binfmt-support \ - qemu \ - qemu-user \ - qemu-user-static - displayName: 'Install QEMU' - - script: | - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - displayName: 'Enable Docker multi-architecture support' - - script: | - git clean -d -f -x - displayName: 'Clean source directory' - - script: | - cat > docker-script.sh < Optional["_sklearn_Tags"]: diff --git a/tests/python_package_test/test_sklearn.py b/tests/python_package_test/test_sklearn.py index 6eca66ff20d3..fcfd23adb7ba 100644 --- a/tests/python_package_test/test_sklearn.py +++ b/tests/python_package_test/test_sklearn.py @@ -17,11 +17,18 @@ from sklearn.metrics import accuracy_score, log_loss, mean_squared_error, r2_score from sklearn.model_selection import GridSearchCV, RandomizedSearchCV, train_test_split from sklearn.multioutput import ClassifierChain, MultiOutputClassifier, MultiOutputRegressor, RegressorChain -from sklearn.utils.estimator_checks import parametrize_with_checks +from sklearn.utils.estimator_checks import parametrize_with_checks as sklearn_parametrize_with_checks from sklearn.utils.validation import check_is_fitted import lightgbm as lgb -from lightgbm.compat import DATATABLE_INSTALLED, PANDAS_INSTALLED, dt_DataTable, pd_DataFrame, pd_Series +from lightgbm.compat import ( + DATATABLE_INSTALLED, + PANDAS_INSTALLED, + _sklearn_version, + dt_DataTable, + pd_DataFrame, + pd_Series, +) from .utils import ( assert_silent, @@ -35,6 +42,9 @@ softmax, ) +SKLEARN_MAJOR, SKLEARN_MINOR, *_ = _sklearn_version.split(".") +SKLEARN_VERSION_GTE_1_6 = int(SKLEARN_MAJOR) >= 1 and int(SKLEARN_MINOR) >= 6 + decreasing_generator = itertools.count(0, -1) estimator_classes = (lgb.LGBMModel, lgb.LGBMClassifier, lgb.LGBMRegressor, lgb.LGBMRanker) task_to_model_factory = { @@ -1432,7 +1442,28 @@ def test_getting_feature_names_in_pd_input(estimator_class): np.testing.assert_array_equal(model.feature_names_in_, X.columns) -@parametrize_with_checks([lgb.LGBMClassifier(), lgb.LGBMRegressor()]) +# Starting with scikit-learn 1.6 (https://github.com/scikit-learn/scikit-learn/pull/30149), +# the only API for marking estimator tests as expected to fail is to pass a keyword argument +# to parametrize_with_checks(). That function didn't accept additional arguments in earlier +# versions. +# +# This block defines a patched version of parametrize_with_checks() so lightgbm's tests +# can be compatible with scikit-learn <1.6 and >=1.6. +# +# This should be removed minimum supported scikit-learn version is at least 1.6. +if SKLEARN_VERSION_GTE_1_6: + parametrize_with_checks = sklearn_parametrize_with_checks +else: + + def parametrize_with_checks(estimator, *args, **kwargs): + return sklearn_parametrize_with_checks(estimator) + + +def _get_expected_failed_tests(estimator): + return estimator._more_tags()["_xfail_checks"] + + +@parametrize_with_checks([lgb.LGBMClassifier(), lgb.LGBMRegressor()], expected_failed_checks=_get_expected_failed_tests) def test_sklearn_integration(estimator, check): estimator.set_params(min_child_samples=1, min_data_in_bin=1) check(estimator) @@ -1457,7 +1488,6 @@ def test_sklearn_tags_should_correctly_reflect_lightgbm_specific_values(estimato assert sklearn_tags.input_tags.allow_nan is True assert sklearn_tags.input_tags.sparse is True assert sklearn_tags.target_tags.one_d_labels is True - assert sklearn_tags._xfail_checks == more_tags["_xfail_checks"] @pytest.mark.parametrize("task", all_tasks) From 2ee38634d87b38ea0a736b9e271e19217c11e564 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 11 Nov 2024 21:39:33 -0600 Subject: [PATCH 02/27] more sklearn changes --- python-package/lightgbm/__init__.py | 5 +---- python-package/lightgbm/compat.py | 10 ++++++++++ python-package/lightgbm/sklearn.py | 12 ++++++++++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/python-package/lightgbm/__init__.py b/python-package/lightgbm/__init__.py index b679b3f665b5..ee8a6a77b3f0 100644 --- a/python-package/lightgbm/__init__.py +++ b/python-package/lightgbm/__init__.py @@ -11,11 +11,8 @@ from .basic import Booster, Dataset, Sequence, register_logger from .callback import EarlyStopException, early_stopping, log_evaluation, record_evaluation, reset_parameter from .engine import CVBooster, cv, train +from .sklearn import LGBMClassifier, LGBMModel, LGBMRanker, LGBMRegressor -try: - from .sklearn import LGBMClassifier, LGBMModel, LGBMRanker, LGBMRegressor -except ImportError: - pass try: from .plotting import create_tree_digraph, plot_importance, plot_metric, plot_split_value_histogram, plot_tree except ImportError: diff --git a/python-package/lightgbm/compat.py b/python-package/lightgbm/compat.py index 96dee6522572..0b9444b0ecbf 100644 --- a/python-package/lightgbm/compat.py +++ b/python-package/lightgbm/compat.py @@ -14,6 +14,14 @@ from sklearn.utils.multiclass import check_classification_targets from sklearn.utils.validation import assert_all_finite, check_array, check_X_y + # sklearn.utils Tags types can be imported unconditionally once + # lightgbm's minimum scikit-learn version is 1.6 or higher + try: + from sklearn.utils import ClassifierTags as _sklearn_ClassifierTags + from sklearn.utils import RegressorTags as _sklearn_RegressorTags + except ImportError: + _sklearn_ClassifierTags = None + _sklearn_RegressorTags = None try: from sklearn.exceptions import NotFittedError from sklearn.model_selection import BaseCrossValidator, GroupKFold, StratifiedKFold @@ -140,6 +148,8 @@ class _LGBMRegressorBase: # type: ignore _LGBMCheckClassificationTargets = None _LGBMComputeSampleWeight = None _LGBMValidateData = None + _sklearn_ClassifierTags = None + _sklearn_RegressorTags = None _sklearn_version = None # additional scikit-learn imports only for type hints diff --git a/python-package/lightgbm/sklearn.py b/python-package/lightgbm/sklearn.py index 6947cc6298df..614e3c3cbe7f 100644 --- a/python-package/lightgbm/sklearn.py +++ b/python-package/lightgbm/sklearn.py @@ -40,6 +40,8 @@ _LGBMModelBase, _LGBMRegressorBase, _LGBMValidateData, + _sklearn_ClassifierTags, + _sklearn_RegressorTags, _sklearn_version, dt_DataTable, pd_DataFrame, @@ -1290,7 +1292,10 @@ def _more_tags(self) -> Dict[str, Any]: return tags def __sklearn_tags__(self) -> "_sklearn_Tags": - return LGBMModel.__sklearn_tags__(self) + tags = LGBMModel.__sklearn_tags__(self) + tags.estimator_type = "regressor" + tags.regressor_tags = _sklearn_RegressorTags(multi_label=False) + return tags def fit( # type: ignore[override] self, @@ -1349,7 +1354,10 @@ def _more_tags(self) -> Dict[str, Any]: return tags def __sklearn_tags__(self) -> "_sklearn_Tags": - return LGBMModel.__sklearn_tags__(self) + tags = LGBMModel.__sklearn_tags__(self) + tags.estimator_type = "classifier" + tags.classifier_tags = _sklearn_ClassifierTags(multi_class=True, multi_label=False) + return tags def fit( # type: ignore[override] self, From dd2b65c6bfd740422737d492c0472dbeee7fa817 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 11 Nov 2024 21:51:43 -0600 Subject: [PATCH 03/27] check if r-package tests are working --- .github/workflows/r_package.yml | 367 ++++++++++++++++++++++++++++++++ 1 file changed, 367 insertions(+) create mode 100644 .github/workflows/r_package.yml diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml new file mode 100644 index 000000000000..8811f53b61c0 --- /dev/null +++ b/.github/workflows/r_package.yml @@ -0,0 +1,367 @@ +name: R-package + +on: + push: + branches: + - master + pull_request: + branches: + - master + +# automatically cancel in-progress builds if another commit is pushed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + # https://github.com/actions/checkout/issues/1590#issuecomment-2207052044 + # + # this could be removed (hopefully) when R 3.6 support is removed + ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true + # in CMake-driven builds, parallelize compilation + CMAKE_BUILD_PARALLEL_LEVEL: 4 + # on Debian-based images, avoid interactive prompts + DEBIAN_FRONTEND: noninteractive + # parallelize compilation (extra important for Linux, where CRAN doesn't supply pre-compiled binaries) + MAKEFLAGS: "-j4" + # hack to get around this: + # https://stat.ethz.ch/pipermail/r-package-devel/2020q3/005930.html + _R_CHECK_SYSTEM_CLOCK_: 0 + # ignore R CMD CHECK NOTE checking how long it has + # been since the last submission + _R_CHECK_CRAN_INCOMING_REMOTE_: 0 + # CRAN ignores the "installed size is too large" NOTE, + # so our CI can too. Setting to a large value here just + # to catch extreme problems + _R_CHECK_PKG_SIZES_THRESHOLD_: 100 + +jobs: + test: + name: ${{ matrix.task }} (${{ matrix.os }}, ${{ matrix.compiler }}, R ${{ matrix.r_version }}, ${{ matrix.build_type }}) + runs-on: ${{ matrix.os }} + container: ${{ matrix.container }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + ################ + # CMake builds # + ################ + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 3.6 + build_type: cmake + container: 'ubuntu:18.04' + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 4.3 + build_type: cmake + container: 'ubuntu:22.04' + - os: ubuntu-latest + task: r-package + compiler: clang + r_version: 4.3 + build_type: cmake + container: 'ubuntu:22.04' + - os: macos-13 + task: r-package + compiler: gcc + r_version: 4.3 + build_type: cmake + container: null + - os: macos-13 + task: r-package + compiler: clang + r_version: 4.3 + build_type: cmake + container: null + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cmake + container: null + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.3 + build_type: cmake + container: null + # Visual Studio 2019 + - os: windows-2019 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 3.6 + build_type: cmake + container: null + # Visual Studio 2022 + - os: windows-2022 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 4.3 + build_type: cmake + container: null + ############### + # CRAN builds # + ############### + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cran + container: null + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.3 + build_type: cran + container: null + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 4.3 + build_type: cran + container: 'ubuntu:22.04' + - os: macos-13 + task: r-package + compiler: clang + r_version: 4.3 + build_type: cran + container: null + # macos-14 = arm64 + - os: macos-14 + task: r-package + compiler: clang + r_version: 4.3 + build_type: cran + container: null + steps: + - name: Prevent conversion of line endings on Windows + if: startsWith(matrix.os, 'windows') + shell: pwsh + run: git config --global core.autocrlf false + - name: Install packages used by third-party actions + if: startsWith(matrix.os, 'ubuntu') + shell: bash + run: | + apt-get update -y + apt-get install --no-install-recommends -y \ + ca-certificates \ + dirmngr \ + gpg \ + gpg-agent \ + software-properties-common \ + sudo + # install newest version of git + # ref: + # - https://unix.stackexchange.com/a/170831/550004 + # - https://git-scm.com/download/linux + add-apt-repository ppa:git-core/ppa -y + apt-get update -y + apt-get install --no-install-recommends -y \ + git + - name: Trust git cloning LightGBM + if: startsWith(matrix.os, 'ubuntu') + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 5 + submodules: true + - name: Install pandoc + uses: r-lib/actions/setup-pandoc@v2 + if: matrix.container != 'ubuntu:18.04' + # R 3.6 binary isn't easily available on Ubuntu 18.04, + # but setup-pandoc>=2.7.1 is uses a too-new glibc for it. + # ref: https://github.com/microsoft/LightGBM/issues/6298 + - name: Install pandoc + uses: r-lib/actions/setup-pandoc@v2.6.0 + if: matrix.container == 'ubuntu:18.04' + - name: Install tinytex + if: startsWith(matrix.os, 'windows') + uses: r-lib/actions/setup-tinytex@v2 + env: + CTAN_MIRROR: https://ctan.math.illinois.edu/systems/win32/miktex + TINYTEX_INSTALLER: TinyTeX + - name: Setup and run tests on Linux and macOS + if: startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'ubuntu') + shell: bash + run: | + export TASK="${{ matrix.task }}" + export COMPILER="${{ matrix.compiler }}" + if [[ "${{ matrix.os }}" =~ ^macos ]]; then + export OS_NAME="macos" + elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then + export OS_NAME="linux" + export IN_UBUNTU_BASE_CONTAINER="true" + fi + export BUILD_DIRECTORY="$GITHUB_WORKSPACE" + export R_VERSION="${{ matrix.r_version }}" + export R_BUILD_TYPE="${{ matrix.build_type }}" + $GITHUB_WORKSPACE/.ci/setup.sh + $GITHUB_WORKSPACE/.ci/test.sh + - name: Setup and run tests on Windows + if: startsWith(matrix.os, 'windows') + shell: pwsh -command ". {0}" + run: | + $env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE + $env:LGB_VER = (Get-Content -TotalCount 1 $env:BUILD_SOURCESDIRECTORY\VERSION.txt).trim().replace('rc', '-') + $env:TOOLCHAIN = "${{ matrix.toolchain }}" + $env:R_VERSION = "${{ matrix.r_version }}" + $env:R_BUILD_TYPE = "${{ matrix.build_type }}" + $env:COMPILER = "${{ matrix.compiler }}" + $env:TASK = "${{ matrix.task }}" + & "$env:GITHUB_WORKSPACE/.ci/test-windows.ps1" + test-r-sanitizers: + name: r-sanitizers (ubuntu-latest, R-devel, ${{ matrix.compiler }} ASAN/UBSAN) + timeout-minutes: 60 + runs-on: ubuntu-latest + container: wch1/r-debug + strategy: + fail-fast: false + matrix: + include: + - r_customization: san + compiler: gcc + - r_customization: csan + compiler: clang + steps: + - name: Trust git cloning LightGBM + run: | + git config --global --add safe.directory "${GITHUB_WORKSPACE}" + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 5 + submodules: true + - name: Install packages + shell: bash + run: | + RDscript${{ matrix.r_customization }} -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl', 'testthat'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + sh build-cran-package.sh --r-executable=RD${{ matrix.r_customization }} + RD${{ matrix.r_customization }} CMD INSTALL lightgbm_*.tar.gz || exit 1 + - name: Run tests with sanitizers + shell: bash + run: | + cd R-package/tests + exit_code=0 + RDscript${{ matrix.r_customization }} testthat.R >> tests.log 2>&1 || exit_code=-1 + cat ./tests.log + exit ${exit_code} + test-r-extra-checks: + name: r-package (${{ matrix.image }}, R-devel) + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + # references: + # * CRAN "additional checks": https://cran.r-project.org/web/checks/check_issue_kinds.html + # * images: https://r-hub.github.io/containers/containers.html + image: + - clang16 + - clang17 + - clang18 + - clang19 + - gcc14 + - intel + - rchk + runs-on: ubuntu-latest + container: ghcr.io/r-hub/containers/${{ matrix.image }}:latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 5 + submodules: true + - name: Install pandoc + uses: r-lib/actions/setup-pandoc@v2 + - name: Install LaTeX + shell: bash + run: | + if type -f apt 2>&1 > /dev/null; then + apt-get update + apt-get install --no-install-recommends -y \ + devscripts \ + texinfo \ + texlive-latex-extra \ + texlive-latex-recommended \ + texlive-fonts-recommended \ + texlive-fonts-extra \ + tidy \ + qpdf + else + yum update -y + yum install -y \ + devscripts \ + qpdf \ + texinfo \ + texinfo-tex \ + texlive-latex \ + tidy + fi + - name: Install packages and run tests + shell: bash + run: | + Rscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + sh build-cran-package.sh + + # 'rchk' isn't run through 'R CMD check', use the approach documented at + # https://r-hub.github.io/containers/local.html + if [[ "${{ matrix.image }}" =~ "rchk" ]]; then + r-check "$(pwd)" \ + | tee ./rchk-logs.txt 2>&1 + + # the '-v' exceptions below are from R/rchk itself and not LightGBM: + # https://github.com/kalibera/rchk/issues/22#issuecomment-656036156 + if grep -E '\[PB\]|ERROR' ./rchk-logs.txt \ + | grep -v 'too many states' \ + > /dev/null; \ + then + echo "rchk found issues" + exit 1 + else + echo "rchk did not find any issues" + exit 0 + fi + fi + + # 'testthat' is not needed by 'rchk', so avoid installing it until here + Rscript -e "install.packages('testthat', repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" + + if [[ "${{ matrix.image }}" =~ "clang" ]]; then + # allowing the following NOTEs (produced by default in the clang images): + # + # * checking compilation flags used ... NOTE + # Compilation used the following non-portable flag(s): + # ‘-Wp,-D_FORTIFY_SOURCE=3’ + # + # even though CRAN itself sets that: + # https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang + # + declare -i allowed_notes=1 + else + declare -i allowed_notes=0 + fi + + bash .ci/run-r-cmd-check.sh \ + "$(echo lightgbm_$(head -1 VERSION.txt).tar.gz)" \ + "${allowed_notes}" + all-r-package-jobs-successful: + if: always() + runs-on: ubuntu-latest + needs: [test, test-r-sanitizers, test-r-extra-checks] + steps: + - name: Note that all tests succeeded + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} From 175aa5418ca16cbf86815f60f5f2715d496dc1bc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 11 Nov 2024 22:03:42 -0600 Subject: [PATCH 04/27] pin to an older version of waldo in R tests --- .ci/test-r-package.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index ae205213d787..24ae3e8d86ff 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -108,10 +108,11 @@ if [[ $OS_NAME == "macos" ]]; then export R_TIDYCMD=/usr/local/bin/tidy fi -# fix for issue where CRAN was not returning {lattice} and {evaluate} when using R 3.6 +# fix for issue where CRAN was not returning {evaluate}, {lattice}, or {waldo} when using R 3.6 # "Warning: dependency ‘lattice’ is not available" if [[ "${R_MAJOR_VERSION}" == "3" ]]; then - Rscript --vanilla -e "install.packages(c('https://cran.r-project.org/src/contrib/Archive/lattice/lattice_0.20-41.tar.gz', 'https://cran.r-project.org/src/contrib/Archive/evaluate/evaluate_0.23.tar.gz'), repos = NULL, lib = '${R_LIB_PATH}')" + cran_archive='https://cran.r-project.org/src/contrib/Archive' + Rscript --vanilla -e "install.packages(c('${cran_archive}/lattice/lattice_0.20-41.tar.gz', '${cran_archive}/evaluate/evaluate_0.23.tar.gz', '${cran_archive}/waldo/waldo_0.6.0.tar.gz'), repos = NULL, lib = '${R_LIB_PATH}')" else # {Matrix} needs {lattice}, so this needs to run before manually installing {Matrix}. # This should be unnecessary on R >=4.4.0 From 829b8ce1fbef04ee94ab46310864a99b22da897c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 11 Nov 2024 22:17:28 -0600 Subject: [PATCH 05/27] even older waldo --- .ci/test-r-package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index 24ae3e8d86ff..ae408fb309d6 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -112,7 +112,7 @@ fi # "Warning: dependency ‘lattice’ is not available" if [[ "${R_MAJOR_VERSION}" == "3" ]]; then cran_archive='https://cran.r-project.org/src/contrib/Archive' - Rscript --vanilla -e "install.packages(c('${cran_archive}/lattice/lattice_0.20-41.tar.gz', '${cran_archive}/evaluate/evaluate_0.23.tar.gz', '${cran_archive}/waldo/waldo_0.6.0.tar.gz'), repos = NULL, lib = '${R_LIB_PATH}')" + Rscript --vanilla -e "install.packages(c('${cran_archive}/lattice/lattice_0.20-41.tar.gz', '${cran_archive}/evaluate/evaluate_0.23.tar.gz', '${cran_archive}/waldo/waldo_0.5.3.tar.gz'), repos = NULL, lib = '${R_LIB_PATH}')" else # {Matrix} needs {lattice}, so this needs to run before manually installing {Matrix}. # This should be unnecessary on R >=4.4.0 From 1172e328ca765c136f75c9982116af16231e595e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 11 Nov 2024 22:28:24 -0600 Subject: [PATCH 06/27] try dependencies = FALSE --- .ci/test-r-package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index ae408fb309d6..af4c74ef4575 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -112,7 +112,7 @@ fi # "Warning: dependency ‘lattice’ is not available" if [[ "${R_MAJOR_VERSION}" == "3" ]]; then cran_archive='https://cran.r-project.org/src/contrib/Archive' - Rscript --vanilla -e "install.packages(c('${cran_archive}/lattice/lattice_0.20-41.tar.gz', '${cran_archive}/evaluate/evaluate_0.23.tar.gz', '${cran_archive}/waldo/waldo_0.5.3.tar.gz'), repos = NULL, lib = '${R_LIB_PATH}')" + Rscript --vanilla -e "install.packages(c('${cran_archive}/lattice/lattice_0.20-41.tar.gz', '${cran_archive}/evaluate/evaluate_0.23.tar.gz', '${cran_archive}/waldo/waldo_0.5.3.tar.gz'), repos = NULL, lib = '${R_LIB_PATH}', dependencies = FALSE)" else # {Matrix} needs {lattice}, so this needs to run before manually installing {Matrix}. # This should be unnecessary on R >=4.4.0 From 29ef27e1b00236edf8c96706f07bdf4e1805a260 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 11 Nov 2024 23:22:55 -0600 Subject: [PATCH 07/27] try pinning a lot more packages on R 3.6 --- .ci/install-old-r-packages.R | 30 ++++++++++++++++++++++++++++++ .ci/test-r-package.sh | 3 +-- 2 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 .ci/install-old-r-packages.R diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R new file mode 100644 index 000000000000..72cad357fa44 --- /dev/null +++ b/.ci/install-old-r-packages.R @@ -0,0 +1,30 @@ +# [description] +# +# Installs a pinned set of packages that worked together +# as of the last R 3.6 release. +# + +cran_archive <- "https://cran.r-project.org/src/contrib/Archive" + +packages <- c( + "brio/brio_1.1.4.tar.gz" + , "cli/cli_3.6.2.tar.gz" + , "diffobj/diffobj_0.3.4.tar.gz" + , "evaluate/evaluate_0.23.tar.gz" + , "glue/glue_1.7.0.tar.gz" + , "lattice/lattice_0.20-41.tar.gz" + , "lifecycle/lifecycle_1.0.3.tar.gz" + , "pillar/pillar_1.8.1.tar.gz" + , "R6/R6_2.5.0.tar.gz" + , "rematch2/rematch2_2.1.1.tar.gz" + , "rlang/rlang_1.1.3.tar.gz" + , "testthat/testthat_3.2.1.tar.gz" + , "tibble/tibble_3.2.0.tar.gz" + , "waldo/waldo_0.5.3.tar.gz" + , "vctrs/vctrs_0.6.4.tar.gz" +) + +install.packages( + pkgs = paste(cran_archive, packages, sep = "/") + , repos = NULL +) diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index af4c74ef4575..a076fab0186c 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -111,8 +111,7 @@ fi # fix for issue where CRAN was not returning {evaluate}, {lattice}, or {waldo} when using R 3.6 # "Warning: dependency ‘lattice’ is not available" if [[ "${R_MAJOR_VERSION}" == "3" ]]; then - cran_archive='https://cran.r-project.org/src/contrib/Archive' - Rscript --vanilla -e "install.packages(c('${cran_archive}/lattice/lattice_0.20-41.tar.gz', '${cran_archive}/evaluate/evaluate_0.23.tar.gz', '${cran_archive}/waldo/waldo_0.5.3.tar.gz'), repos = NULL, lib = '${R_LIB_PATH}', dependencies = FALSE)" + Rscript --vanilla ./.ci/install-old-r-packages.R else # {Matrix} needs {lattice}, so this needs to run before manually installing {Matrix}. # This should be unnecessary on R >=4.4.0 From 8fd01c193ff1e73c58fb013198a79e56d293d089 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 11 Nov 2024 23:54:12 -0600 Subject: [PATCH 08/27] try even more packages --- .ci/install-old-r-packages.R | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index 72cad357fa44..173c9041f696 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -4,27 +4,36 @@ # as of the last R 3.6 release. # -cran_archive <- "https://cran.r-project.org/src/contrib/Archive" +.install_packages <- function(packages) { + install.packages( + pkgs = paste("https://cran.r-project.org/src/contrib/Archive", packages, sep = "/") + , repos = NULL + ) +} -packages <- c( +.install_packages(c( "brio/brio_1.1.4.tar.gz" , "cli/cli_3.6.2.tar.gz" + , "crayon/crayon_1.5.2.tar.gz" , "diffobj/diffobj_0.3.4.tar.gz" + , "digest/digest_0.6.36.tar.gz" + , "fansi/fansi_1.0.5.tar.gz" , "evaluate/evaluate_0.23.tar.gz" , "glue/glue_1.7.0.tar.gz" + , "jsonlite/jsonlite_1.8.8.tar.gz" , "lattice/lattice_0.20-41.tar.gz" , "lifecycle/lifecycle_1.0.3.tar.gz" + , "magrittr/magrittr_2.0.2.tar.gz" + , "pkgconfig/pkgconfig_2.0.2.tar.gz" + , "pkgload/pkgload_1.3.4.tar.gz" , "pillar/pillar_1.8.1.tar.gz" + , "processx/processx_3.8.3.tar.gz" , "R6/R6_2.5.0.tar.gz" , "rematch2/rematch2_2.1.1.tar.gz" , "rlang/rlang_1.1.3.tar.gz" , "testthat/testthat_3.2.1.tar.gz" , "tibble/tibble_3.2.0.tar.gz" - , "waldo/waldo_0.5.3.tar.gz" + , "utf8/utf8_1.2.3.tar.gz" , "vctrs/vctrs_0.6.4.tar.gz" -) - -install.packages( - pkgs = paste(cran_archive, packages, sep = "/") - , repos = NULL -) + , "waldo/waldo_0.5.3.tar.gz" +)) From 05245e9819e009e5f65bb59829dffdf714162d69 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 15:44:37 -0600 Subject: [PATCH 09/27] pin even more packages --- .ci/install-old-r-packages.R | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index 173c9041f696..c445a840937c 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -6,18 +6,24 @@ .install_packages <- function(packages) { install.packages( - pkgs = paste("https://cran.r-project.org/src/contrib/Archive", packages, sep = "/") + pkgs = ( + paste("https://cran.r-project.org/src/contrib/Archive", packages, sep = "/") + , "https://cran.r-project.org/src/contrib/praise_1.0.0.tar.gz" + ) , repos = NULL ) } .install_packages(c( "brio/brio_1.1.4.tar.gz" + , "callr/callr_3.7.5.tar.gz" , "cli/cli_3.6.2.tar.gz" , "crayon/crayon_1.5.2.tar.gz" + , "desc/desc_1.4.2.tar.gz" , "diffobj/diffobj_0.3.4.tar.gz" , "digest/digest_0.6.36.tar.gz" , "fansi/fansi_1.0.5.tar.gz" + , "fs/fs_1.6.4.tar.gz" , "evaluate/evaluate_0.23.tar.gz" , "glue/glue_1.7.0.tar.gz" , "jsonlite/jsonlite_1.8.8.tar.gz" @@ -25,15 +31,19 @@ , "lifecycle/lifecycle_1.0.3.tar.gz" , "magrittr/magrittr_2.0.2.tar.gz" , "pkgconfig/pkgconfig_2.0.2.tar.gz" + , "pkgbuild/pkgbuild_1.6.4.tar.gz" , "pkgload/pkgload_1.3.4.tar.gz" , "pillar/pillar_1.8.1.tar.gz" , "processx/processx_3.8.3.tar.gz" + , "ps/ps_1.8.0.tar.gz" , "R6/R6_2.5.0.tar.gz" , "rematch2/rematch2_2.1.1.tar.gz" , "rlang/rlang_1.1.3.tar.gz" + , "rprojroot/rprojroot_2.0.3.tar.gz" , "testthat/testthat_3.2.1.tar.gz" , "tibble/tibble_3.2.0.tar.gz" , "utf8/utf8_1.2.3.tar.gz" , "vctrs/vctrs_0.6.4.tar.gz" , "waldo/waldo_0.5.3.tar.gz" + , "withr/withr_3.0.1.tar.gz" )) From 40a663f78163f6d95b931f934906166fef50b33d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 18:18:30 -0600 Subject: [PATCH 10/27] fix syntax --- .ci/install-old-r-packages.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index c445a840937c..20e8af62b0a4 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -6,7 +6,7 @@ .install_packages <- function(packages) { install.packages( - pkgs = ( + pkgs = c( paste("https://cran.r-project.org/src/contrib/Archive", packages, sep = "/") , "https://cran.r-project.org/src/contrib/praise_1.0.0.tar.gz" ) From 256837e50e1d0f961862063597c8737ff5b27754 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 20:03:42 -0600 Subject: [PATCH 11/27] fix package versions --- .ci/install-old-r-packages.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index 20e8af62b0a4..3811b0047b9a 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -30,8 +30,8 @@ , "lattice/lattice_0.20-41.tar.gz" , "lifecycle/lifecycle_1.0.3.tar.gz" , "magrittr/magrittr_2.0.2.tar.gz" + , "pkgbuild/pkgbuild_1.4.4.tar.gz" , "pkgconfig/pkgconfig_2.0.2.tar.gz" - , "pkgbuild/pkgbuild_1.6.4.tar.gz" , "pkgload/pkgload_1.3.4.tar.gz" , "pillar/pillar_1.8.1.tar.gz" , "processx/processx_3.8.3.tar.gz" From feeb13ef008e619688834eea18d5e6ed68d3732c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 20:56:57 -0600 Subject: [PATCH 12/27] set lib --- .ci/install-old-r-packages.R | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index 3811b0047b9a..b5225dac20d6 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -10,6 +10,7 @@ paste("https://cran.r-project.org/src/contrib/Archive", packages, sep = "/") , "https://cran.r-project.org/src/contrib/praise_1.0.0.tar.gz" ) + , lib = Sys.getenv("R_LIBS") , repos = NULL ) } From 69ded072b04843d3044caa13f51905e76ee21821 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 21:04:15 -0600 Subject: [PATCH 13/27] R_LIB_PATH --- .ci/install-old-r-packages.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index b5225dac20d6..b4886209d429 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -10,7 +10,7 @@ paste("https://cran.r-project.org/src/contrib/Archive", packages, sep = "/") , "https://cran.r-project.org/src/contrib/praise_1.0.0.tar.gz" ) - , lib = Sys.getenv("R_LIBS") + , lib = Sys.getenv("R_LIB_PATH") , repos = NULL ) } From f04e658831b429d29692d8227fa217a0584810e8 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 21:15:27 -0600 Subject: [PATCH 14/27] R_LIBS --- .ci/install-old-r-packages.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index b4886209d429..b5225dac20d6 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -10,7 +10,7 @@ paste("https://cran.r-project.org/src/contrib/Archive", packages, sep = "/") , "https://cran.r-project.org/src/contrib/praise_1.0.0.tar.gz" ) - , lib = Sys.getenv("R_LIB_PATH") + , lib = Sys.getenv("R_LIBS") , repos = NULL ) } From 2be59e14db17c5c7ea7f0e4075769dfba099716d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 21:30:07 -0600 Subject: [PATCH 15/27] dependencies = FALSE --- .ci/install-old-r-packages.R | 1 + 1 file changed, 1 insertion(+) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index b5225dac20d6..df4a52285067 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -10,6 +10,7 @@ paste("https://cran.r-project.org/src/contrib/Archive", packages, sep = "/") , "https://cran.r-project.org/src/contrib/praise_1.0.0.tar.gz" ) + , dependencies = FALSE , lib = Sys.getenv("R_LIBS") , repos = NULL ) From c607c997648c3bc9de45e7106b43de22d8d9effc Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 21:30:36 -0600 Subject: [PATCH 16/27] comment out some R jobs --- .github/workflows/r_package.yml | 180 ++++++++++++++++---------------- 1 file changed, 90 insertions(+), 90 deletions(-) diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index 8811f53b61c0..b0a2c8ca6eff 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -54,96 +54,96 @@ jobs: r_version: 3.6 build_type: cmake container: 'ubuntu:18.04' - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cmake - container: 'ubuntu:22.04' - - os: ubuntu-latest - task: r-package - compiler: clang - r_version: 4.3 - build_type: cmake - container: 'ubuntu:22.04' - - os: macos-13 - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cmake - container: null - - os: macos-13 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cmake - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MINGW - r_version: 3.6 - build_type: cmake - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MSYS - r_version: 4.3 - build_type: cmake - container: null - # Visual Studio 2019 - - os: windows-2019 - task: r-package - compiler: MSVC - toolchain: MSVC - r_version: 3.6 - build_type: cmake - container: null - # Visual Studio 2022 - - os: windows-2022 - task: r-package - compiler: MSVC - toolchain: MSVC - r_version: 4.3 - build_type: cmake - container: null - ############### - # CRAN builds # - ############### - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MINGW - r_version: 3.6 - build_type: cran - container: null - - os: windows-latest - task: r-package - compiler: MINGW - toolchain: MSYS - r_version: 4.3 - build_type: cran - container: null - - os: ubuntu-latest - task: r-package - compiler: gcc - r_version: 4.3 - build_type: cran - container: 'ubuntu:22.04' - - os: macos-13 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cran - container: null - # macos-14 = arm64 - - os: macos-14 - task: r-package - compiler: clang - r_version: 4.3 - build_type: cran - container: null + # - os: ubuntu-latest + # task: r-package + # compiler: gcc + # r_version: 4.3 + # build_type: cmake + # container: 'ubuntu:22.04' + # - os: ubuntu-latest + # task: r-package + # compiler: clang + # r_version: 4.3 + # build_type: cmake + # container: 'ubuntu:22.04' + # - os: macos-13 + # task: r-package + # compiler: gcc + # r_version: 4.3 + # build_type: cmake + # container: null + # - os: macos-13 + # task: r-package + # compiler: clang + # r_version: 4.3 + # build_type: cmake + # container: null + # - os: windows-latest + # task: r-package + # compiler: MINGW + # toolchain: MINGW + # r_version: 3.6 + # build_type: cmake + # container: null + # - os: windows-latest + # task: r-package + # compiler: MINGW + # toolchain: MSYS + # r_version: 4.3 + # build_type: cmake + # container: null + # # Visual Studio 2019 + # - os: windows-2019 + # task: r-package + # compiler: MSVC + # toolchain: MSVC + # r_version: 3.6 + # build_type: cmake + # container: null + # # Visual Studio 2022 + # - os: windows-2022 + # task: r-package + # compiler: MSVC + # toolchain: MSVC + # r_version: 4.3 + # build_type: cmake + # container: null + # ############### + # # CRAN builds # + # ############### + # - os: windows-latest + # task: r-package + # compiler: MINGW + # toolchain: MINGW + # r_version: 3.6 + # build_type: cran + # container: null + # - os: windows-latest + # task: r-package + # compiler: MINGW + # toolchain: MSYS + # r_version: 4.3 + # build_type: cran + # container: null + # - os: ubuntu-latest + # task: r-package + # compiler: gcc + # r_version: 4.3 + # build_type: cran + # container: 'ubuntu:22.04' + # - os: macos-13 + # task: r-package + # compiler: clang + # r_version: 4.3 + # build_type: cran + # container: null + # # macos-14 = arm64 + # - os: macos-14 + # task: r-package + # compiler: clang + # r_version: 4.3 + # build_type: cran + # container: null steps: - name: Prevent conversion of line endings on Windows if: startsWith(matrix.os, 'windows') From 2cff81ce1d5b6d16a9400353cbd3cceb4d7be6e0 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 22:00:40 -0600 Subject: [PATCH 17/27] split into groups --- .ci/install-old-r-packages.R | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index df4a52285067..37874775e4ee 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -16,36 +16,48 @@ ) } +# when confronted with a bunch of URLs like this, install.packages() sometimes +# struggles to determine install order... so install packages in batches here, +# starting from the root of the dependency graph and working up +print("---- group 1 ----") .install_packages(c( "brio/brio_1.1.4.tar.gz" - , "callr/callr_3.7.5.tar.gz" , "cli/cli_3.6.2.tar.gz" , "crayon/crayon_1.5.2.tar.gz" - , "desc/desc_1.4.2.tar.gz" - , "diffobj/diffobj_0.3.4.tar.gz" , "digest/digest_0.6.36.tar.gz" + , "evaluate/evaluate_0.23.tar.gz" , "fansi/fansi_1.0.5.tar.gz" , "fs/fs_1.6.4.tar.gz" - , "evaluate/evaluate_0.23.tar.gz" , "glue/glue_1.7.0.tar.gz" , "jsonlite/jsonlite_1.8.8.tar.gz" , "lattice/lattice_0.20-41.tar.gz" - , "lifecycle/lifecycle_1.0.3.tar.gz" , "magrittr/magrittr_2.0.2.tar.gz" - , "pkgbuild/pkgbuild_1.4.4.tar.gz" , "pkgconfig/pkgconfig_2.0.2.tar.gz" + , "R6/R6_2.5.0.tar.gz" + , "rlang/rlang_1.1.3.tar.gz" + , "rprojroot/rprojroot_2.0.3.tar.gz" + , "utf8/utf8_1.2.3.tar.gz" + , "withr/withr_3.0.1.tar.gz" +)) + +print("---- group 2 ----") +.install_packages(c( + "diffobj/diffobj_0.3.4.tar.gz" +)) + +print("---- group 3 ----") +.install_packages(c( + "callr/callr_3.7.5.tar.gz" + , "desc/desc_1.4.2.tar.gz" + , "lifecycle/lifecycle_1.0.3.tar.gz" + , "pkgbuild/pkgbuild_1.4.4.tar.gz" , "pkgload/pkgload_1.3.4.tar.gz" , "pillar/pillar_1.8.1.tar.gz" , "processx/processx_3.8.3.tar.gz" , "ps/ps_1.8.0.tar.gz" - , "R6/R6_2.5.0.tar.gz" , "rematch2/rematch2_2.1.1.tar.gz" - , "rlang/rlang_1.1.3.tar.gz" - , "rprojroot/rprojroot_2.0.3.tar.gz" , "testthat/testthat_3.2.1.tar.gz" , "tibble/tibble_3.2.0.tar.gz" - , "utf8/utf8_1.2.3.tar.gz" , "vctrs/vctrs_0.6.4.tar.gz" , "waldo/waldo_0.5.3.tar.gz" - , "withr/withr_3.0.1.tar.gz" )) From dc43998187f1e4a8a11d4a447f4e2f718196d83e Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 22:07:34 -0600 Subject: [PATCH 18/27] split into groups --- .ci/install-old-r-packages.R | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index 37874775e4ee..d15d3088f957 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -33,6 +33,7 @@ print("---- group 1 ----") , "lattice/lattice_0.20-41.tar.gz" , "magrittr/magrittr_2.0.2.tar.gz" , "pkgconfig/pkgconfig_2.0.2.tar.gz" + , "ps/ps_1.8.0.tar.gz" , "R6/R6_2.5.0.tar.gz" , "rlang/rlang_1.1.3.tar.gz" , "rprojroot/rprojroot_2.0.3.tar.gz" @@ -42,22 +43,25 @@ print("---- group 1 ----") print("---- group 2 ----") .install_packages(c( - "diffobj/diffobj_0.3.4.tar.gz" + "desc/desc_1.4.2.tar.gz" + , "diffobj/diffobj_0.3.4.tar.gz" )) -print("---- group 3 ----") +print("--- group 3 ----") +.install_packages(c( + "waldo/waldo_0.5.3.tar.gz" +)) + +print("---- group 4 ----") .install_packages(c( "callr/callr_3.7.5.tar.gz" - , "desc/desc_1.4.2.tar.gz" , "lifecycle/lifecycle_1.0.3.tar.gz" , "pkgbuild/pkgbuild_1.4.4.tar.gz" , "pkgload/pkgload_1.3.4.tar.gz" , "pillar/pillar_1.8.1.tar.gz" , "processx/processx_3.8.3.tar.gz" - , "ps/ps_1.8.0.tar.gz" , "rematch2/rematch2_2.1.1.tar.gz" , "testthat/testthat_3.2.1.tar.gz" , "tibble/tibble_3.2.0.tar.gz" , "vctrs/vctrs_0.6.4.tar.gz" - , "waldo/waldo_0.5.3.tar.gz" )) From 2563bba4b08c0c814f9a8f6390b394e8bf4084dd Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 22:20:24 -0600 Subject: [PATCH 19/27] install more dependencies --- .ci/install-old-r-packages.R | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index d15d3088f957..49a42d50dddc 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -45,23 +45,27 @@ print("---- group 2 ----") .install_packages(c( "desc/desc_1.4.2.tar.gz" , "diffobj/diffobj_0.3.4.tar.gz" + , "processx/processx_3.8.3.tar.gz" )) print("--- group 3 ----") .install_packages(c( - "waldo/waldo_0.5.3.tar.gz" + "tibble/tibble_3.2.0.tar.gz" )) -print("---- group 4 ----") +print("--- group 4 ---") +.install_packages(c( + "rematch2/rematch2_2.1.1.tar.gz" + , "waldo/waldo_0.5.3.tar.gz" +)) + +print("---- group 5 ----") .install_packages(c( "callr/callr_3.7.5.tar.gz" , "lifecycle/lifecycle_1.0.3.tar.gz" , "pkgbuild/pkgbuild_1.4.4.tar.gz" , "pkgload/pkgload_1.3.4.tar.gz" , "pillar/pillar_1.8.1.tar.gz" - , "processx/processx_3.8.3.tar.gz" - , "rematch2/rematch2_2.1.1.tar.gz" , "testthat/testthat_3.2.1.tar.gz" - , "tibble/tibble_3.2.0.tar.gz" , "vctrs/vctrs_0.6.4.tar.gz" )) From 35f2c0bd9ee9c9c27732bd44ebf21d87dccd7ac6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 22:28:30 -0600 Subject: [PATCH 20/27] more rearranging --- .ci/install-old-r-packages.R | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index 49a42d50dddc..75846c08007c 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -45,27 +45,31 @@ print("---- group 2 ----") .install_packages(c( "desc/desc_1.4.2.tar.gz" , "diffobj/diffobj_0.3.4.tar.gz" + , "lifecycle/lifecycle_1.0.3.tar.gz" , "processx/processx_3.8.3.tar.gz" )) -print("--- group 3 ----") +print("--- group 3 ---") +.install_packages(c( + "callr/callr_3.7.5.tar.gz" + , "vctrs/vctrs_0.6.4.tar.gz" +)) + +print("--- group 4 ----") .install_packages(c( "tibble/tibble_3.2.0.tar.gz" )) -print("--- group 4 ---") +print("--- group 5 ---") .install_packages(c( "rematch2/rematch2_2.1.1.tar.gz" , "waldo/waldo_0.5.3.tar.gz" )) -print("---- group 5 ----") +print("---- group 6 ----") .install_packages(c( - "callr/callr_3.7.5.tar.gz" - , "lifecycle/lifecycle_1.0.3.tar.gz" - , "pkgbuild/pkgbuild_1.4.4.tar.gz" + "pkgbuild/pkgbuild_1.4.4.tar.gz" , "pkgload/pkgload_1.3.4.tar.gz" , "pillar/pillar_1.8.1.tar.gz" , "testthat/testthat_3.2.1.tar.gz" - , "vctrs/vctrs_0.6.4.tar.gz" )) From 530474a31fa3cab2429664469d6cb99cbfafe621 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 22:30:20 -0600 Subject: [PATCH 21/27] bit more --- .ci/install-old-r-packages.R | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index 75846c08007c..2ef4fb9d4f45 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -6,9 +6,10 @@ .install_packages <- function(packages) { install.packages( - pkgs = c( - paste("https://cran.r-project.org/src/contrib/Archive", packages, sep = "/") - , "https://cran.r-project.org/src/contrib/praise_1.0.0.tar.gz" + pkgs = paste( + "https://cran.r-project.org/src/contrib/Archive" + , packages + , sep = "/" ) , dependencies = FALSE , lib = Sys.getenv("R_LIBS") @@ -19,6 +20,15 @@ # when confronted with a bunch of URLs like this, install.packages() sometimes # struggles to determine install order... so install packages in batches here, # starting from the root of the dependency graph and working up + +# there was only a single release of {praise}, so there is no contrib/Archive link for it +install.packages( + pkgs = "https://cran.r-project.org/src/contrib/praise_1.0.0.tar.gz" + , dependencies = FALSE + , lib = Sys.getenv("R_LIBS") + , repos = NULL +) + print("---- group 1 ----") .install_packages(c( "brio/brio_1.1.4.tar.gz" From cffba58cc0667a7a541dcdb005eaacffe30f1449 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 22:37:18 -0600 Subject: [PATCH 22/27] tibble needs pillar --- .ci/install-old-r-packages.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index 2ef4fb9d4f45..08c5c39b6445 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -67,7 +67,8 @@ print("--- group 3 ---") print("--- group 4 ----") .install_packages(c( - "tibble/tibble_3.2.0.tar.gz" + "pillar/pillar_1.8.1.tar.gz" + , "tibble/tibble_3.2.0.tar.gz" )) print("--- group 5 ---") @@ -80,6 +81,5 @@ print("---- group 6 ----") .install_packages(c( "pkgbuild/pkgbuild_1.4.4.tar.gz" , "pkgload/pkgload_1.3.4.tar.gz" - , "pillar/pillar_1.8.1.tar.gz" , "testthat/testthat_3.2.1.tar.gz" )) From 44c77a877c3a18db64b6f0fdd9ccb7abd9943550 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 22:47:44 -0600 Subject: [PATCH 23/27] clean up --- .ci/install-old-r-packages.R | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index 08c5c39b6445..4fcc6693e60c 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -21,7 +21,7 @@ # struggles to determine install order... so install packages in batches here, # starting from the root of the dependency graph and working up -# there was only a single release of {praise}, so there is no contrib/Archive link for it +# there was only a single release of {praise}, so there is no contrib/Archive URL for it install.packages( pkgs = "https://cran.r-project.org/src/contrib/praise_1.0.0.tar.gz" , dependencies = FALSE @@ -29,7 +29,6 @@ install.packages( , repos = NULL ) -print("---- group 1 ----") .install_packages(c( "brio/brio_1.1.4.tar.gz" , "cli/cli_3.6.2.tar.gz" @@ -51,7 +50,6 @@ print("---- group 1 ----") , "withr/withr_3.0.1.tar.gz" )) -print("---- group 2 ----") .install_packages(c( "desc/desc_1.4.2.tar.gz" , "diffobj/diffobj_0.3.4.tar.gz" @@ -59,27 +57,23 @@ print("---- group 2 ----") , "processx/processx_3.8.3.tar.gz" )) -print("--- group 3 ---") .install_packages(c( "callr/callr_3.7.5.tar.gz" , "vctrs/vctrs_0.6.4.tar.gz" )) -print("--- group 4 ----") .install_packages(c( "pillar/pillar_1.8.1.tar.gz" , "tibble/tibble_3.2.0.tar.gz" )) -print("--- group 5 ---") .install_packages(c( - "rematch2/rematch2_2.1.1.tar.gz" + "pkgbuild/pkgbuild_1.4.4.tar.gz" + , "rematch2/rematch2_2.1.1.tar.gz" , "waldo/waldo_0.5.3.tar.gz" )) -print("---- group 6 ----") .install_packages(c( - "pkgbuild/pkgbuild_1.4.4.tar.gz" - , "pkgload/pkgload_1.3.4.tar.gz" + "pkgload/pkgload_1.3.4.tar.gz" , "testthat/testthat_3.2.1.tar.gz" )) From 94ef6a6b672657ff4960af770071b6f995c67b0c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 22:50:12 -0600 Subject: [PATCH 24/27] restore all CI --- .appveyor.yml | 42 +++ .github/workflows/cuda.yml | 136 ++++++++ .github/workflows/r_package.yml | 180 +++++------ .vsts-ci.yml | 481 ++++++++++++++++++++++++++++ python-package/lightgbm/__init__.py | 6 +- 5 files changed, 754 insertions(+), 91 deletions(-) create mode 100644 .appveyor.yml create mode 100644 .github/workflows/cuda.yml create mode 100644 .vsts-ci.yml diff --git a/.appveyor.yml b/.appveyor.yml new file mode 100644 index 000000000000..3819447db96b --- /dev/null +++ b/.appveyor.yml @@ -0,0 +1,42 @@ +version: 4.5.0.99.{build} + +image: Visual Studio 2015 +platform: x64 +configuration: + - '3.8' + +# only build on 'master' and pull requests targeting it +branches: + only: + - master + +environment: + matrix: + - COMPILER: MSVC + TASK: python + - COMPILER: MINGW + TASK: python + +clone_depth: 5 + +install: + - git submodule update --init --recursive # get `external_libs` folder + - set PATH=C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;%PATH% + - set PYTHON_VERSION=%CONFIGURATION% + - ps: | + $env:ALLOW_SKIP_ARROW_TESTS = "1" + $env:APPVEYOR = "true" + $env:CMAKE_BUILD_PARALLEL_LEVEL = 4 + $env:MINICONDA = "C:\Miniconda3-x64" + $env:PATH = "$env:MINICONDA;$env:MINICONDA\Scripts;$env:PATH" + $env:BUILD_SOURCESDIRECTORY = "$env:APPVEYOR_BUILD_FOLDER" + +build: false + +test_script: + - conda config --remove channels defaults + - conda config --add channels nodefaults + - conda config --add channels conda-forge + - conda config --set channel_priority strict + - conda init powershell + - powershell.exe -ExecutionPolicy Bypass -File %APPVEYOR_BUILD_FOLDER%\.ci\test-windows.ps1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml new file mode 100644 index 000000000000..f6bb354bc5a9 --- /dev/null +++ b/.github/workflows/cuda.yml @@ -0,0 +1,136 @@ +name: CUDA Version + +on: + push: + branches: + - master + pull_request: + branches: + - master + # Run manually by clicking a button in the UI + workflow_dispatch: + inputs: + restart_docker: + description: 'Restart nvidia-docker on the runner before building?' + required: true + type: boolean + default: false + +# automatically cancel in-progress builds if another commit is pushed +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Optionally reinstall + restart docker on the runner before building. + # This is safe as long as only 1 of these jobs runs at a time. + restart-docker: + name: set up docker + runs-on: [self-hosted, linux] + timeout-minutes: 30 + steps: + - name: Setup or update software on host machine + if: ${{ inputs.restart_docker }} + run: | + # install core packages + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg-agent \ + lsb-release \ + software-properties-common + # set up nvidia-docker + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" -y + curl -sL https://nvidia.github.io/nvidia-docker/gpgkey | sudo apt-key add - + curl -sL https://nvidia.github.io/nvidia-docker/$(. /etc/os-release;echo $ID$VERSION_ID)/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + containerd.io \ + docker-ce \ + docker-ce-cli \ + nvidia-docker2 + sudo chmod a+rw /var/run/docker.sock + sudo systemctl restart docker + - name: mark job successful + run: | + exit 0 + test: + name: ${{ matrix.task }} ${{ matrix.cuda_version }} ${{ matrix.method }} (${{ matrix.linux_version }}, ${{ matrix.compiler }}, Python ${{ matrix.python_version }}) + runs-on: [self-hosted, linux] + needs: [restart-docker] + container: + image: nvcr.io/nvidia/cuda:${{ matrix.cuda_version }}-devel-${{ matrix.linux_version }} + env: + CMAKE_BUILD_PARALLEL_LEVEL: 4 + COMPILER: ${{ matrix.compiler }} + CONDA: /tmp/miniforge + DEBIAN_FRONTEND: noninteractive + METHOD: ${{ matrix.method }} + OS_NAME: linux + PYTHON_VERSION: ${{ matrix.python_version }} + TASK: ${{ matrix.task }} + SKBUILD_STRICT_CONFIG: true + options: --gpus all + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + include: + - method: wheel + compiler: gcc + python_version: "3.10" + cuda_version: "12.6.1" + linux_version: "ubuntu22.04" + task: cuda + - method: source + compiler: gcc + python_version: "3.12" + cuda_version: "12.2.2" + linux_version: "ubuntu22.04" + task: cuda + - method: pip + compiler: clang + python_version: "3.11" + cuda_version: "11.8.0" + linux_version: "ubuntu20.04" + task: cuda + steps: + - name: Install latest git and sudo + run: | + apt-get update + apt-get install --no-install-recommends -y \ + ca-certificates \ + software-properties-common + add-apt-repository ppa:git-core/ppa -y + apt-get update + apt-get install --no-install-recommends -y \ + git \ + sudo + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 5 + submodules: true + - name: Setup and run tests + run: | + export BUILD_DIRECTORY="$GITHUB_WORKSPACE" + export PATH=$CONDA/bin:$PATH + + # check GPU usage + nvidia-smi + + # build and test + $GITHUB_WORKSPACE/.ci/setup.sh + $GITHUB_WORKSPACE/.ci/test.sh + all-cuda-jobs-successful: + if: always() + runs-on: ubuntu-latest + needs: [test] + steps: + - name: Note that all tests succeeded + uses: re-actors/alls-green@v1.2.2 + with: + jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/r_package.yml b/.github/workflows/r_package.yml index b0a2c8ca6eff..8811f53b61c0 100644 --- a/.github/workflows/r_package.yml +++ b/.github/workflows/r_package.yml @@ -54,96 +54,96 @@ jobs: r_version: 3.6 build_type: cmake container: 'ubuntu:18.04' - # - os: ubuntu-latest - # task: r-package - # compiler: gcc - # r_version: 4.3 - # build_type: cmake - # container: 'ubuntu:22.04' - # - os: ubuntu-latest - # task: r-package - # compiler: clang - # r_version: 4.3 - # build_type: cmake - # container: 'ubuntu:22.04' - # - os: macos-13 - # task: r-package - # compiler: gcc - # r_version: 4.3 - # build_type: cmake - # container: null - # - os: macos-13 - # task: r-package - # compiler: clang - # r_version: 4.3 - # build_type: cmake - # container: null - # - os: windows-latest - # task: r-package - # compiler: MINGW - # toolchain: MINGW - # r_version: 3.6 - # build_type: cmake - # container: null - # - os: windows-latest - # task: r-package - # compiler: MINGW - # toolchain: MSYS - # r_version: 4.3 - # build_type: cmake - # container: null - # # Visual Studio 2019 - # - os: windows-2019 - # task: r-package - # compiler: MSVC - # toolchain: MSVC - # r_version: 3.6 - # build_type: cmake - # container: null - # # Visual Studio 2022 - # - os: windows-2022 - # task: r-package - # compiler: MSVC - # toolchain: MSVC - # r_version: 4.3 - # build_type: cmake - # container: null - # ############### - # # CRAN builds # - # ############### - # - os: windows-latest - # task: r-package - # compiler: MINGW - # toolchain: MINGW - # r_version: 3.6 - # build_type: cran - # container: null - # - os: windows-latest - # task: r-package - # compiler: MINGW - # toolchain: MSYS - # r_version: 4.3 - # build_type: cran - # container: null - # - os: ubuntu-latest - # task: r-package - # compiler: gcc - # r_version: 4.3 - # build_type: cran - # container: 'ubuntu:22.04' - # - os: macos-13 - # task: r-package - # compiler: clang - # r_version: 4.3 - # build_type: cran - # container: null - # # macos-14 = arm64 - # - os: macos-14 - # task: r-package - # compiler: clang - # r_version: 4.3 - # build_type: cran - # container: null + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 4.3 + build_type: cmake + container: 'ubuntu:22.04' + - os: ubuntu-latest + task: r-package + compiler: clang + r_version: 4.3 + build_type: cmake + container: 'ubuntu:22.04' + - os: macos-13 + task: r-package + compiler: gcc + r_version: 4.3 + build_type: cmake + container: null + - os: macos-13 + task: r-package + compiler: clang + r_version: 4.3 + build_type: cmake + container: null + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cmake + container: null + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.3 + build_type: cmake + container: null + # Visual Studio 2019 + - os: windows-2019 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 3.6 + build_type: cmake + container: null + # Visual Studio 2022 + - os: windows-2022 + task: r-package + compiler: MSVC + toolchain: MSVC + r_version: 4.3 + build_type: cmake + container: null + ############### + # CRAN builds # + ############### + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MINGW + r_version: 3.6 + build_type: cran + container: null + - os: windows-latest + task: r-package + compiler: MINGW + toolchain: MSYS + r_version: 4.3 + build_type: cran + container: null + - os: ubuntu-latest + task: r-package + compiler: gcc + r_version: 4.3 + build_type: cran + container: 'ubuntu:22.04' + - os: macos-13 + task: r-package + compiler: clang + r_version: 4.3 + build_type: cran + container: null + # macos-14 = arm64 + - os: macos-14 + task: r-package + compiler: clang + r_version: 4.3 + build_type: cran + container: null steps: - name: Prevent conversion of line endings on Windows if: startsWith(matrix.os, 'windows') diff --git a/.vsts-ci.yml b/.vsts-ci.yml new file mode 100644 index 000000000000..3a111e10898e --- /dev/null +++ b/.vsts-ci.yml @@ -0,0 +1,481 @@ +trigger: + branches: + include: + - master + tags: + include: + - v* +pr: +- master +variables: + AZURE: 'true' + CMAKE_BUILD_PARALLEL_LEVEL: 4 + PYTHON_VERSION: '3.12' + runCodesignValidationInjection: false + skipComponentGovernanceDetection: true + Codeql.Enabled: false + Codeql.SkipTaskAutoInjection: true + DOTNET_CLI_TELEMETRY_OPTOUT: true + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true + SKBUILD_STRICT_CONFIG: true +resources: + # The __work/ directory, where Azure DevOps writes the source files, needs to be read-write because + # LightGBM's CI jobs write files in the source directory. + # + # For all the containers included here, all other directories that Azure mounts in are mounted as read-only + # to minimize the risk of side effects from one run affecting future runs. + # ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container + containers: + - container: linux-artifact-builder + image: lightgbm/vsts-agent:manylinux_2_28_x86_64 + mountReadOnly: + work: false + externals: true + tools: true + tasks: true + - container: ubuntu-latest + image: 'ubuntu:22.04' + options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" + mountReadOnly: + work: false + externals: true + tools: true + tasks: true + - container: rbase + image: wch1/r-debug + mountReadOnly: + work: false + externals: true + tools: true + tasks: true +jobs: +########################################### +- job: Maintenance +########################################### + pool: mariner-20240410-0 + container: ubuntu-latest + # routine maintenance (like periodically deleting old files), + # to be run on 1 random CI runner in the self-hosted pool each runner + steps: + - script: | + print-diagnostics(){ + echo "---- df -h -m ----" + df -h -m + echo "---- docker system df ----" + /tmp/docker system df + echo "---- docker images ----" + /tmp/docker images + } + # check disk usage + print-diagnostics + # remove old containers, container images, volumes + # ref: https://stackoverflow.com/a/32723127/3986677) + echo "---- running 'docker system prune' ----" + /tmp/docker system prune \ + --all \ + --force \ + --filter until=720h + # check disk usage again + print-diagnostics + displayName: clean +########################################### +- job: Linux +########################################### + variables: + COMPILER: gcc + SETUP_CONDA: 'false' + OS_NAME: 'linux' + PRODUCES_ARTIFACTS: 'true' + pool: mariner-20240410-0 + container: linux-artifact-builder + strategy: + matrix: + regular: + TASK: regular + PYTHON_VERSION: '3.10' + sdist: + TASK: sdist + PYTHON_VERSION: '3.8' + bdist: + TASK: bdist + PYTHON_VERSION: '3.9' + inference: + TASK: if-else + mpi_source: + TASK: mpi + METHOD: source + PYTHON_VERSION: '3.9' + gpu_source: + TASK: gpu + METHOD: source + swig: + TASK: swig + steps: + - script: | + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + echo "##vso[task.prependpath]/usr/lib64/openmpi/bin" + echo "##vso[task.prependpath]$CONDA/bin" + displayName: 'Set variables' + - script: | + git clean -d -f -x + displayName: 'Clean source directory' + - script: | + echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt' + displayName: 'Add commit hash to artifacts archive' + - task: Bash@3 + displayName: Setup + inputs: + filePath: $(Build.SourcesDirectory)/.ci/setup.sh + targetType: filePath + - task: Bash@3 + displayName: Test + inputs: + filePath: $(Build.SourcesDirectory)/.ci/test.sh + targetType: filePath + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container +########################################### +- job: Linux_latest +########################################### + variables: + COMPILER: clang-17 + DEBIAN_FRONTEND: 'noninteractive' + IN_UBUNTU_BASE_CONTAINER: 'true' + OS_NAME: 'linux' + SETUP_CONDA: 'true' + pool: mariner-20240410-0 + container: ubuntu-latest + strategy: + matrix: + regular: + TASK: regular + sdist: + TASK: sdist + bdist: + TASK: bdist + PYTHON_VERSION: '3.10' + inference: + TASK: if-else + mpi_source: + TASK: mpi + METHOD: source + mpi_pip: + TASK: mpi + METHOD: pip + PYTHON_VERSION: '3.11' + mpi_wheel: + TASK: mpi + METHOD: wheel + PYTHON_VERSION: '3.9' + gpu_source: + TASK: gpu + METHOD: source + PYTHON_VERSION: '3.11' + gpu_pip: + TASK: gpu + METHOD: pip + PYTHON_VERSION: '3.10' + gpu_wheel: + TASK: gpu + METHOD: wheel + PYTHON_VERSION: '3.9' + cpp_tests: + TASK: cpp-tests + METHOD: with-sanitizers + steps: + - script: | + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + CONDA=$HOME/miniforge + echo "##vso[task.setvariable variable=CONDA]$CONDA" + echo "##vso[task.prependpath]$CONDA/bin" + displayName: 'Set variables' + # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 + - script: | + /tmp/docker exec -t -u 0 ci-container \ + sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" + displayName: 'Install sudo' + - script: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends git + git clean -d -f -x + displayName: 'Clean source directory' + - task: Bash@3 + displayName: Setup + inputs: + filePath: $(Build.SourcesDirectory)/.ci/setup.sh + targetType: 'filePath' + - task: Bash@3 + displayName: Test + inputs: + filePath: $(Build.SourcesDirectory)/.ci/test.sh + targetType: 'filePath' +########################################### +- job: QEMU_multiarch +########################################### + variables: + BUILD_DIRECTORY: /LightGBM + COMPILER: gcc + PRODUCES_ARTIFACTS: 'true' + pool: + vmImage: ubuntu-22.04 + timeoutInMinutes: 180 + strategy: + matrix: + bdist: + TASK: bdist + ARCH: aarch64 + steps: + - script: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y \ + binfmt-support \ + qemu \ + qemu-user \ + qemu-user-static + displayName: 'Install QEMU' + - script: | + docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + displayName: 'Enable Docker multi-architecture support' + - script: | + git clean -d -f -x + displayName: 'Clean source directory' + - script: | + cat > docker-script.sh < Date: Tue, 12 Nov 2024 22:51:36 -0600 Subject: [PATCH 25/27] whitespace --- python-package/lightgbm/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python-package/lightgbm/__init__.py b/python-package/lightgbm/__init__.py index 650bfd4b25d6..b679b3f665b5 100644 --- a/python-package/lightgbm/__init__.py +++ b/python-package/lightgbm/__init__.py @@ -16,7 +16,6 @@ from .sklearn import LGBMClassifier, LGBMModel, LGBMRanker, LGBMRegressor except ImportError: pass - try: from .plotting import create_tree_digraph, plot_importance, plot_metric, plot_split_value_histogram, plot_tree except ImportError: From dda842da6762dd1fa3c4442728c04926a8997f6d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 12 Nov 2024 23:09:20 -0600 Subject: [PATCH 26/27] ignore lintr warnings --- .ci/install-old-r-packages.R | 68 ++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.ci/install-old-r-packages.R b/.ci/install-old-r-packages.R index 4fcc6693e60c..e402c4d5ca12 100644 --- a/.ci/install-old-r-packages.R +++ b/.ci/install-old-r-packages.R @@ -5,8 +5,8 @@ # .install_packages <- function(packages) { - install.packages( - pkgs = paste( + install.packages( # nolint: undesirable_function + pkgs = paste( # nolint: paste "https://cran.r-project.org/src/contrib/Archive" , packages , sep = "/" @@ -22,7 +22,7 @@ # starting from the root of the dependency graph and working up # there was only a single release of {praise}, so there is no contrib/Archive URL for it -install.packages( +install.packages( # nolint: undesirable_function pkgs = "https://cran.r-project.org/src/contrib/praise_1.0.0.tar.gz" , dependencies = FALSE , lib = Sys.getenv("R_LIBS") @@ -30,50 +30,50 @@ install.packages( ) .install_packages(c( - "brio/brio_1.1.4.tar.gz" - , "cli/cli_3.6.2.tar.gz" - , "crayon/crayon_1.5.2.tar.gz" - , "digest/digest_0.6.36.tar.gz" - , "evaluate/evaluate_0.23.tar.gz" - , "fansi/fansi_1.0.5.tar.gz" - , "fs/fs_1.6.4.tar.gz" - , "glue/glue_1.7.0.tar.gz" - , "jsonlite/jsonlite_1.8.8.tar.gz" - , "lattice/lattice_0.20-41.tar.gz" - , "magrittr/magrittr_2.0.2.tar.gz" - , "pkgconfig/pkgconfig_2.0.2.tar.gz" - , "ps/ps_1.8.0.tar.gz" - , "R6/R6_2.5.0.tar.gz" - , "rlang/rlang_1.1.3.tar.gz" - , "rprojroot/rprojroot_2.0.3.tar.gz" - , "utf8/utf8_1.2.3.tar.gz" - , "withr/withr_3.0.1.tar.gz" + "brio/brio_1.1.4.tar.gz" # nolint: non_portable_path + , "cli/cli_3.6.2.tar.gz" # nolint: non_portable_path + , "crayon/crayon_1.5.2.tar.gz" # nolint: non_portable_path + , "digest/digest_0.6.36.tar.gz" # nolint: non_portable_path + , "evaluate/evaluate_0.23.tar.gz" # nolint: non_portable_path + , "fansi/fansi_1.0.5.tar.gz" # nolint: non_portable_path + , "fs/fs_1.6.4.tar.gz" # nolint: non_portable_path + , "glue/glue_1.7.0.tar.gz" # nolint: non_portable_path + , "jsonlite/jsonlite_1.8.8.tar.gz" # nolint: non_portable_path + , "lattice/lattice_0.20-41.tar.gz" # nolint: non_portable_path + , "magrittr/magrittr_2.0.2.tar.gz" # nolint: non_portable_path + , "pkgconfig/pkgconfig_2.0.2.tar.gz" # nolint: non_portable_path + , "ps/ps_1.8.0.tar.gz" # nolint: non_portable_path + , "R6/R6_2.5.0.tar.gz" # nolint: non_portable_path + , "rlang/rlang_1.1.3.tar.gz" # nolint: non_portable_path + , "rprojroot/rprojroot_2.0.3.tar.gz" # nolint: non_portable_path + , "utf8/utf8_1.2.3.tar.gz" # nolint: non_portable_path + , "withr/withr_3.0.1.tar.gz" # nolint: non_portable_path )) .install_packages(c( - "desc/desc_1.4.2.tar.gz" - , "diffobj/diffobj_0.3.4.tar.gz" - , "lifecycle/lifecycle_1.0.3.tar.gz" - , "processx/processx_3.8.3.tar.gz" + "desc/desc_1.4.2.tar.gz" # nolint: non_portable_path + , "diffobj/diffobj_0.3.4.tar.gz" # nolint: non_portable_path + , "lifecycle/lifecycle_1.0.3.tar.gz" # nolint: non_portable_path + , "processx/processx_3.8.3.tar.gz" # nolint: non_portable_path )) .install_packages(c( - "callr/callr_3.7.5.tar.gz" - , "vctrs/vctrs_0.6.4.tar.gz" + "callr/callr_3.7.5.tar.gz" # nolint: non_portable_path + , "vctrs/vctrs_0.6.4.tar.gz" # nolint: non_portable_path )) .install_packages(c( - "pillar/pillar_1.8.1.tar.gz" - , "tibble/tibble_3.2.0.tar.gz" + "pillar/pillar_1.8.1.tar.gz" # nolint: non_portable_path + , "tibble/tibble_3.2.0.tar.gz" # nolint: non_portable_path )) .install_packages(c( - "pkgbuild/pkgbuild_1.4.4.tar.gz" - , "rematch2/rematch2_2.1.1.tar.gz" - , "waldo/waldo_0.5.3.tar.gz" + "pkgbuild/pkgbuild_1.4.4.tar.gz" # nolint: non_portable_path + , "rematch2/rematch2_2.1.1.tar.gz" # nolint: non_portable_path + , "waldo/waldo_0.5.3.tar.gz" # nolint: non_portable_path )) .install_packages(c( - "pkgload/pkgload_1.3.4.tar.gz" - , "testthat/testthat_3.2.1.tar.gz" + "pkgload/pkgload_1.3.4.tar.gz" # nolint: non_portable_path + , "testthat/testthat_3.2.1.tar.gz" # nolint: non_portable_path )) From 8e36ef00a2a2a984e4a24e44a59ec0d771347f5c Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 14 Nov 2024 17:55:22 -0600 Subject: [PATCH 27/27] Apply suggestions from code review Co-authored-by: Nikita Titov --- tests/python_package_test/test_sklearn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python_package_test/test_sklearn.py b/tests/python_package_test/test_sklearn.py index fcfd23adb7ba..d187e9df5a9f 100644 --- a/tests/python_package_test/test_sklearn.py +++ b/tests/python_package_test/test_sklearn.py @@ -43,7 +43,7 @@ ) SKLEARN_MAJOR, SKLEARN_MINOR, *_ = _sklearn_version.split(".") -SKLEARN_VERSION_GTE_1_6 = int(SKLEARN_MAJOR) >= 1 and int(SKLEARN_MINOR) >= 6 +SKLEARN_VERSION_GTE_1_6 = (int(SKLEARN_MAJOR), int(SKLEARN_MINOR)) >= (1, 6) decreasing_generator = itertools.count(0, -1) estimator_classes = (lgb.LGBMModel, lgb.LGBMClassifier, lgb.LGBMRegressor, lgb.LGBMRanker) @@ -1450,7 +1450,7 @@ def test_getting_feature_names_in_pd_input(estimator_class): # This block defines a patched version of parametrize_with_checks() so lightgbm's tests # can be compatible with scikit-learn <1.6 and >=1.6. # -# This should be removed minimum supported scikit-learn version is at least 1.6. +# This should be removed once minimum supported scikit-learn version is at least 1.6. if SKLEARN_VERSION_GTE_1_6: parametrize_with_checks = sklearn_parametrize_with_checks else: