diff --git a/.appveyor.yml b/.appveyor.yml index f27fa78fb89c..f3404faa68b9 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -13,11 +13,6 @@ branches: environment: matrix: - - COMPILER: MINGW - TASK: r-package - - COMPILER: MSVC - TASK: r-package - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019 - COMPILER: MSVC TASK: python - COMPILER: MINGW diff --git a/.ci/download-miktex.R b/.ci/download-miktex.R new file mode 100644 index 000000000000..f0d4b98cac5a --- /dev/null +++ b/.ci/download-miktex.R @@ -0,0 +1,25 @@ +# mirrors that host miktexsetup.zip do so only with explicitly-named +# files like miktexsetup-2.4.5.zip, so hard-coding a link to an archive as a +# way to peg to one mirror does not work +# +# this script will find the specific version of miktexsetup.zip at a given +# mirror +library(httr) +args <- commandArgs(trailingOnly = TRUE) +DESTFILE <- args[[1L]] +MIRROR <- "https://ctan.math.illinois.edu/systems/win32/miktex/setup/windows-x64/" +mirror_contents <- httr::content( + httr::RETRY("GET", MIRROR) + , as = "text" +) +content_lines <- strsplit(mirror_contents, "\n")[[1L]] +content_lines <- content_lines[grepl("miktexsetup-.*", content_lines)] +zip_loc <- regexpr(">miktexsetup-[0-9]+.*x64\\.zip", content_lines) +zip_name <- gsub(">", "", regmatches(content_lines, zip_loc)) +full_zip_url <- file.path(MIRROR, zip_name) +print(sprintf("downloading %s", full_zip_url)) +download.file( + url = full_zip_url + , destfile = DESTFILE +) +print(sprintf("MiKTeX setup downloaded to %s", DESTFILE)) diff --git a/.ci/setup.sh b/.ci/setup.sh index 0c4101189346..73adee173ca8 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -36,12 +36,12 @@ else # Linux mv $AMDAPPSDK_PATH/lib/x86_64/sdk/* $AMDAPPSDK_PATH/lib/x86_64/ echo libamdocl64.so > $OPENCL_VENDOR_PATH/amdocl64.icd fi - if [[ $TRAVIS == "true" ]]; then + if [[ $TRAVIS == "true" ]] || [[ $GITHUB_ACTIONS == "true" ]]; then wget -q -O conda.sh https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh fi fi -if [[ $TRAVIS == "true" ]] || [[ $OS_NAME == "macos" ]]; then +if [[ $TRAVIS == "true" ]] || [[ $GITHUB_ACTIONS == "true" ]] || [[ $OS_NAME == "macos" ]]; then sh conda.sh -b -p $CONDA fi conda config --set always_yes yes --set changeps1 no diff --git a/.ci/test.sh b/.ci/test.sh index 3cb25ee10d14..4a1144733df8 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -13,7 +13,7 @@ source activate $CONDA_ENV cd $BUILD_DIRECTORY -if [[ $TRAVIS == "true" ]] && [[ $TASK == "check-docs" ]]; then +if [[ $TASK == "check-docs" ]]; then cd $BUILD_DIRECTORY/docs conda install -q -y -n $CONDA_ENV -c conda-forge doxygen pip install --user -r requirements.txt rstcheck @@ -40,7 +40,7 @@ if [[ $TRAVIS == "true" ]] && [[ $TASK == "check-docs" ]]; then exit 0 fi -if [[ $TRAVIS == "true" ]] && [[ $TASK == "lint" ]]; then +if [[ $TASK == "lint" ]]; then conda install -q -y -n $CONDA_ENV \ pycodestyle \ pydocstyle \ diff --git a/.ci/test_r_package.sh b/.ci/test_r_package.sh index fcd515d23c18..d9377857a518 100755 --- a/.ci/test_r_package.sh +++ b/.ci/test_r_package.sh @@ -13,7 +13,7 @@ export PATH="$R_LIB_PATH/R/bin:$PATH" # # This only needs to get run on Travis because R environment for Linux # used by Azure pipelines is set up in https://github.com/guolinke/lightgbm-ci-docker -if [[ $TRAVIS == "true" ]] && [[ $OS_NAME == "linux" ]]; then +if [[ $AZURE != "true" ]] && [[ $OS_NAME == "linux" ]]; then sudo apt-key adv \ --keyserver keyserver.ubuntu.com \ --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 966cc3dc0e91..73712d5d8c9a 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -48,11 +48,15 @@ Write-Output "Installing Rtools" Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/Rtools" ; Check-Output $? Write-Output "Done installing Rtools" +Write-Output "Installing dependencies" +$packages = "c('data.table', 'jsonlite', 'httr', 'Matrix', 'processx', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" +Rscript --vanilla -e "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH')" ; Check-Output $? + # MiKTeX and pandoc can be skipped on non-MINGW builds, since we don't # build the package documentation for those if ($env:COMPILER -eq "MINGW") { Write-Output "Downloading MiKTeX" - Download-File-With-Retries -url "https://miktex.org/download/win/miktexsetup-x64.zip" -destfile "miktexsetup-x64.zip" + Rscript $env:BUILD_SOURCESDIRECTORY\.ci\download-miktex.R "miktexsetup-x64.zip" Add-Type -AssemblyName System.IO.Compression.FileSystem [System.IO.Compression.ZipFile]::ExtractToDirectory("miktexsetup-x64.zip", "miktex") Write-Output "Setting up MiKTeX" @@ -65,10 +69,6 @@ if ($env:COMPILER -eq "MINGW") { conda install -q -y --no-deps pandoc } -Write-Output "Installing dependencies" -$packages = "c('data.table', 'jsonlite', 'Matrix', 'processx', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" -Rscript --vanilla -e "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH')" ; Check-Output $? - Write-Output "Building R package" # R CMD check is not used for MSVC builds @@ -94,8 +94,8 @@ if ($env:COMPILER -ne "MSVC") { Check-Output $False } - $note_str = Get-Content "${LOG_FILE_NAME}" | Select-String -Pattern ' NOTE' | Out-String ; Check-Output $? - $relevant_line = $note_str -match '.*Status: (\d+) NOTE.*' + $note_str = Get-Content -Path "${LOG_FILE_NAME}" | Select-String -Pattern '.*Status.* NOTE' | Out-String ; Check-Output $? + $relevant_line = $note_str -match '(\d+) NOTE' $NUM_CHECK_NOTES = $matches[1] $ALLOWED_CHECK_NOTES = 3 if ([int]$NUM_CHECK_NOTES -gt $ALLOWED_CHECK_NOTES) { diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000000..f653753a7e54 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,87 @@ +name: GitHub Actions + +on: [push] + +jobs: + test: + name: ${{ matrix.task }} (${{ matrix.os }}, ${{ matrix.subtitle }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + task: r-package + compiler: gcc + subtitle: gcc + - os: ubuntu-latest + task: r-package + compiler: clang + subtitle: clang + - os: macOS-latest + task: r-package + compiler: gcc + subtitle: gcc + - os: macOS-latest + task: r-package + compiler: clang + subtitle: clang + - os: windows-latest + task: r-package + compiler: MINGW + subtitle: MINGW + - os: windows-latest + task: r-package + compiler: MSVC + subtitle: MSVC + - os: ubuntu-latest + task: lint + steps: + - name: Checkout repository + uses: actions/checkout@v1 + with: + fetch-depth: 5 + submodules: true + - name: Setup and run tests on Linux and macOS + if: matrix.os != 'windows-latest' + shell: bash + env: + COMPILER: "${{ matrix.compiler }}" + CONDA_ENV: "test-env" + GITHUB_ACTIONS: "true" + TASK: "${{ matrix.task }}" + run: | + if [[ "${{ matrix.os }}" == "macOS-latest" ]]; then + export OS_NAME="macos" + export R_MAC_VERSION=3.6.3 + elif [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then + export OS_NAME="linux" + export R_TRAVIS_LINUX_VERSION=3.6.3-1bionic; + fi + if [ -z ${{ matrix.compiler }} ]; then + export COMPILER=${{ matrix.compiler }} + fi + export BUILD_DIRECTORY="$GITHUB_WORKSPACE" + export CONDA="$HOME/miniconda" + export PATH="$CONDA/bin:${HOME}/.local/bin:$PATH" + export LGB_VER=$(head -n 1 VERSION.txt) + $GITHUB_WORKSPACE/.ci/setup.sh + $GITHUB_WORKSPACE/.ci/test.sh + - name: Use conda on Windows + if: matrix.os == 'windows-latest' + uses: goanpeca/setup-miniconda@v1 + with: + auto-update-conda: false + - name: Setup and run tests on Windows + if: matrix.os == 'windows-latest' + # have to use this explicit command to override the GitHub Actions + # default, where any command writing to stderr causes build failures + shell: pwsh -NonInteractive -ExecutionPolicy Bypass -Command "& '{0}'" + env: + COMPILER: "${{ matrix.compiler }}" + GITHUB_ACTIONS: "true" + TASK: "${{ matrix.task }}" + run: | + $env:BUILD_SOURCESDIRECTORY = $env:GITHUB_WORKSPACE + conda init powershell + & "$env:GITHUB_WORKSPACE/.ci/test_windows.ps1" diff --git a/.travis.yml b/.travis.yml index 882d0f85f7ed..052a04d3e5b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,13 +18,11 @@ env: - TASK=sdist PYTHON_VERSION=2.7 - TASK=bdist - TASK=if-else - - TASK=lint - TASK=check-docs - TASK=mpi METHOD=source - TASK=mpi METHOD=pip PYTHON_VERSION=3.7 - TASK=gpu METHOD=source PYTHON_VERSION=3.5 - TASK=gpu METHOD=pip PYTHON_VERSION=3.6 - - TASK=r-package matrix: exclude: @@ -32,8 +30,6 @@ matrix: env: TASK=gpu METHOD=source PYTHON_VERSION=3.5 - os: osx env: TASK=gpu METHOD=pip PYTHON_VERSION=3.6 - - os: osx - env: TASK=lint - os: osx env: TASK=check-docs @@ -45,11 +41,9 @@ before_install: - if [[ $TRAVIS_OS_NAME == "osx" ]]; then export OS_NAME="macos"; export COMPILER="gcc"; - export R_MAC_VERSION=3.6.3; else export OS_NAME="linux"; export COMPILER="clang"; - export R_TRAVIS_LINUX_VERSION=3.6.3-1bionic; fi - export CONDA="$HOME/miniconda" - export PATH="$CONDA/bin:$PATH" diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 359ecc4389b9..0d061dec7743 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -22,7 +22,7 @@ jobs: vmImage: 'ubuntu-latest' container: ubuntu1404 strategy: - maxParallel: 7 + maxParallel: 6 matrix: regular: TASK: regular @@ -41,8 +41,6 @@ jobs: TASK: gpu METHOD: source PYTHON_VERSION: 3.6 - r_package: - TASK: r-package steps: - script: | echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" @@ -75,7 +73,7 @@ jobs: pool: vmImage: 'macOS-10.14' strategy: - maxParallel: 4 + maxParallel: 3 matrix: regular: TASK: regular @@ -85,8 +83,6 @@ jobs: PYTHON_VERSION: 3.5 bdist: TASK: bdist - r_package: - TASK: r-package steps: - script: | echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" @@ -98,7 +94,6 @@ jobs: echo "##vso[task.setvariable variable=CONDA]$CONDA" echo "##vso[task.prependpath]$CONDA/bin" echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64" - echo "##vso[task.setvariable variable=R_MAC_VERSION]3.6.3" displayName: 'Set variables' - bash: $(Build.SourcesDirectory)/.ci/setup.sh displayName: Setup