From 93c0e9d03358787cf426e352e202e2e64a23d465 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 29 Mar 2020 17:52:46 -0500 Subject: [PATCH] cleaned up debugging code and moved R first in CI --- .appveyor.yml | 6 +- .ci/test_r_package_windows.ps1 | 22 ++- .vsts-ci.yml | 292 +++++++++++++++++---------------- CMakeLists.txt | 4 + build_r.R | 9 +- 5 files changed, 175 insertions(+), 158 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 0fbd8d88f8c1..669b7b516199 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -7,15 +7,15 @@ configuration: # a trick to construct a build matrix with multiple Python versi environment: matrix: + - COMPILER: MINGW + TASK: r-package + R_WINDOWS_VERSION: 3.6.3 - COMPILER: MSVC TASK: appveyor-python CONDA_ENV: test-env - COMPILER: MINGW TASK: appveyor-python CONDA_ENV: test-env - - COMPILER: MINGW - TASK: r-package - R_WINDOWS_VERSION: 3.6.3 clone_depth: 5 diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index f16ba1dc3d23..6a39f0b83542 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -1,5 +1,5 @@ # Download a file and retry upon failure. This looks like -# an infinite looop +# an infinite loop but CI-level timeouts will kill it function Download-File-With-Retries { param( [string]$url, @@ -15,6 +15,7 @@ function Download-File-With-Retries { $env:R_LIB_PATH = "C:/RLibrary" $env:PATH = "$env:R_LIB_PATH/Rtools/bin;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH $env:BINPREF = "C:/mingw-w64/x86_64-8.1.0-posix-seh-rt_v6-rev0/mingw64/bin/" +$env:CRAN_MIRROR = "https://cloud.r-project.org/" cd $env:BUILD_SOURCESDIRECTORY tzutil /s "GMT Standard Time" @@ -60,24 +61,27 @@ initexmf --set-config-value [MPM]AutoInstall=1 conda install -y --no-deps pandoc Add-Content .Renviron "R_LIBS=$env:R_LIB_PATH" -Add-Content .Rprofile "options(repos = 'https://cran.rstudio.com')" -Add-Content .Rprofile "options(pkgType = 'binary')" -Add-Content .Rprofile "options(install.packages.check.source = 'no')" Write-Output "Installing dependencies" -Rscript.exe -e "install.packages(c('data.table', 'jsonlite', 'Matrix', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo'), lib = '$env:R_LIB_PATH')" ; Check-Output $? +$packages = "c('data.table', 'jsonlite', 'Matrix', 'R6', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')" +Rscript --vanilla -e "install.packages($packages, repos = '$env:CRAN_MIRROR', pkgType = 'binary', lib = '$env:R_LIB_PATH', install.packages.check.source = 'no')" ; Check-Output $? Write-Output "Building R package" -Rscript build_r.R ; Check-Output $? +Rscript build_r.R --skip-install ; Check-Output $? $PKG_FILE_NAME = Get-Item *.tar.gz -$PKG_NAME = $PKG_FILE_NAME.BaseName.split("_")[0] -$LOG_FILE_NAME = "$PKG_NAME.Rcheck/00check.log" +$LOG_FILE_NAME = "lightgbm.Rcheck/00check.log" Write-Output "Running R CMD check" +$check_flags = "--as-cran --no-multiarch" +if ($env:AZURE -eq "true") { + $check_flags = "--no-multiarch --no-manual --ignore-vignettes" +} +Write-Output "using check flags '$check_flags'" $env:_R_CHECK_FORCE_SUGGESTS_=0 -R.exe CMD check "${PKG_FILE_NAME}" --as-cran --no-multiarch; Check-Output $? +R.exe CMD check "${PKG_FILE_NAME}" $check_flags ; Check-Output $? +Write-Output "Looking for issues with R CMD check results" if (Get-Content "$LOG_FILE_NAME" | Select-String -Pattern "WARNING" -Quiet) { echo "WARNINGS have been found by R CMD check!" Check-Output $False diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 6996e2dd19db..85311f6eac03 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -16,101 +16,101 @@ jobs: ########################################### - job: Linux ########################################### - variables: - COMPILER: gcc - pool: - vmImage: 'ubuntu-latest' - container: ubuntu1404 - strategy: - maxParallel: 6 - matrix: - regular: - TASK: regular - sdist: - TASK: sdist - PYTHON_VERSION: 3.5 - bdist: - TASK: bdist - PYTHON_VERSION: 3.6 - inference: - TASK: if-else - mpi_source: - TASK: mpi - METHOD: source - PYTHON_VERSION: 2.7 - gpu_source: - TASK: gpu - METHOD: source - PYTHON_VERSION: 3.6 - r_package: - TASK: r-package - steps: - - script: | - echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.setvariable variable=OS_NAME]linux" - echo "##vso[task.setvariable variable=AZURE]true" - echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - echo "##vso[task.prependpath]$CONDA/bin" - AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK - echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" - LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH - echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" - echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" - displayName: 'Set variables' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container + variables: + COMPILER: gcc + pool: + vmImage: 'ubuntu-latest' + container: ubuntu1404 + strategy: + maxParallel: 6 + matrix: + regular: + TASK: regular + sdist: + TASK: sdist + PYTHON_VERSION: 3.5 + bdist: + TASK: bdist + PYTHON_VERSION: 3.6 + inference: + TASK: if-else + mpi_source: + TASK: mpi + METHOD: source + PYTHON_VERSION: 2.7 + gpu_source: + TASK: gpu + METHOD: source + PYTHON_VERSION: 3.6 + r_package: + TASK: r-package + steps: + - script: | + echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + echo "##vso[task.setvariable variable=OS_NAME]linux" + echo "##vso[task.setvariable variable=AZURE]true" + echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" + echo "##vso[task.prependpath]$CONDA/bin" + AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK + echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" + LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH + echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" + echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" + displayName: 'Set variables' + - bash: $(Build.SourcesDirectory)/.ci/setup.sh + displayName: Setup + - bash: $(Build.SourcesDirectory)/.ci/test.sh + displayName: Test + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container ########################################### - job: MacOS ########################################### - variables: - COMPILER: clang - pool: - vmImage: 'macOS-10.14' - strategy: - maxParallel: 3 - matrix: - regular: - TASK: regular - PYTHON_VERSION: 3.6 - sdist: - TASK: sdist - PYTHON_VERSION: 3.5 - bdist: - TASK: bdist - r_package: - TASK: r-package - steps: - - script: | - echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" - echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" - echo "##vso[task.setvariable variable=OS_NAME]macos" - echo "##vso[task.setvariable variable=AZURE]true" - echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" - CONDA=$AGENT_HOMEDIRECTORY/miniconda - echo "##vso[task.setvariable variable=CONDA]$CONDA" - echo "##vso[task.prependpath]$CONDA/bin" - echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64" - echo "##vso[task.setvariable variable=R_MAC_VERSION]3.6.3" - displayName: 'Set variables' - - bash: $(Build.SourcesDirectory)/.ci/setup.sh - displayName: Setup - - bash: $(Build.SourcesDirectory)/.ci/test.sh - displayName: Test - - task: PublishBuildArtifacts@1 - condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: PackageAssets - artifactType: container + variables: + COMPILER: clang + pool: + vmImage: 'macOS-10.14' + strategy: + maxParallel: 3 + matrix: + regular: + TASK: regular + PYTHON_VERSION: 3.6 + sdist: + TASK: sdist + PYTHON_VERSION: 3.5 + bdist: + TASK: bdist + r_package: + TASK: r-package + steps: + - script: | + echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + echo "##vso[task.setvariable variable=OS_NAME]macos" + echo "##vso[task.setvariable variable=AZURE]true" + echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" + CONDA=$AGENT_HOMEDIRECTORY/miniconda + echo "##vso[task.setvariable variable=CONDA]$CONDA" + echo "##vso[task.prependpath]$CONDA/bin" + echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64" + echo "##vso[task.setvariable variable=R_MAC_VERSION]3.6.3" + displayName: 'Set variables' + - bash: $(Build.SourcesDirectory)/.ci/setup.sh + displayName: Setup + - bash: $(Build.SourcesDirectory)/.ci/test.sh + displayName: Test + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container ########################################### - job: Windows ########################################### @@ -119,21 +119,23 @@ jobs: strategy: maxParallel: 3 matrix: + rpkg: + TASK: r-package + R_WINDOWS_VERSION: 3.6.3 regular: TASK: regular PYTHON_VERSION: 3.7 sdist: - TASK: sdist - PYTHON_VERSION: 2.7 + TASK: sdist + PYTHON_VERSION: 2.7 bdist: - TASK: bdist - PYTHON_VERSION: 3.5 - rpkg: - TASK: r-package - R_WINDOWS_VERSION: 3.6.3 + TASK: bdist + PYTHON_VERSION: 3.5 steps: - - powershell: Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" - displayName: Enable conda + - powershell: | + Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" + Write-Host "##vso[task.setvariable variable=AZURE]true" + displayName: 'Set Variables' - script: | cmd /c "conda init powershell" cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1" @@ -148,47 +150,47 @@ jobs: ########################################### - job: Package ########################################### - dependsOn: - - Linux - - MacOS - - Windows - condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) - pool: - vmImage: 'vs2017-win2016' - steps: - # Download all agent packages from all previous phases - - task: DownloadBuildArtifacts@0 - displayName: Download package assets - inputs: - artifactName: PackageAssets - downloadPath: $(Build.SourcesDirectory)/binaries - - script: | - python %BUILD_SOURCESDIRECTORY%/.nuget/create_nuget.py %BUILD_SOURCESDIRECTORY%/binaries/PackageAssets - displayName: 'Create NuGet configuration files' - - task: NuGetCommand@2 - inputs: - command: pack - packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec' - packDestination: '$(Build.ArtifactStagingDirectory)' - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: '$(Build.ArtifactStagingDirectory)' - artifactName: NuGet - artifactType: container - - task: GitHubRelease@0 - displayName: 'Create GitHub Release' - condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) - inputs: - gitHubConnection: guolinke - repositoryName: '$(Build.Repository.Name)' - action: 'create' - target: '$(Build.SourceVersion)' - tagSource: 'auto' - title: '$(Build.SourceBranchName)' - assets: | - $(Build.SourcesDirectory)/binaries/PackageAssets/* - $(Build.ArtifactStagingDirectory)/*.nupkg - assetUploadMode: 'delete' - isDraft: true - isPreRelease: false - addChangeLog: false + dependsOn: + - Linux + - MacOS + - Windows + condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + pool: + vmImage: 'vs2017-win2016' + steps: + # Download all agent packages from all previous phases + - task: DownloadBuildArtifacts@0 + displayName: Download package assets + inputs: + artifactName: PackageAssets + downloadPath: $(Build.SourcesDirectory)/binaries + - script: | + python %BUILD_SOURCESDIRECTORY%/.nuget/create_nuget.py %BUILD_SOURCESDIRECTORY%/binaries/PackageAssets + displayName: 'Create NuGet configuration files' + - task: NuGetCommand@2 + inputs: + command: pack + packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec' + packDestination: '$(Build.ArtifactStagingDirectory)' + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: NuGet + artifactType: container + - task: GitHubRelease@0 + displayName: 'Create GitHub Release' + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) + inputs: + gitHubConnection: guolinke + repositoryName: '$(Build.Repository.Name)' + action: 'create' + target: '$(Build.SourceVersion)' + tagSource: 'auto' + title: '$(Build.SourceBranchName)' + assets: | + $(Build.SourcesDirectory)/binaries/PackageAssets/* + $(Build.ArtifactStagingDirectory)/*.nupkg + assetUploadMode: 'delete' + isDraft: true + isPreRelease: false + addChangeLog: false diff --git a/CMakeLists.txt b/CMakeLists.txt index a651cfbdf62c..e8e170c9d2b9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -310,6 +310,10 @@ endif() if(BUILD_FOR_R) if(MSVC) # https://docs.microsoft.com/en-us/cpp/build/reference/link-input-files?redirectedfrom=MSDN&view=vs-2019 + set_property( + TARGET _lightgbm + PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreadedDLL" + ) TARGET_LINK_LIBRARIES(_lightgbm ${CMAKE_CURRENT_BINARY_DIR}/R.lib) else() TARGET_LINK_LIBRARIES(_lightgbm ${LIBR_CORE_LIBRARY}) diff --git a/build_r.R b/build_r.R index 88133c248071..1ca97d70d397 100644 --- a/build_r.R +++ b/build_r.R @@ -5,6 +5,9 @@ # Sys.setenv("CXX" = "/usr/local/bin/g++-8") # Sys.setenv("CC" = "/usr/local/bin/gcc-8") +args <- commandArgs(trailingOnly=TRUE) +INSTALL_AFTER_BUILD <- !("--skip-install" %in% args) + # R returns FALSE (not a non-zero exit code) if a file copy operation # breaks. Let's fix that .handle_result <- function(res) { @@ -86,4 +89,8 @@ version <- gsub( tarball <- file.path(getwd(), sprintf("lightgbm_%s.tar.gz", version)) cmd <- sprintf("R CMD INSTALL %s --no-multiarch --with-keep.source", tarball) -.run_shell_command(cmd) +if (INSTALL_AFTER_BUILD){ + .run_shell_command(cmd) +} else { + print(sprintf("Skipping installation. Install the package with command '%s'", cmd)) +}