From 596b5efe90b551a8edd680e0567a8ac1a36f82a6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 6 May 2020 08:00:00 +0100 Subject: [PATCH] removing debugging stuff --- .appveyor.yml | 8 +- .ci/test_r_package_windows.ps1 | 25 --- .travis.yml | 20 +-- .vsts-ci.yml | 320 ++++++++++++++++----------------- 4 files changed, 168 insertions(+), 205 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index 76fb965647fd..20807e36e6f5 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -9,10 +9,10 @@ environment: matrix: - COMPILER: MINGW TASK: r-package - # - COMPILER: MSVC - # TASK: python - # - COMPILER: MINGW - # TASK: python + - COMPILER: MSVC + TASK: python + - COMPILER: MINGW + TASK: python clone_depth: 5 diff --git a/.ci/test_r_package_windows.ps1 b/.ci/test_r_package_windows.ps1 index 2efa970490c0..5df2f03577bf 100644 --- a/.ci/test_r_package_windows.ps1 +++ b/.ci/test_r_package_windows.ps1 @@ -12,31 +12,6 @@ function Download-File-With-Retries { } while(!$?); } -# in the virtual machine we get from Azure DevOps, the write throughput of -# D:\ (the default location builds run from) is 10x lower than the write throughput -# of C:\. This matters for very write-heavy steps like generating a build makefile -# with `cmake -G"Visual Studio..."`. -# -# To check the disk performance, run the following -# -# Get-CimInstance Win32_PerfRawData_PerfDisk_LogicalDisk -# Get-CimInstance Win32_PerfRawData_PerfDisk_PhysicalDisk -# -# More details: https://github.com/microsoft/LightGBM/pull/2965#issuecomment-624421092 -# -#if ($env:AZURE -eq "true") { -# Write-Output "Moving files to C:\ and changing BUILD_SOURCESDIRECTORY" -# $env:NEW_BUILD_DIRECTORY = "C:\LightGBM" -# Copy-Item -Path "$env:BUILD_SOURCESDIRECTORY\compute" -Destination "$env:NEW_BUILD_DIRECTORY\compute" -Recurse -# Copy-Item -Path "$env:BUILD_SOURCESDIRECTORY\include" -Destination "$env:NEW_BUILD_DIRECTORY\include" -Recurse -# Copy-Item -Path "$env:BUILD_SOURCESDIRECTORY\R-package" -Destination "$env:NEW_BUILD_DIRECTORY\R-package" -Recurse -# Copy-Item -Path "$env:BUILD_SOURCESDIRECTORY\src" -Destination "$env:NEW_BUILD_DIRECTORY\src" -Recurse -# Copy-Item -Path "$env:BUILD_SOURCESDIRECTORY\build_r.R" -Destination "$env:NEW_BUILD_DIRECTORY\build_r.R" -# Copy-Item -Path "$env:BUILD_SOURCESDIRECTORY\CMakeLists.txt" -Destination "$env:NEW_BUILD_DIRECTORY\CMakeLists.txt" -# $env:BUILD_SOURCESDIRECTORY = "$env:NEW_BUILD_DIRECTORY" -# Write-Output "new BUILD_SOURCESDIRECTORY: $env:BUILD_SOURCESDIRECTORY" -#} - $env:R_WINDOWS_VERSION = "3.6.3" $env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/' $env:PATH = "$env:R_LIB_PATH/Rtools/bin;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:R_LIB_PATH/miktex/texmfs/install/miktex/bin/x64;" + $env:PATH diff --git a/.travis.yml b/.travis.yml index 8e6373cf52a6..3f13504929c7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,16 +14,16 @@ env: global: # default values - PYTHON_VERSION=3.8 matrix: - # - TASK=regular PYTHON_VERSION=3.6 - # - TASK=sdist PYTHON_VERSION=2.7 - # - TASK=bdist - # - TASK=if-else - # - TASK=lint - # - TASK=check-docs - # - TASK=mpi METHOD=source - # - TASK=mpi METHOD=pip PYTHON_VERSION=3.7 - # - TASK=gpu METHOD=source PYTHON_VERSION=3.5 - # - TASK=gpu METHOD=pip PYTHON_VERSION=3.6 + - TASK=regular PYTHON_VERSION=3.6 + - TASK=sdist PYTHON_VERSION=2.7 + - TASK=bdist + - TASK=if-else + - TASK=lint + - TASK=check-docs + - TASK=mpi METHOD=source + - TASK=mpi METHOD=pip PYTHON_VERSION=3.7 + - TASK=gpu METHOD=source PYTHON_VERSION=3.5 + - TASK=gpu METHOD=pip PYTHON_VERSION=3.6 - TASK=r-package matrix: diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 33bc6692f3e5..2de7169398e3 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -13,135 +13,123 @@ resources: - container: ubuntu1404 image: lightgbm/vsts-agent:ubuntu-14.04 jobs: -############################################ -#- job: Linux -############################################ -# variables: -# COMPILER: gcc -# pool: -# vmImage: 'ubuntu-latest' -# container: ubuntu1404 -# strategy: -# maxParallel: 7 -# matrix: -# # regular: -# # TASK: regular -# # sdist: -# # TASK: sdist -# # PYTHON_VERSION: 3.5 -# # bdist: -# # TASK: bdist -# # PYTHON_VERSION: 3.7 -# # inference: -# # TASK: if-else -# # mpi_source: -# # TASK: mpi -# # METHOD: source -# # PYTHON_VERSION: 2.7 -# # gpu_source: -# # TASK: gpu -# # METHOD: source -# # PYTHON_VERSION: 3.6 -# r_package: -# TASK: r-package -# steps: -# - script: | -# echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" -# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" -# echo "##vso[task.setvariable variable=OS_NAME]linux" -# echo "##vso[task.setvariable variable=AZURE]true" -# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" -# echo "##vso[task.prependpath]$CONDA/bin" -# AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK -# echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" -# LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH -# echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" -# echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" -# displayName: 'Set variables' -# - bash: $(Build.SourcesDirectory)/.ci/setup.sh -# displayName: Setup -# - bash: $(Build.SourcesDirectory)/.ci/test.sh -# displayName: Test -# - task: PublishBuildArtifacts@1 -# condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) -# inputs: -# pathtoPublish: '$(Build.ArtifactStagingDirectory)' -# artifactName: PackageAssets -# artifactType: container -############################################ -#- job: MacOS -############################################ -# variables: -# COMPILER: clang -# pool: -# vmImage: 'macOS-10.14' -# strategy: -# maxParallel: 4 -# matrix: -# # regular: -# # TASK: regular -# # PYTHON_VERSION: 3.7 -# # sdist: -# # TASK: sdist -# # PYTHON_VERSION: 3.5 -# # bdist: -# # TASK: bdist -# r_package: -# TASK: r-package -# steps: -# - script: | -# echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" -# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" -# echo "##vso[task.setvariable variable=OS_NAME]macos" -# echo "##vso[task.setvariable variable=AZURE]true" -# echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" -# CONDA=$AGENT_HOMEDIRECTORY/miniconda -# echo "##vso[task.setvariable variable=CONDA]$CONDA" -# echo "##vso[task.prependpath]$CONDA/bin" -# echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64" -# echo "##vso[task.setvariable variable=R_MAC_VERSION]3.6.3" -# displayName: 'Set variables' -# - bash: $(Build.SourcesDirectory)/.ci/setup.sh -# displayName: Setup -# - bash: $(Build.SourcesDirectory)/.ci/test.sh -# displayName: Test -# - task: PublishBuildArtifacts@1 -# condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) -# inputs: -# pathtoPublish: '$(Build.ArtifactStagingDirectory)' -# artifactName: PackageAssets -# artifactType: container ########################################### -- job: Windows +- job: Linux ########################################### + variables: + COMPILER: gcc pool: - vmImage: 'vs2017-win2016' + vmImage: 'ubuntu-latest' + container: ubuntu1404 strategy: - maxParallel: 5 + maxParallel: 7 matrix: + regular: + TASK: regular + sdist: + TASK: sdist + PYTHON_VERSION: 3.5 + bdist: + TASK: bdist + PYTHON_VERSION: 3.7 + inference: + TASK: if-else + mpi_source: + TASK: mpi + METHOD: source + PYTHON_VERSION: 2.7 + gpu_source: + TASK: gpu + METHOD: source + PYTHON_VERSION: 3.6 r_package: TASK: r-package - COMPILER: MSVC - r_package2: - TASK: r-package - COMPILER: MSVC - r_package3: - TASK: r-package - COMPILER: MSVC - r_packag4: + steps: + - script: | + echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + echo "##vso[task.setvariable variable=OS_NAME]linux" + echo "##vso[task.setvariable variable=AZURE]true" + echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" + echo "##vso[task.prependpath]$CONDA/bin" + AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK + echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH" + LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH + echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH" + echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors" + displayName: 'Set variables' + - bash: $(Build.SourcesDirectory)/.ci/setup.sh + displayName: Setup + - bash: $(Build.SourcesDirectory)/.ci/test.sh + displayName: Test + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container +########################################### +- job: MacOS +########################################### + variables: + COMPILER: clang + pool: + vmImage: 'macOS-10.14' + strategy: + maxParallel: 4 + matrix: + regular: + TASK: regular + PYTHON_VERSION: 3.7 + sdist: + TASK: sdist + PYTHON_VERSION: 3.5 + bdist: + TASK: bdist + r_package: TASK: r-package - COMPILER: MSVC - r_package5: + steps: + - script: | + echo "##vso[task.setvariable variable=HOME_DIRECTORY]$AGENT_HOMEDIRECTORY" + echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" + echo "##vso[task.setvariable variable=OS_NAME]macos" + echo "##vso[task.setvariable variable=AZURE]true" + echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)" + CONDA=$AGENT_HOMEDIRECTORY/miniconda + echo "##vso[task.setvariable variable=CONDA]$CONDA" + echo "##vso[task.prependpath]$CONDA/bin" + echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64" + echo "##vso[task.setvariable variable=R_MAC_VERSION]3.6.3" + displayName: 'Set variables' + - bash: $(Build.SourcesDirectory)/.ci/setup.sh + displayName: Setup + - bash: $(Build.SourcesDirectory)/.ci/test.sh + displayName: Test + - task: PublishBuildArtifacts@1 + condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: PackageAssets + artifactType: container +########################################### +- job: Windows +########################################### + pool: + vmImage: 'vs2017-win2016' + strategy: + maxParallel: 4 + matrix: + r_package: TASK: r-package COMPILER: MSVC - # regular: - # TASK: regular - # PYTHON_VERSION: 3.6 - # sdist: - # TASK: sdist - # PYTHON_VERSION: 2.7 - # bdist: - # TASK: bdist + regular: + TASK: regular + PYTHON_VERSION: 3.6 + sdist: + TASK: sdist + PYTHON_VERSION: 2.7 + bdist: + TASK: bdist steps: - powershell: | Write-Host "##vso[task.prependpath]$env:CONDA\Scripts" @@ -158,50 +146,50 @@ jobs: artifactName: PackageAssets artifactType: container -# ########################################### -# - job: Package -# ########################################### -# dependsOn: -# - Linux -# - MacOS -# - Windows -# condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) -# pool: -# vmImage: 'vs2017-win2016' -# steps: -# # Download all agent packages from all previous phases -# - task: DownloadBuildArtifacts@0 -# displayName: Download package assets -# inputs: -# artifactName: PackageAssets -# downloadPath: $(Build.SourcesDirectory)/binaries -# - script: | -# python %BUILD_SOURCESDIRECTORY%/.nuget/create_nuget.py %BUILD_SOURCESDIRECTORY%/binaries/PackageAssets -# displayName: 'Create NuGet configuration files' -# - task: NuGetCommand@2 -# inputs: -# command: pack -# packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec' -# packDestination: '$(Build.ArtifactStagingDirectory)' -# - task: PublishBuildArtifacts@1 -# inputs: -# pathtoPublish: '$(Build.ArtifactStagingDirectory)' -# artifactName: NuGet -# artifactType: container -# - task: GitHubRelease@0 -# displayName: 'Create GitHub Release' -# condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) -# inputs: -# gitHubConnection: guolinke -# repositoryName: '$(Build.Repository.Name)' -# action: 'create' -# target: '$(Build.SourceVersion)' -# tagSource: 'auto' -# title: '$(Build.SourceBranchName)' -# assets: | -# $(Build.SourcesDirectory)/binaries/PackageAssets/* -# $(Build.ArtifactStagingDirectory)/*.nupkg -# assetUploadMode: 'delete' -# isDraft: true -# isPreRelease: false -# addChangeLog: false +########################################### +- job: Package +########################################### + dependsOn: + - Linux + - MacOS + - Windows + condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))) + pool: + vmImage: 'vs2017-win2016' + steps: + # Download all agent packages from all previous phases + - task: DownloadBuildArtifacts@0 + displayName: Download package assets + inputs: + artifactName: PackageAssets + downloadPath: $(Build.SourcesDirectory)/binaries + - script: | + python %BUILD_SOURCESDIRECTORY%/.nuget/create_nuget.py %BUILD_SOURCESDIRECTORY%/binaries/PackageAssets + displayName: 'Create NuGet configuration files' + - task: NuGetCommand@2 + inputs: + command: pack + packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec' + packDestination: '$(Build.ArtifactStagingDirectory)' + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(Build.ArtifactStagingDirectory)' + artifactName: NuGet + artifactType: container + - task: GitHubRelease@0 + displayName: 'Create GitHub Release' + condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) + inputs: + gitHubConnection: guolinke + repositoryName: '$(Build.Repository.Name)' + action: 'create' + target: '$(Build.SourceVersion)' + tagSource: 'auto' + title: '$(Build.SourceBranchName)' + assets: | + $(Build.SourcesDirectory)/binaries/PackageAssets/* + $(Build.ArtifactStagingDirectory)/*.nupkg + assetUploadMode: 'delete' + isDraft: true + isPreRelease: false + addChangeLog: false