Skip to content

Commit

Permalink
[ci] check PowerShell scripts with PSScriptAnalyzer (part 3) (#6710)
Browse files Browse the repository at this point in the history 
Co-authored-by: James Lamb <[email protected]>
  • Loading branch information
@StrikerRUS @jameslamb
StrikerRUS and jameslamb authored Nov 3, 2024
1 parent 13f2e92 commit e007191
Show file tree
Hide file tree
Showing 5 changed files with 134 additions and 53 deletions.
12 changes: 9 additions & 3 deletions .ci/install-opencl.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@ $installer = "AMD-APP-SDKInstaller-v3.0.130.135-GA-windows-F-x64.exe"

Write-Output "Downloading OpenCL platform installer"
$ProgressPreference = "SilentlyContinue" # progress bar bug extremely slows down download speed
Invoke-WebRequest -OutFile "$installer" -Uri "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$installer"
$params = @{
OutFile = "$installer"
Uri = "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$installer"
}
Invoke-WebRequest @params

if (Test-Path "$installer") {
Write-Output "Successfully downloaded OpenCL platform installer"
Expand All @@ -17,10 +21,12 @@ if (Test-Path "$installer") {

# Install OpenCL platform from installer executable
Write-Output "Running OpenCL installer"
Invoke-Command -ScriptBlock { Start-Process "$installer" -ArgumentList '/S /V"/quiet /norestart /passive /log opencl.log"' -Wait }
Invoke-Command -ScriptBlock {
Start-Process "$installer" -ArgumentList '/S /V"/quiet /norestart /passive /log opencl.log"' -Wait
}

$property = Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors
if ($property -eq $null) {
if ($null -eq $property) {
Write-Output "Unable to install OpenCL CPU platform"
Write-Output "OpenCL installation log:"
Get-Content "opencl.log"
Expand Down
2 changes: 1 addition & 1 deletion .ci/lint-powershell.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,4 @@ $settings = @{
}
}

Invoke-ScriptAnalyzer -Path "$env:BUILD_DIRECTORY/.ci" -Recurse -EnableExit -Settings $settings
Invoke-ScriptAnalyzer -Path ./ -Recurse -EnableExit -Settings $settings
94 changes: 73 additions & 21 deletions .ci/test-r-package-windows.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,13 @@ $env:CMAKE_VERSION = "3.30.0"
$env:R_LIB_PATH = "$env:BUILD_SOURCESDIRECTORY/RLibrary" -replace '[\\]', '/'
$env:R_LIBS = "$env:R_LIB_PATH"
$env:CMAKE_PATH = "$env:BUILD_SOURCESDIRECTORY/CMake_installation"
$env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:CMAKE_PATH/cmake-$env:CMAKE_VERSION-windows-x86_64/bin;" + $env:PATH
$env:PATH = @(
"$env:RTOOLS_BIN",
"$env:RTOOLS_MINGW_BIN",
"$env:R_LIB_PATH/R/bin/x64",
"$env:CMAKE_PATH/cmake-$env:CMAKE_VERSION-windows-x86_64/bin",
"$env:PATH"
) -join ";"
if ([version]$env:R_VERSION -lt [version]"4.0") {
$env:CRAN_MIRROR = "https://cran-archive.r-project.org"
} else {
Expand All @@ -116,24 +122,50 @@ if (($env:COMPILER -eq "MINGW") -and ($env:R_BUILD_TYPE -eq "cmake")) {
$env:CC = "$env:RTOOLS_MINGW_BIN/gcc.exe"
}

cd $env:BUILD_SOURCESDIRECTORY
Set-Location "$env:BUILD_SOURCESDIRECTORY"
tzutil /s "GMT Standard Time"
[Void][System.IO.Directory]::CreateDirectory($env:R_LIB_PATH)
[Void][System.IO.Directory]::CreateDirectory($env:CMAKE_PATH)
[Void][System.IO.Directory]::CreateDirectory("$env:R_LIB_PATH")
[Void][System.IO.Directory]::CreateDirectory("$env:CMAKE_PATH")

# download R, RTools and CMake
Write-Output "Downloading R, Rtools and CMake"
Get-File-With-Tenacity -url "$env:CRAN_MIRROR/bin/windows/base/old/$env:R_WINDOWS_VERSION/R-$env:R_WINDOWS_VERSION-win.exe" -destfile "R-win.exe"
Get-File-With-Tenacity -url "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$env:RTOOLS_EXE_FILE" -destfile "Rtools.exe"
Get-File-With-Tenacity -url "https://github.com/Kitware/CMake/releases/download/v$env:CMAKE_VERSION/cmake-$env:CMAKE_VERSION-windows-x86_64.zip" -destfile "$env:CMAKE_PATH/cmake.zip"
$params = @{
url = "$env:CRAN_MIRROR/bin/windows/base/old/$env:R_WINDOWS_VERSION/R-$env:R_WINDOWS_VERSION-win.exe"
destfile = "R-win.exe"
}
Get-File-With-Tenacity @params

$params = @{
url = "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$env:RTOOLS_EXE_FILE"
destfile = "Rtools.exe"
}
Get-File-With-Tenacity @params

$params = @{
url = "https://github.com/Kitware/CMake/releases/download/v{0}/cmake-{0}-windows-x86_64.zip" -f $env:CMAKE_VERSION
destfile = "$env:CMAKE_PATH/cmake.zip"
}
Get-File-With-Tenacity @params

# Install R
Write-Output "Installing R"
Start-Process -FilePath R-win.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/R /COMPONENTS=main,x64,i386" ; Assert-Output $?
$params = @{
FilePath = "R-win.exe"
NoNewWindow = $true
Wait = $true
ArgumentList = "/VERYSILENT /DIR=$env:R_LIB_PATH/R /COMPONENTS=main,x64,i386"
}
Start-Process @params ; Assert-Output $?
Write-Output "Done installing R"

Write-Output "Installing Rtools"
Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /SUPPRESSMSGBOXES /DIR=$RTOOLS_INSTALL_PATH" ; Assert-Output $?
$params = @{
FilePath = "Rtools.exe"
NoNewWindow = $true
Wait = $true
ArgumentList = "/VERYSILENT /SUPPRESSMSGBOXES /DIR=$RTOOLS_INSTALL_PATH"
}
Start-Process @params; Assert-Output $?
Write-Output "Done installing Rtools"

Write-Output "Installing CMake"
Expand All @@ -144,8 +176,16 @@ Remove-Item "$env:RTOOLS_MINGW_BIN/cmake.exe" -Force -ErrorAction Ignore
Write-Output "Done installing CMake"

Write-Output "Installing dependencies"
$packages = "c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'processx', 'R6', 'RhpcBLASctl', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')"
Invoke-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Assert-Output $?
$packages = -join @(
"c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'processx', 'R6', 'RhpcBLASctl', 'testthat'), ",
"dependencies = c('Imports', 'Depends', 'LinkingTo')"
)
$params = -join @(
"options(install.packages.check.source = 'no'); ",
"install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', ",
"lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())"
)
Invoke-R-Code-Redirect-Stderr $params ; Assert-Output $?

Write-Output "Building R-package"

Expand All @@ -168,16 +208,21 @@ if ($env:COMPILER -ne "MSVC") {
Write-Output "[ERROR] Unrecognized toolchain: $env:TOOLCHAIN"
Assert-Output $false
}
Invoke-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"; Assert-Output $?
Invoke-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"
Assert-Output $?
} elseif ($env:R_BUILD_TYPE -eq "cran") {
# NOTE: gzip and tar are needed to create a CRAN package on Windows, but
# some flavors of tar.exe can fail in some settings on Windows.
# Putting the msys64 utilities at the beginning of PATH temporarily to be
# sure they're used for that purpose.
if ($env:R_MAJOR_VERSION -eq "3") {
$env:PATH = "C:\msys64\usr\bin;" + $env:PATH
$env:PATH = @("C:\msys64\usr\bin", "$env:PATH") -join ";"
}
Invoke-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Assert-Output $?
$params = -join @(
"result <- processx::run(command = 'sh', args = 'build-cran-package.sh', ",
"echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)"
)
Invoke-R-Code-Redirect-Stderr $params ; Assert-Output $?
Remove-From-Path ".*msys64.*"
# Test CRAN source .tar.gz in a directory that is not this repo or below it.
# When people install.packages('lightgbm'), they won't have the LightGBM
Expand All @@ -186,7 +231,7 @@ if ($env:COMPILER -ne "MSVC") {
$R_CMD_CHECK_DIR = "tmp-r-cmd-check"
New-Item -Path "C:\" -Name $R_CMD_CHECK_DIR -ItemType "directory" > $null
Move-Item -Path "$PKG_FILE_NAME" -Destination "C:\$R_CMD_CHECK_DIR\" > $null
cd "C:\$R_CMD_CHECK_DIR\"
Set-Location "C:\$R_CMD_CHECK_DIR\"
}

Write-Output "Running R CMD check"
Expand All @@ -196,7 +241,11 @@ if ($env:COMPILER -ne "MSVC") {
} else {
$check_args = "c('CMD', 'check', '--no-multiarch', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')"
}
Invoke-R-Code-Redirect-Stderr "result <- processx::run(command = 'R.exe', args = $check_args, echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; $check_succeeded = $?
$params = -join (
"result <- processx::run(command = 'R.exe', args = $check_args, ",
"echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)"
)
Invoke-R-Code-Redirect-Stderr $params ; $check_succeeded = $?

Write-Output "R CMD check build logs:"
$INSTALL_LOG_FILE_NAME = "lightgbm.Rcheck\00install.out"
Expand All @@ -206,10 +255,9 @@ if ($env:COMPILER -ne "MSVC") {

Write-Output "Looking for issues with R CMD check results"
if (Get-Content "$LOG_FILE_NAME" | Select-String -Pattern "NOTE|WARNING|ERROR" -CaseSensitive -Quiet) {
echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check"
Write-Output "NOTEs, WARNINGs, or ERRORs have been found by R CMD check"
Assert-Output $False
}

} else {
$INSTALL_LOG_FILE_NAME = "$env:BUILD_SOURCESDIRECTORY\00install_out.txt"
Invoke-R-Code-Redirect-Stderr "source('build_r.R')" 1> $INSTALL_LOG_FILE_NAME ; $install_succeeded = $?
Expand All @@ -219,7 +267,7 @@ if ($env:COMPILER -ne "MSVC") {
Assert-Output $install_succeeded
# some errors are not raised above, but can be found in the logs
if (Get-Content "$INSTALL_LOG_FILE_NAME" | Select-String -Pattern "ERROR" -CaseSensitive -Quiet) {
echo "ERRORs have been found installing lightgbm"
Write-Output "ERRORs have been found installing lightgbm"
Assert-Output $False
}
}
Expand All @@ -229,7 +277,11 @@ if ($env:TOOLCHAIN -ne "MSVC") {
$checks = Select-String -Path "${LOG_FILE_NAME}" -Pattern "using R version $env:R_WINDOWS_VERSION"
$checks_cnt = $checks.Matches.length
} else {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "R version passed into FindLibR.* $env:R_WINDOWS_VERSION"
$checksParams = @{
Path = "${INSTALL_LOG_FILE_NAME}"
Pattern = "R version passed into FindLibR.* $env:R_WINDOWS_VERSION"
}
$checks = Select-String @checksParams
$checks_cnt = $checks.Matches.length
}
if ($checks_cnt -eq 0) {
Expand Down Expand Up @@ -299,7 +351,7 @@ if ($env:R_BUILD_TYPE -eq "cmake") {

if ($env:COMPILER -eq "MSVC") {
Write-Output "Running tests with testthat.R"
cd R-package/tests
Set-Location R-package/tests
# NOTE: using Rscript.exe intentionally here, instead of Invoke-R-Code-Redirect-Stderr,
# because something about the interaction between Invoke-R-Code-Redirect-Stderr
# and testthat results in failing tests not exiting with a non-0 exit code.
Expand Down
77 changes: 50 additions & 27 deletions .ci/test-windows.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,19 @@ if ($env:TASK -eq "cpp-tests") {
if ($env:TASK -eq "swig") {
$env:JAVA_HOME = $env:JAVA_HOME_8_X64 # there is pre-installed Eclipse Temurin 8 somewhere
$ProgressPreference = "SilentlyContinue" # progress bar bug extremely slows down download speed
Invoke-WebRequest -Uri "https://sourceforge.net/projects/swig/files/latest/download" -OutFile $env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip -UserAgent "curl"
$params = @{
Uri = "https://sourceforge.net/projects/swig/files/latest/download"
OutFile = "$env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip"
UserAgent = "curl"
}
Invoke-WebRequest @params
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory("$env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip", "$env:BUILD_SOURCESDIRECTORY/swig") ; Assert-Output $?
[System.IO.Compression.ZipFile]::ExtractToDirectory(
"$env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip",
"$env:BUILD_SOURCESDIRECTORY/swig"
) ; Assert-Output $?
$SwigFolder = Get-ChildItem -Directory -Name -Path "$env:BUILD_SOURCESDIRECTORY/swig"
$env:PATH = "$env:BUILD_SOURCESDIRECTORY/swig/$SwigFolder;" + $env:PATH
$env:PATH = @("$env:BUILD_SOURCESDIRECTORY/swig/$SwigFolder", "$env:PATH") -join ";"
$BuildLogFileName = "$env:BUILD_SOURCESDIRECTORY\cmake_build.log"
cmake -B build -S . -A x64 -DUSE_SWIG=ON *> "$BuildLogFileName" ; $build_succeeded = $?
Write-Output "CMake build logs:"
Expand Down Expand Up @@ -68,30 +76,30 @@ if ($env:PYTHON_VERSION -eq "3.7") {
$env:CONDA_REQUIREMENT_FILE = "$env:BUILD_SOURCESDIRECTORY/.ci/conda-envs/ci-core.txt"
}

conda create `
-y `
-n $env:CONDA_ENV `
--file $env:CONDA_REQUIREMENT_FILE `
"python=$env:PYTHON_VERSION[build=*cpython]" ; Assert-Output $?
$condaParams = @(
"-y",
"-n", "$env:CONDA_ENV",
"--file", "$env:CONDA_REQUIREMENT_FILE",
"python=$env:PYTHON_VERSION[build=*cpython]"
)
conda create @condaParams ; Assert-Output $?

if ($env:TASK -ne "bdist") {
conda activate $env:CONDA_ENV
}

cd $env:BUILD_SOURCESDIRECTORY
Set-Location "$env:BUILD_SOURCESDIRECTORY"
if ($env:TASK -eq "regular") {
cmake -B build -S . -A x64 ; Assert-Output $?
cmake --build build --target ALL_BUILD --config Release ; Assert-Output $?
sh ./build-python.sh install --precompile ; Assert-Output $?
cp ./Release/lib_lightgbm.dll $env:BUILD_ARTIFACTSTAGINGDIRECTORY
cp ./Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY
}
elseif ($env:TASK -eq "sdist") {
cp ./Release/lib_lightgbm.dll "$env:BUILD_ARTIFACTSTAGINGDIRECTORY"
cp ./Release/lightgbm.exe "$env:BUILD_ARTIFACTSTAGINGDIRECTORY"
} elseif ($env:TASK -eq "sdist") {
sh ./build-python.sh sdist ; Assert-Output $?
sh ./.ci/check-python-dists.sh ./dist ; Assert-Output $?
cd dist; pip install @(Get-ChildItem *.gz) -v ; Assert-Output $?
}
elseif ($env:TASK -eq "bdist") {
Set-Location dist; pip install @(Get-ChildItem *.gz) -v ; Assert-Output $?
} elseif ($env:TASK -eq "bdist") {
# Import the Chocolatey profile module so that the RefreshEnv command
# invoked below properly updates the current PowerShell session environment.
$module = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
Expand All @@ -104,8 +112,8 @@ elseif ($env:TASK -eq "bdist") {
conda activate $env:CONDA_ENV
sh "build-python.sh" bdist_wheel --integrated-opencl ; Assert-Output $?
sh ./.ci/check-python-dists.sh ./dist ; Assert-Output $?
cd dist; pip install @(Get-ChildItem *py3-none-win_amd64.whl) ; Assert-Output $?
cp @(Get-ChildItem *py3-none-win_amd64.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
Set-Location dist; pip install @(Get-ChildItem *py3-none-win_amd64.whl) ; Assert-Output $?
cp @(Get-ChildItem *py3-none-win_amd64.whl) "$env:BUILD_ARTIFACTSTAGINGDIRECTORY"
} elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
if ($env:COMPILER -eq "MINGW") {
sh ./build-python.sh install --mingw ; Assert-Output $?
Expand All @@ -116,9 +124,9 @@ elseif ($env:TASK -eq "bdist") {

if (($env:TASK -eq "sdist") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
# cannot test C API with "sdist" task
$tests = $env:BUILD_SOURCESDIRECTORY + "/tests/python_package_test"
$tests = "$env:BUILD_SOURCESDIRECTORY/tests/python_package_test"
} else {
$tests = $env:BUILD_SOURCESDIRECTORY + "/tests"
$tests = "$env:BUILD_SOURCESDIRECTORY/tests"
}
if ($env:TASK -eq "bdist") {
# Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py
Expand All @@ -128,15 +136,30 @@ if ($env:TASK -eq "bdist") {
pytest $tests ; Assert-Output $?

if (($env:TASK -eq "regular") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide
Set-Location "$env:BUILD_SOURCESDIRECTORY/examples/python-guide"
@("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py"
(Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py" # prevent interactive window mode
# Prevent interactive window mode
(Get-Content "plot_example.py").replace(
'graph.render(view=True)',
'graph.render(view=False)'
) | Set-Content "plot_example.py"
conda install -y -n $env:CONDA_ENV "h5py>=3.10" "ipywidgets>=8.1.2" "notebook>=7.1.2"
# Run all examples
foreach ($file in @(Get-ChildItem *.py)) {
@("import sys, warnings", "warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))") + (Get-Content $file) | Set-Content $file
@(
"import sys, warnings",
-join @(
"warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: ",
"sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))"
)
) + (Get-Content $file) | Set-Content $file
python $file ; Assert-Output $?
} # run all examples
cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide/notebooks
(Get-Content "interactive_plot_example.ipynb").replace('INTERACTIVE = False', 'assert False, \"Interactive mode disabled\"') | Set-Content "interactive_plot_example.ipynb"
jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb ; Assert-Output $? # run all notebooks
}
# Run all notebooks
Set-Location "$env:BUILD_SOURCESDIRECTORY/examples/python-guide/notebooks"
(Get-Content "interactive_plot_example.ipynb").replace(
'INTERACTIVE = False',
'assert False, \"Interactive mode disabled\"'
) | Set-Content "interactive_plot_example.ipynb"
jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb ; Assert-Output $?
}
2 changes: 1 addition & 1 deletion .ci/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ fi
if [[ $TASK == "lint" ]]; then
pwsh -command "Install-Module -Name PSScriptAnalyzer -Scope CurrentUser -SkipPublisherCheck"
echo "Linting PowerShell code"
pwsh -file "./.ci/lint-powershell.ps1" || :
pwsh -file ./.ci/lint-powershell.ps1 || exit 1
conda create -q -y -n "${CONDA_ENV}" \
"${CONDA_PYTHON_REQUIREMENT}" \
'biome>=1.9.3' \
Expand Down

0 comments on commit e007191

Please sign in to comment.