Skip to content

Commit

Permalink
[ci] check PowerShell scripts with PSScriptAnalyzer (part 2) (#6709)
Browse files Browse the repository at this point in the history
  • Loading branch information
@StrikerRUS
StrikerRUS authored Nov 1, 2024
1 parent 8d5dca2 commit 92aa07b
Show file tree
Hide file tree
Showing 4 changed files with 214 additions and 214 deletions.
28 changes: 14 additions & 14 deletions .ci/install-opencl.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ $ProgressPreference = "SilentlyContinue" # progress bar bug extremely slows dow
Invoke-WebRequest -OutFile "$installer" -Uri "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$installer"

if (Test-Path "$installer") {
Write-Output "Successfully downloaded OpenCL platform installer"
Write-Output "Successfully downloaded OpenCL platform installer"
} else {
Write-Output "Unable to download OpenCL platform installer"
Write-Output "Setting EXIT"
$host.SetShouldExit(-1)
exit 1
Write-Output "Unable to download OpenCL platform installer"
Write-Output "Setting EXIT"
$host.SetShouldExit(-1)
exit 1
}

# Install OpenCL platform from installer executable
Expand All @@ -21,14 +21,14 @@ Invoke-Command -ScriptBlock { Start-Process "$installer" -ArgumentList '/S /V"/q

$property = Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors
if ($property -eq $null) {
Write-Output "Unable to install OpenCL CPU platform"
Write-Output "OpenCL installation log:"
Get-Content "opencl.log"
Write-Output "Setting EXIT"
$host.SetShouldExit(-1)
exit 1
Write-Output "Unable to install OpenCL CPU platform"
Write-Output "OpenCL installation log:"
Get-Content "opencl.log"
Write-Output "Setting EXIT"
$host.SetShouldExit(-1)
exit 1
} else {
Write-Output "Successfully installed OpenCL CPU platform"
Write-Output "Current OpenCL drivers:"
Write-Output $property
Write-Output "Successfully installed OpenCL CPU platform"
Write-Output "Current OpenCL drivers:"
Write-Output $property
}
242 changes: 121 additions & 121 deletions .ci/test-r-package-windows.ps1
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,22 +75,22 @@ Remove-Item C:\rtools43 -Force -Recurse -ErrorAction Ignore
# * some paths and file names are different on R4.0
$env:R_MAJOR_VERSION = $env:R_VERSION.split('.')[0]
if ($env:R_MAJOR_VERSION -eq "3") {
# Rtools 3.x has to be installed at C:\Rtools\
# * https://stackoverflow.com/a/46619260/3986677
$RTOOLS_INSTALL_PATH = "C:\Rtools"
$env:RTOOLS_BIN = "$RTOOLS_INSTALL_PATH\bin"
$env:RTOOLS_MINGW_BIN = "$RTOOLS_INSTALL_PATH\mingw_64\bin"
$env:RTOOLS_EXE_FILE = "rtools35-x86_64.exe"
$env:R_WINDOWS_VERSION = "3.6.3"
# Rtools 3.x has to be installed at C:\Rtools\
# * https://stackoverflow.com/a/46619260/3986677
$RTOOLS_INSTALL_PATH = "C:\Rtools"
$env:RTOOLS_BIN = "$RTOOLS_INSTALL_PATH\bin"
$env:RTOOLS_MINGW_BIN = "$RTOOLS_INSTALL_PATH\mingw_64\bin"
$env:RTOOLS_EXE_FILE = "rtools35-x86_64.exe"
$env:R_WINDOWS_VERSION = "3.6.3"
} elseif ($env:R_MAJOR_VERSION -eq "4") {
$RTOOLS_INSTALL_PATH = "C:\rtools43"
$env:RTOOLS_BIN = "$RTOOLS_INSTALL_PATH\usr\bin"
$env:RTOOLS_MINGW_BIN = "$RTOOLS_INSTALL_PATH\x86_64-w64-mingw32.static.posix\bin"
$env:RTOOLS_EXE_FILE = "rtools43-5550-5548.exe"
$env:R_WINDOWS_VERSION = "4.3.1"
$RTOOLS_INSTALL_PATH = "C:\rtools43"
$env:RTOOLS_BIN = "$RTOOLS_INSTALL_PATH\usr\bin"
$env:RTOOLS_MINGW_BIN = "$RTOOLS_INSTALL_PATH\x86_64-w64-mingw32.static.posix\bin"
$env:RTOOLS_EXE_FILE = "rtools43-5550-5548.exe"
$env:R_WINDOWS_VERSION = "4.3.1"
} else {
Write-Output "[ERROR] Unrecognized R version: $env:R_VERSION"
Assert-Output $false
Write-Output "[ERROR] Unrecognized R version: $env:R_VERSION"
Assert-Output $false
}
$env:CMAKE_VERSION = "3.30.0"

Expand All @@ -99,9 +99,9 @@ $env:R_LIBS = "$env:R_LIB_PATH"
$env:CMAKE_PATH = "$env:BUILD_SOURCESDIRECTORY/CMake_installation"
$env:PATH = "$env:RTOOLS_BIN;" + "$env:RTOOLS_MINGW_BIN;" + "$env:R_LIB_PATH/R/bin/x64;" + "$env:CMAKE_PATH/cmake-$env:CMAKE_VERSION-windows-x86_64/bin;" + $env:PATH
if ([version]$env:R_VERSION -lt [version]"4.0") {
$env:CRAN_MIRROR = "https://cran-archive.r-project.org"
$env:CRAN_MIRROR = "https://cran-archive.r-project.org"
} else {
$env:CRAN_MIRROR = "https://cran.rstudio.com"
$env:CRAN_MIRROR = "https://cran.rstudio.com"
}
$env:MIKTEX_EXCEPTION_PATH = "$env:TEMP\miktex"

Expand All @@ -112,8 +112,8 @@ if ($env:R_BUILD_TYPE -ne "cran") {
}

if (($env:COMPILER -eq "MINGW") -and ($env:R_BUILD_TYPE -eq "cmake")) {
$env:CXX = "$env:RTOOLS_MINGW_BIN/g++.exe"
$env:CC = "$env:RTOOLS_MINGW_BIN/gcc.exe"
$env:CXX = "$env:RTOOLS_MINGW_BIN/g++.exe"
$env:CC = "$env:RTOOLS_MINGW_BIN/gcc.exe"
}

cd $env:BUILD_SOURCESDIRECTORY
Expand Down Expand Up @@ -152,158 +152,158 @@ Write-Output "Building R-package"
# R CMD check is not used for MSVC builds
if ($env:COMPILER -ne "MSVC") {

$PKG_FILE_NAME = "lightgbm_$env:LGB_VER.tar.gz"
$LOG_FILE_NAME = "lightgbm.Rcheck/00check.log"
$PKG_FILE_NAME = "lightgbm_$env:LGB_VER.tar.gz"
$LOG_FILE_NAME = "lightgbm.Rcheck/00check.log"

if ($env:R_BUILD_TYPE -eq "cmake") {
if ($env:TOOLCHAIN -eq "MINGW") {
Write-Output "Telling R to use MinGW"
$env:BUILD_R_FLAGS = "c('--skip-install', '--use-mingw', '-j4')"
} elseif ($env:TOOLCHAIN -eq "MSYS") {
Write-Output "Telling R to use MSYS"
$env:BUILD_R_FLAGS = "c('--skip-install', '--use-msys2', '-j4')"
} elseif ($env:TOOLCHAIN -eq "MSVC") {
$env:BUILD_R_FLAGS = "'--skip-install'"
} else {
Write-Output "[ERROR] Unrecognized toolchain: $env:TOOLCHAIN"
Assert-Output $false
}
Invoke-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"; Assert-Output $?
} elseif ($env:R_BUILD_TYPE -eq "cran") {
# NOTE: gzip and tar are needed to create a CRAN package on Windows, but
# some flavors of tar.exe can fail in some settings on Windows.
# Putting the msys64 utilities at the beginning of PATH temporarily to be
# sure they're used for that purpose.
if ($env:R_MAJOR_VERSION -eq "3") {
$env:PATH = "C:\msys64\usr\bin;" + $env:PATH
if ($env:R_BUILD_TYPE -eq "cmake") {
if ($env:TOOLCHAIN -eq "MINGW") {
Write-Output "Telling R to use MinGW"
$env:BUILD_R_FLAGS = "c('--skip-install', '--use-mingw', '-j4')"
} elseif ($env:TOOLCHAIN -eq "MSYS") {
Write-Output "Telling R to use MSYS"
$env:BUILD_R_FLAGS = "c('--skip-install', '--use-msys2', '-j4')"
} elseif ($env:TOOLCHAIN -eq "MSVC") {
$env:BUILD_R_FLAGS = "'--skip-install'"
} else {
Write-Output "[ERROR] Unrecognized toolchain: $env:TOOLCHAIN"
Assert-Output $false
}
Invoke-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"; Assert-Output $?
} elseif ($env:R_BUILD_TYPE -eq "cran") {
# NOTE: gzip and tar are needed to create a CRAN package on Windows, but
# some flavors of tar.exe can fail in some settings on Windows.
# Putting the msys64 utilities at the beginning of PATH temporarily to be
# sure they're used for that purpose.
if ($env:R_MAJOR_VERSION -eq "3") {
$env:PATH = "C:\msys64\usr\bin;" + $env:PATH
}
Invoke-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Assert-Output $?
Remove-From-Path ".*msys64.*"
# Test CRAN source .tar.gz in a directory that is not this repo or below it.
# When people install.packages('lightgbm'), they won't have the LightGBM
# git repo around. This is to protect against the use of relative paths
# like ../../CMakeLists.txt that would only work if you are in the repoo
$R_CMD_CHECK_DIR = "tmp-r-cmd-check"
New-Item -Path "C:\" -Name $R_CMD_CHECK_DIR -ItemType "directory" > $null
Move-Item -Path "$PKG_FILE_NAME" -Destination "C:\$R_CMD_CHECK_DIR\" > $null
cd "C:\$R_CMD_CHECK_DIR\"
}
Invoke-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Assert-Output $?
Remove-From-Path ".*msys64.*"
# Test CRAN source .tar.gz in a directory that is not this repo or below it.
# When people install.packages('lightgbm'), they won't have the LightGBM
# git repo around. This is to protect against the use of relative paths
# like ../../CMakeLists.txt that would only work if you are in the repoo
$R_CMD_CHECK_DIR = "tmp-r-cmd-check"
New-Item -Path "C:\" -Name $R_CMD_CHECK_DIR -ItemType "directory" > $null
Move-Item -Path "$PKG_FILE_NAME" -Destination "C:\$R_CMD_CHECK_DIR\" > $null
cd "C:\$R_CMD_CHECK_DIR\"
}

Write-Output "Running R CMD check"
if ($env:R_BUILD_TYPE -eq "cran") {
# CRAN packages must pass without --no-multiarch (build on 64-bit and 32-bit)
$check_args = "c('CMD', 'check', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')"
} else {
$check_args = "c('CMD', 'check', '--no-multiarch', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')"
}
Invoke-R-Code-Redirect-Stderr "result <- processx::run(command = 'R.exe', args = $check_args, echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; $check_succeeded = $?
Write-Output "Running R CMD check"
if ($env:R_BUILD_TYPE -eq "cran") {
# CRAN packages must pass without --no-multiarch (build on 64-bit and 32-bit)
$check_args = "c('CMD', 'check', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')"
} else {
$check_args = "c('CMD', 'check', '--no-multiarch', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')"
}
Invoke-R-Code-Redirect-Stderr "result <- processx::run(command = 'R.exe', args = $check_args, echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; $check_succeeded = $?

Write-Output "R CMD check build logs:"
$INSTALL_LOG_FILE_NAME = "lightgbm.Rcheck\00install.out"
Get-Content -Path "$INSTALL_LOG_FILE_NAME"
Write-Output "R CMD check build logs:"
$INSTALL_LOG_FILE_NAME = "lightgbm.Rcheck\00install.out"
Get-Content -Path "$INSTALL_LOG_FILE_NAME"

Assert-Output $check_succeeded
Assert-Output $check_succeeded

Write-Output "Looking for issues with R CMD check results"
if (Get-Content "$LOG_FILE_NAME" | Select-String -Pattern "NOTE|WARNING|ERROR" -CaseSensitive -Quiet) {
echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check"
Assert-Output $False
}
Write-Output "Looking for issues with R CMD check results"
if (Get-Content "$LOG_FILE_NAME" | Select-String -Pattern "NOTE|WARNING|ERROR" -CaseSensitive -Quiet) {
echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check"
Assert-Output $False
}

} else {
$INSTALL_LOG_FILE_NAME = "$env:BUILD_SOURCESDIRECTORY\00install_out.txt"
Invoke-R-Code-Redirect-Stderr "source('build_r.R')" 1> $INSTALL_LOG_FILE_NAME ; $install_succeeded = $?
Write-Output "----- build and install logs -----"
Get-Content -Path "$INSTALL_LOG_FILE_NAME"
Write-Output "----- end of build and install logs -----"
Assert-Output $install_succeeded
# some errors are not raised above, but can be found in the logs
if (Get-Content "$INSTALL_LOG_FILE_NAME" | Select-String -Pattern "ERROR" -CaseSensitive -Quiet) {
echo "ERRORs have been found installing lightgbm"
Assert-Output $False
}
$INSTALL_LOG_FILE_NAME = "$env:BUILD_SOURCESDIRECTORY\00install_out.txt"
Invoke-R-Code-Redirect-Stderr "source('build_r.R')" 1> $INSTALL_LOG_FILE_NAME ; $install_succeeded = $?
Write-Output "----- build and install logs -----"
Get-Content -Path "$INSTALL_LOG_FILE_NAME"
Write-Output "----- end of build and install logs -----"
Assert-Output $install_succeeded
# some errors are not raised above, but can be found in the logs
if (Get-Content "$INSTALL_LOG_FILE_NAME" | Select-String -Pattern "ERROR" -CaseSensitive -Quiet) {
echo "ERRORs have been found installing lightgbm"
Assert-Output $False
}
}

# Checking that the correct R version was used
if ($env:TOOLCHAIN -ne "MSVC") {
$checks = Select-String -Path "${LOG_FILE_NAME}" -Pattern "using R version $env:R_WINDOWS_VERSION"
$checks_cnt = $checks.Matches.length
$checks = Select-String -Path "${LOG_FILE_NAME}" -Pattern "using R version $env:R_WINDOWS_VERSION"
$checks_cnt = $checks.Matches.length
} else {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "R version passed into FindLibR.* $env:R_WINDOWS_VERSION"
$checks_cnt = $checks.Matches.length
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "R version passed into FindLibR.* $env:R_WINDOWS_VERSION"
$checks_cnt = $checks.Matches.length
}
if ($checks_cnt -eq 0) {
Write-Output "Wrong R version was found (expected '$env:R_WINDOWS_VERSION'). Check the build logs."
Assert-Output $False
Write-Output "Wrong R version was found (expected '$env:R_WINDOWS_VERSION'). Check the build logs."
Assert-Output $False
}

# Checking that we actually got the expected compiler. The R-package has some logic
# to fail back to MinGW if MSVC fails, but for CI builds we need to check that the correct
# compiler was used.
if ($env:R_BUILD_TYPE -eq "cmake") {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "Check for working CXX compiler.*$env:COMPILER"
if ($checks.Matches.length -eq 0) {
Write-Output "The wrong compiler was used. Check the build logs."
Assert-Output $False
}
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "Check for working CXX compiler.*$env:COMPILER"
if ($checks.Matches.length -eq 0) {
Write-Output "The wrong compiler was used. Check the build logs."
Assert-Output $False
}
}

# Checking that we got the right toolchain for MinGW. If using MinGW, both
# MinGW and MSYS toolchains are supported
if (($env:COMPILER -eq "MINGW") -and ($env:R_BUILD_TYPE -eq "cmake")) {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "Trying to build with.*$env:TOOLCHAIN"
if ($checks.Matches.length -eq 0) {
Write-Output "The wrong toolchain was used. Check the build logs."
Assert-Output $False
}
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "Trying to build with.*$env:TOOLCHAIN"
if ($checks.Matches.length -eq 0) {
Write-Output "The wrong toolchain was used. Check the build logs."
Assert-Output $False
}
}

# Checking that MM_PREFETCH preprocessor definition is actually used in CI builds.
if ($env:R_BUILD_TYPE -eq "cran") {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "checking whether MM_PREFETCH work.*yes"
$checks_cnt = $checks.Matches.length
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "checking whether MM_PREFETCH work.*yes"
$checks_cnt = $checks.Matches.length
} elseif ($env:TOOLCHAIN -ne "MSVC") {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Performing Test MM_PREFETCH - Success"
$checks_cnt = $checks.Matches.length
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Performing Test MM_PREFETCH - Success"
$checks_cnt = $checks.Matches.length
} else {
$checks_cnt = 1
$checks_cnt = 1
}
if ($checks_cnt -eq 0) {
Write-Output "MM_PREFETCH preprocessor definition wasn't used. Check the build logs."
Assert-Output $False
Write-Output "MM_PREFETCH preprocessor definition wasn't used. Check the build logs."
Assert-Output $False
}

# Checking that MM_MALLOC preprocessor definition is actually used in CI builds.
if ($env:R_BUILD_TYPE -eq "cran") {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "checking whether MM_MALLOC work.*yes"
$checks_cnt = $checks.Matches.length
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "checking whether MM_MALLOC work.*yes"
$checks_cnt = $checks.Matches.length
} elseif ($env:TOOLCHAIN -ne "MSVC") {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Performing Test MM_MALLOC - Success"
$checks_cnt = $checks.Matches.length
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Performing Test MM_MALLOC - Success"
$checks_cnt = $checks.Matches.length
} else {
$checks_cnt = 1
$checks_cnt = 1
}
if ($checks_cnt -eq 0) {
Write-Output "MM_MALLOC preprocessor definition wasn't used. Check the build logs."
Assert-Output $False
Write-Output "MM_MALLOC preprocessor definition wasn't used. Check the build logs."
Assert-Output $False
}

# Checking that OpenMP is actually used in CMake builds.
if ($env:R_BUILD_TYPE -eq "cmake") {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Found OpenMP: TRUE.*"
if ($checks.Matches.length -eq 0) {
Write-Output "OpenMP wasn't found. Check the build logs."
Assert-Output $False
}
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Found OpenMP: TRUE.*"
if ($checks.Matches.length -eq 0) {
Write-Output "OpenMP wasn't found. Check the build logs."
Assert-Output $False
}
}

if ($env:COMPILER -eq "MSVC") {
Write-Output "Running tests with testthat.R"
cd R-package/tests
# NOTE: using Rscript.exe intentionally here, instead of Invoke-R-Code-Redirect-Stderr,
# because something about the interaction between Invoke-R-Code-Redirect-Stderr
# and testthat results in failing tests not exiting with a non-0 exit code.
Rscript.exe --vanilla "testthat.R" ; Assert-Output $?
Write-Output "Running tests with testthat.R"
cd R-package/tests
# NOTE: using Rscript.exe intentionally here, instead of Invoke-R-Code-Redirect-Stderr,
# because something about the interaction between Invoke-R-Code-Redirect-Stderr
# and testthat results in failing tests not exiting with a non-0 exit code.
Rscript.exe --vanilla "testthat.R" ; Assert-Output $?
}

Write-Output "No issues were found checking the R-package"
Loading

0 comments on commit 92aa07b

Please sign in to comment.