From 82d8c6fe4b071f79a31fa312b2a1c3a476bc9a2f Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Sat, 7 Sep 2024 21:40:33 +0100 Subject: [PATCH 01/22] Fixing shellcheck in .ci/test.sh .ci/test-r-package.sh .ci/setup.sh --- .ci/setup.sh | 10 ++++----- .ci/test-r-package.sh | 34 ++++++++++++++-------------- .ci/test.sh | 52 +++++++++++++++++++++++-------------------- 3 files changed, 50 insertions(+), 46 deletions(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index d3b4fffb5ca5..e51076695e34 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -29,7 +29,7 @@ if [[ $OS_NAME == "macos" ]]; then brew install swig fi else # Linux - if type -f apt 2>&1 > /dev/null; then + if { type -f apt > /dev/null ;} 2>&1 then sudo apt-get update sudo apt-get install --no-install-recommends -y \ ca-certificates \ @@ -42,10 +42,10 @@ else # Linux fi CMAKE_VERSION="3.30.0" curl -O -L \ - https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${ARCH}.sh \ + "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${ARCH}.sh" \ || exit 1 sudo mkdir /opt/cmake || exit 1 - sudo sh cmake-${CMAKE_VERSION}-linux-${ARCH}.sh --skip-license --prefix=/opt/cmake || exit 1 + sudo sh "cmake-${CMAKE_VERSION}-linux-${ARCH}.sh" --skip-license --prefix=/opt/cmake || exit 1 sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake || exit 1 if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then @@ -147,8 +147,8 @@ if [[ "${TASK}" != "r-package" ]] && [[ "${TASK}" != "r-rchk" ]]; then curl \ -sL \ -o miniforge.sh \ - https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-${ARCH}.sh - sh miniforge.sh -b -p $CONDA + "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-${ARCH}.sh" + sh miniforge.sh -b -p "${CONDA}" fi conda config --set always_yes yes --set changeps1 no conda update -q -y conda diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index 5eba7669ad14..6db0beb8293b 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -19,13 +19,13 @@ if [[ $R_BUILD_TYPE != "cran" ]]; then fi # Get details needed for installing R components -R_MAJOR_VERSION=( ${R_VERSION//./ } ) -if [[ "${R_MAJOR_VERSION}" == "3" ]]; then +R_MAJOR_VERSION=( "${R_VERSION//./ }" ) +if [[ "${R_MAJOR_VERSION[0]}" == "3" ]]; then export R_MAC_VERSION=3.6.3 export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/R-${R_MAC_VERSION}.nn.pkg export R_LINUX_VERSION="3.6.3-1bionic" export R_APT_REPO="bionic-cran35/" -elif [[ "${R_MAJOR_VERSION}" == "4" ]]; then +elif [[ "${R_MAJOR_VERSION[0]}" == "4" ]]; then export R_MAC_VERSION=4.3.1 export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/big-sur-${ARCH}/base/R-${R_MAC_VERSION}-${ARCH}.pkg export R_LINUX_VERSION="4.3.1-1.2204.0" @@ -69,7 +69,7 @@ if [[ $OS_NAME == "linux" ]]; then sudo apt-get install \ --no-install-recommends \ -y \ - autoconf=$(cat R-package/AUTOCONF_UBUNTU_VERSION) \ + "autoconf=$(cat R-package/AUTOCONF_UBUNTU_VERSION)" \ automake \ || exit 1 fi @@ -90,9 +90,9 @@ if [[ $OS_NAME == "macos" ]]; then sudo tlmgr --verify-repo=none update --self || exit 1 sudo tlmgr --verify-repo=none install inconsolata helvetic rsfs || exit 1 - curl -sL ${R_MAC_PKG_URL} -o R.pkg || exit 1 + curl -sL "${R_MAC_PKG_URL}" -o R.pkg || exit 1 sudo installer \ - -pkg $(pwd)/R.pkg \ + -pkg "$(pwd)/R.pkg" \ -target / || exit 1 # install tidy v5.8.0 @@ -100,7 +100,7 @@ if [[ $OS_NAME == "macos" ]]; then TIDY_URL=https://github.com/htacg/tidy-html5/releases/download/5.8.0/tidy-5.8.0-macos-x86_64+arm64.pkg curl -sL ${TIDY_URL} -o tidy.pkg sudo installer \ - -pkg $(pwd)/tidy.pkg \ + -pkg "$(pwd)/tidy.pkg" \ -target / # ensure that this newer version of 'tidy' is used by 'R CMD check' @@ -110,7 +110,7 @@ fi # fix for issue where CRAN was not returning {lattice} and {evaluate} when using R 3.6 # "Warning: dependency ‘lattice’ is not available" -if [[ "${R_MAJOR_VERSION}" == "3" ]]; then +if [[ "${R_MAJOR_VERSION[0]}" == "3" ]]; then Rscript --vanilla -e "install.packages(c('https://cran.r-project.org/src/contrib/Archive/lattice/lattice_0.20-41.tar.gz', 'https://cran.r-project.org/src/contrib/Archive/evaluate/evaluate_0.23.tar.gz'), repos = NULL, lib = '${R_LIB_PATH}')" else # {Matrix} needs {lattice}, so this needs to run before manually installing {Matrix}. @@ -169,24 +169,24 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then if [[ "${TASK}" == "r-rchk" ]]; then echo "Checking R package with rchk" mkdir -p packages - cp ${PKG_TARBALL} packages + cp "${PKG_TARBALL}" packages RCHK_LOG_FILE="rchk-logs.txt" - docker run \ - -v $(pwd)/packages:/rchk/packages \ + { docker run \ + -v "$(pwd)/packages:/rchk/packages" \ kalibera/rchk:latest \ "/rchk/packages/${PKG_TARBALL}" \ - 2>&1 > ${RCHK_LOG_FILE} \ + > ${RCHK_LOG_FILE}; } 2>&1 \ || (cat ${RCHK_LOG_FILE} && exit 1) cat ${RCHK_LOG_FILE} # the exceptions below are from R itself and not LightGBM: # https://github.com/kalibera/rchk/issues/22#issuecomment-656036156 - exit $( + exit "$( cat ${RCHK_LOG_FILE} \ | grep -v "in function strptime_internal" \ | grep -v "in function RunGenCollect" \ | grep --count -E '\[PB\]|ERROR' - ) + )" fi # Test CRAN source .tar.gz in a directory that is not this repo or below it. @@ -194,9 +194,9 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then # git repo around. This is to protect against the use of relative paths # like ../../CMakeLists.txt that would only work if you are in the repo R_CMD_CHECK_DIR="${HOME}/tmp-r-cmd-check/" - mkdir -p ${R_CMD_CHECK_DIR} - mv ${PKG_TARBALL} ${R_CMD_CHECK_DIR} - cd ${R_CMD_CHECK_DIR} + mkdir -p "${R_CMD_CHECK_DIR}" + mv "${PKG_TARBALL}" "${R_CMD_CHECK_DIR}" + cd "${R_CMD_CHECK_DIR}" fi declare -i allowed_notes=0 diff --git a/.ci/test.sh b/.ci/test.sh index a2e2dde070a2..2a20ce054c9e 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -56,17 +56,18 @@ if [[ "$TASK" == "cpp-tests" ]]; then else extra_cmake_opts="" fi - cmake -B build -S . -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON $extra_cmake_opts + cmake -B build -S . -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON "$extra_cmake_opts" cmake --build build --target testlightgbm -j4 || exit 1 ./testlightgbm || exit 1 exit 0 fi # including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy -CONDA_PYTHON_REQUIREMENT="python=$PYTHON_VERSION[build=*cpython]" +CONDA_PYTHON_REQUIREMENT="python=${PYTHON_VERSION}[build=*cpython]" if [[ $TASK == "if-else" ]]; then - mamba create -q -y -n $CONDA_ENV ${CONDA_PYTHON_REQUIREMENT} numpy + mamba create -q -y -n $CONDA_ENV "${CONDA_PYTHON_REQUIREMENT}" numpy + # shellcheck disable=SC1091 source activate $CONDA_ENV cmake -B build -S . || exit 1 cmake --build build --target lightgbm -j4 || exit 1 @@ -89,14 +90,14 @@ if [[ $TASK == "swig" ]]; then python ./.ci/check-dynamic-dependencies.py ./objdump.log || exit 1 fi if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp ./build/lightgbmlib.jar $BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_$OS_NAME.jar + cp ./build/lightgbmlib.jar "$BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_$OS_NAME.jar" fi exit 0 fi if [[ $TASK == "lint" ]]; then mamba create -q -y -n $CONDA_ENV \ - ${CONDA_PYTHON_REQUIREMENT} \ + "${CONDA_PYTHON_REQUIREMENT}" \ 'cmakelint>=1.4.3' \ 'cpplint>=1.6.0' \ 'matplotlib-base>=3.9.1' \ @@ -104,6 +105,7 @@ if [[ $TASK == "lint" ]]; then 'pre-commit>=3.8.0' \ 'pyarrow-core>=17.0' \ 'r-lintr>=3.1.2' + # shellcheck disable=SC1091 source activate $CONDA_ENV echo "Linting Python code" bash ./.ci/lint-python.sh || exit 1 @@ -125,12 +127,13 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then -n $CONDA_ENV \ 'doxygen>=1.10.0' \ 'rstcheck>=6.2.4' || exit 1 + # shellcheck disable=SC1091 source activate $CONDA_ENV # check reStructuredText formatting cd "${BUILD_DIRECTORY}/python-package" - rstcheck --report-level warning $(find . -type f -name "*.rst") || exit 1 + rstcheck --report-level warning "$(find . -type f -name "*.rst")" || exit 1 cd "${BUILD_DIRECTORY}/docs" - rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile $(find . -type f -name "*.rst") || exit 1 + rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile "$(find . -type f -name "*.rst")" || exit 1 # build docs make html || exit 1 if [[ $TASK == "check-links" ]]; then @@ -160,10 +163,11 @@ fi mamba create \ -y \ -n $CONDA_ENV \ - ${CONDA_REQUIREMENT_FILES} \ - ${CONDA_PYTHON_REQUIREMENT} \ + "${CONDA_REQUIREMENT_FILES}" \ + "${CONDA_PYTHON_REQUIREMENT}" \ || exit 1 +# shellcheck disable=SC1091 source activate $CONDA_ENV cd "${BUILD_DIRECTORY}" @@ -171,9 +175,9 @@ cd "${BUILD_DIRECTORY}" if [[ $TASK == "sdist" ]]; then sh ./build-python.sh sdist || exit 1 sh .ci/check-python-dists.sh ./dist || exit 1 - pip install ./dist/lightgbm-$LGB_VER.tar.gz -v || exit 1 + pip install "./dist/lightgbm-$LGB_VER.tar.gz" -v || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp ./dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1 + cp "./dist/lightgbm-$LGB_VER.tar.gz" "$BUILD_ARTIFACTSTAGINGDIRECTORY" || exit 1 fi pytest ./tests/python_package_test || exit 1 exit 0 @@ -182,7 +186,7 @@ elif [[ $TASK == "bdist" ]]; then sh ./build-python.sh bdist_wheel || exit 1 sh .ci/check-python-dists.sh ./dist || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1 + cp dist/lightgbm-"$LGB_VER"-py3-none-macosx*.whl "$BUILD_ARTIFACTSTAGINGDIRECTORY" || exit 1 fi else if [[ $ARCH == "x86_64" ]]; then @@ -198,10 +202,10 @@ elif [[ $TASK == "bdist" ]]; then ./dist/tmp.whl || exit 1 mv \ ./dist/tmp.whl \ - ./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit 1 + "./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl" || exit 1 sh .ci/check-python-dists.sh ./dist || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1 + cp "dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl" "$BUILD_ARTIFACTSTAGINGDIRECTORY" || exit 1 fi # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py export LIGHTGBM_TEST_DUAL_CPU_GPU=1 @@ -220,14 +224,14 @@ if [[ $TASK == "gpu" ]]; then pip install \ -v \ --config-settings=cmake.define.USE_GPU=ON \ - ./dist/lightgbm-$LGB_VER.tar.gz \ + "./dist/lightgbm-$LGB_VER.tar.gz" \ || exit 1 pytest ./tests/python_package_test || exit 1 exit 0 elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --gpu || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install ./dist/lightgbm-$LGB_VER*.whl -v || exit 1 + pip install ./dist/lightgbm-"$LGB_VER"*.whl -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then @@ -245,14 +249,14 @@ elif [[ $TASK == "cuda" ]]; then pip install \ -v \ --config-settings=cmake.define.USE_CUDA=ON \ - ./dist/lightgbm-$LGB_VER.tar.gz \ + "./dist/lightgbm-$LGB_VER.tar.gz" \ || exit 1 pytest ./tests/python_package_test || exit 1 exit 0 elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --cuda || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install ./dist/lightgbm-$LGB_VER*.whl -v || exit 1 + pip install ./dist/lightgbm-"$LGB_VER"*.whl -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then @@ -265,14 +269,14 @@ elif [[ $TASK == "mpi" ]]; then pip install \ -v \ --config-settings=cmake.define.USE_MPI=ON \ - ./dist/lightgbm-$LGB_VER.tar.gz \ + "./dist/lightgbm-${LGB_VER}.tar.gz" \ || exit 1 pytest ./tests/python_package_test || exit 1 exit 0 elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --mpi || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install ./dist/lightgbm-$LGB_VER*.whl -v || exit 1 + pip install ./dist/lightgbm-"$LGB_VER"*.whl -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then @@ -290,13 +294,13 @@ pytest ./tests || exit 1 if [[ $TASK == "regular" ]]; then if [[ $PRODUCES_ARTIFACTS == "true" ]]; then if [[ $OS_NAME == "macos" ]]; then - cp ./lib_lightgbm.dylib $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib + cp ./lib_lightgbm.dylib "$BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib" else if [[ $COMPILER == "gcc" ]]; then objdump -T ./lib_lightgbm.so > ./objdump.log || exit 1 python ./.ci/check-dynamic-dependencies.py ./objdump.log || exit 1 fi - cp ./lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so + cp ./lib_lightgbm.so "$BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so" fi fi cd "$BUILD_DIRECTORY/examples/python-guide" @@ -310,10 +314,10 @@ matplotlib.use\(\"Agg\"\)\ 'h5py>=3.10' \ 'ipywidgets>=8.1.2' \ 'notebook>=7.1.2' - for f in *.py **/*.py; do python $f || exit 1; done # run all examples + for f in *.py **/*.py; do python "$f" || exit 1; done # run all examples cd "$BUILD_DIRECTORY/examples/python-guide/notebooks" sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb - jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb || exit 1 # run all notebooks + jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace ./*.ipynb || exit 1 # run all notebooks # importing the library should succeed even if all optional dependencies are not present conda uninstall -n $CONDA_ENV --force --yes \ From d33c56a4fda8597c294013d9e2e476cdbcc28aa5 Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Sat, 7 Sep 2024 23:50:01 +0100 Subject: [PATCH 02/22] Fixing shellcheck in .ci/test.sh .ci/test-r-package.sh .ci/setup.sh --- .ci/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index e51076695e34..caf25b38062b 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -29,7 +29,7 @@ if [[ $OS_NAME == "macos" ]]; then brew install swig fi else # Linux - if { type -f apt > /dev/null ;} 2>&1 then + if { type -f apt > /dev/null; } 2>&1; then sudo apt-get update sudo apt-get install --no-install-recommends -y \ ca-certificates \ From 3215be92132fb259724d678319b821ca71bcfca3 Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Sun, 8 Sep 2024 00:01:56 +0100 Subject: [PATCH 03/22] fix CI --- .ci/test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/test.sh b/.ci/test.sh index 2a20ce054c9e..b0a8dbd0d766 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -153,17 +153,17 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then fi if [[ $PYTHON_VERSION == "3.7" ]]; then - CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py37.txt" + CONDA_REQUIREMENT_FILES="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py37.txt" elif [[ $PYTHON_VERSION == "3.8" ]]; then - CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py38.txt" + CONDA_REQUIREMENT_FILES="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py38.txt" else - CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt" + CONDA_REQUIREMENT_FILES="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt" fi mamba create \ -y \ -n $CONDA_ENV \ - "${CONDA_REQUIREMENT_FILES}" \ + --file "${CONDA_REQUIREMENT_FILES}" \ "${CONDA_PYTHON_REQUIREMENT}" \ || exit 1 From 36d08e254cd27a48e89b379f1e50e312992a0d29 Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Sun, 8 Sep 2024 10:51:47 +0100 Subject: [PATCH 04/22] squash ci fix commits, fix author --- .ci/test-r-package.sh | 2 +- .ci/test.sh | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index 6db0beb8293b..f60278a285ca 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -19,7 +19,7 @@ if [[ $R_BUILD_TYPE != "cran" ]]; then fi # Get details needed for installing R components -R_MAJOR_VERSION=( "${R_VERSION//./ }" ) +IFS='.' read -ra R_MAJOR_VERSION <<< "$R_VERSION" if [[ "${R_MAJOR_VERSION[0]}" == "3" ]]; then export R_MAC_VERSION=3.6.3 export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/R-${R_MAC_VERSION}.nn.pkg diff --git a/.ci/test.sh b/.ci/test.sh index b0a8dbd0d766..99a0e3737908 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -131,9 +131,11 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then source activate $CONDA_ENV # check reStructuredText formatting cd "${BUILD_DIRECTORY}/python-package" - rstcheck --report-level warning "$(find . -type f -name "*.rst")" || exit 1 + find . -type f -name "*.rst" -exec \ + rstcheck --report-level warning {} + || exit 1 cd "${BUILD_DIRECTORY}/docs" - rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile "$(find . -type f -name "*.rst")" || exit 1 + find . -type f -name "*.rst" -exec \ + rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile {} + || exit 1 # build docs make html || exit 1 if [[ $TASK == "check-links" ]]; then From 7690d8b0eafe6e9f59f1117b1ac6bb6662f40e74 Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Sun, 15 Sep 2024 13:35:37 +0100 Subject: [PATCH 05/22] Update .ci/test.sh Co-authored-by: James Lamb --- .ci/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test.sh b/.ci/test.sh index 99a0e3737908..a888e0c967eb 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -56,7 +56,7 @@ if [[ "$TASK" == "cpp-tests" ]]; then else extra_cmake_opts="" fi - cmake -B build -S . -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON "$extra_cmake_opts" + cmake -B build -S . -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON "${extra_cmake_opts}" cmake --build build --target testlightgbm -j4 || exit 1 ./testlightgbm || exit 1 exit 0 From 9c216f107d148163ad22e2218f44c2cea305fac6 Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Sun, 15 Sep 2024 13:35:45 +0100 Subject: [PATCH 06/22] Update .ci/test.sh Co-authored-by: James Lamb --- .ci/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test.sh b/.ci/test.sh index a888e0c967eb..d04ab291027e 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -316,7 +316,7 @@ matplotlib.use\(\"Agg\"\)\ 'h5py>=3.10' \ 'ipywidgets>=8.1.2' \ 'notebook>=7.1.2' - for f in *.py **/*.py; do python "$f" || exit 1; done # run all examples + for f in *.py **/*.py; do python "${f}" || exit 1; done # run all examples cd "$BUILD_DIRECTORY/examples/python-guide/notebooks" sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace ./*.ipynb || exit 1 # run all notebooks From 810a1fd2e5c0ad2b2e0ab31a466a9d784a74c3e5 Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Sun, 15 Sep 2024 13:35:51 +0100 Subject: [PATCH 07/22] Update .ci/test.sh Co-authored-by: James Lamb --- .ci/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test.sh b/.ci/test.sh index d04ab291027e..b8c0df846e9f 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -296,7 +296,7 @@ pytest ./tests || exit 1 if [[ $TASK == "regular" ]]; then if [[ $PRODUCES_ARTIFACTS == "true" ]]; then if [[ $OS_NAME == "macos" ]]; then - cp ./lib_lightgbm.dylib "$BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib" + cp ./lib_lightgbm.dylib "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.dylib" else if [[ $COMPILER == "gcc" ]]; then objdump -T ./lib_lightgbm.so > ./objdump.log || exit 1 From b5cc760fdb755e7586b4323fb0be39c864f7ec8f Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Sun, 15 Sep 2024 13:35:58 +0100 Subject: [PATCH 08/22] Update .ci/test.sh Co-authored-by: James Lamb --- .ci/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test.sh b/.ci/test.sh index b8c0df846e9f..5b8fd4907d0e 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -90,7 +90,7 @@ if [[ $TASK == "swig" ]]; then python ./.ci/check-dynamic-dependencies.py ./objdump.log || exit 1 fi if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp ./build/lightgbmlib.jar "$BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_$OS_NAME.jar" + cp ./build/lightgbmlib.jar "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbmlib_${OS_NAME}.jar" fi exit 0 fi From 35de6bffc419f03393922e1f1a792c0e0f84bb0d Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Sun, 15 Sep 2024 13:37:30 +0100 Subject: [PATCH 09/22] Update .ci/test.sh Co-authored-by: James Lamb --- .ci/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test.sh b/.ci/test.sh index 5b8fd4907d0e..e55e64c76438 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -302,7 +302,7 @@ if [[ $TASK == "regular" ]]; then objdump -T ./lib_lightgbm.so > ./objdump.log || exit 1 python ./.ci/check-dynamic-dependencies.py ./objdump.log || exit 1 fi - cp ./lib_lightgbm.so "$BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so" + cp ./lib_lightgbm.so "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.so" fi fi cd "$BUILD_DIRECTORY/examples/python-guide" From efdd7034072bf2b754920ae7a86486c190907cdf Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Sun, 15 Sep 2024 13:39:45 +0100 Subject: [PATCH 10/22] adressing PR comment --- .ci/test-r-package.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index f60278a285ca..074bf705453f 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -181,12 +181,13 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then # the exceptions below are from R itself and not LightGBM: # https://github.com/kalibera/rchk/issues/22#issuecomment-656036156 - exit "$( + # shellcheck disable=SC2046 + exit $( cat ${RCHK_LOG_FILE} \ | grep -v "in function strptime_internal" \ | grep -v "in function RunGenCollect" \ | grep --count -E '\[PB\]|ERROR' - )" + ) fi # Test CRAN source .tar.gz in a directory that is not this repo or below it. From 0c453540ab69df0d63ed2867adc6fac6bc33df60 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Sun, 15 Sep 2024 20:33:26 -0500 Subject: [PATCH 11/22] Apply suggestions from code review --- .ci/test.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.ci/test.sh b/.ci/test.sh index e55e64c76438..26706d312d5f 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -177,9 +177,9 @@ cd "${BUILD_DIRECTORY}" if [[ $TASK == "sdist" ]]; then sh ./build-python.sh sdist || exit 1 sh .ci/check-python-dists.sh ./dist || exit 1 - pip install "./dist/lightgbm-$LGB_VER.tar.gz" -v || exit 1 + pip install "./dist/lightgbm-${LGB_VER}.tar.gz" -v || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp "./dist/lightgbm-$LGB_VER.tar.gz" "$BUILD_ARTIFACTSTAGINGDIRECTORY" || exit 1 + cp "./dist/lightgbm-${LGB_VER}.tar.gz" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1 fi pytest ./tests/python_package_test || exit 1 exit 0 @@ -188,7 +188,7 @@ elif [[ $TASK == "bdist" ]]; then sh ./build-python.sh bdist_wheel || exit 1 sh .ci/check-python-dists.sh ./dist || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp dist/lightgbm-"$LGB_VER"-py3-none-macosx*.whl "$BUILD_ARTIFACTSTAGINGDIRECTORY" || exit 1 + cp dist/lightgbm-"${LGB_VER}"-py3-none-macosx*.whl "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1 fi else if [[ $ARCH == "x86_64" ]]; then @@ -204,10 +204,10 @@ elif [[ $TASK == "bdist" ]]; then ./dist/tmp.whl || exit 1 mv \ ./dist/tmp.whl \ - "./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl" || exit 1 + "./dist/lightgbm-${LGB_VER}-py3-none-${PLATFORM}.whl" || exit 1 sh .ci/check-python-dists.sh ./dist || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp "dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl" "$BUILD_ARTIFACTSTAGINGDIRECTORY" || exit 1 + cp "dist/lightgbm-${LGB_VER}-py3-none-${PLATFORM}.whl" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1 fi # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py export LIGHTGBM_TEST_DUAL_CPU_GPU=1 @@ -226,14 +226,14 @@ if [[ $TASK == "gpu" ]]; then pip install \ -v \ --config-settings=cmake.define.USE_GPU=ON \ - "./dist/lightgbm-$LGB_VER.tar.gz" \ + "./dist/lightgbm-${LGB_VER}.tar.gz" \ || exit 1 pytest ./tests/python_package_test || exit 1 exit 0 elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --gpu || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install ./dist/lightgbm-"$LGB_VER"*.whl -v || exit 1 + pip install "$(echo ./dist/lightgbm-${LGB_VER}*.whl)" -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then @@ -251,14 +251,14 @@ elif [[ $TASK == "cuda" ]]; then pip install \ -v \ --config-settings=cmake.define.USE_CUDA=ON \ - "./dist/lightgbm-$LGB_VER.tar.gz" \ + "./dist/lightgbm-${LGB_VER}.tar.gz" \ || exit 1 pytest ./tests/python_package_test || exit 1 exit 0 elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --cuda || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install ./dist/lightgbm-"$LGB_VER"*.whl -v || exit 1 + pip install "$(echo ./dist/lightgbm-${LGB_VER}*.whl)" -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then @@ -278,7 +278,7 @@ elif [[ $TASK == "mpi" ]]; then elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --mpi || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install ./dist/lightgbm-"$LGB_VER"*.whl -v || exit 1 + pip install "$(echo ./dist/lightgbm-${LGB_VER}*.whl)" -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then From 051fef3bf736bea31a80e960e589a1f4d3399b6a Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Mon, 16 Sep 2024 07:38:22 +0100 Subject: [PATCH 12/22] Apply suggestions from code review Co-authored-by: James Lamb --- .ci/setup.sh | 2 +- .ci/test-r-package.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index caf25b38062b..e551b1683aef 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -29,7 +29,7 @@ if [[ $OS_NAME == "macos" ]]; then brew install swig fi else # Linux - if { type -f apt > /dev/null; } 2>&1; then + if type -f apt > /dev/null 2>&1; then sudo apt-get update sudo apt-get install --no-install-recommends -y \ ca-certificates \ diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index 074bf705453f..66c832d01d95 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -20,12 +20,12 @@ fi # Get details needed for installing R components IFS='.' read -ra R_MAJOR_VERSION <<< "$R_VERSION" -if [[ "${R_MAJOR_VERSION[0]}" == "3" ]]; then +if [[ "${R_MAJOR_VERSION}" == "3" ]]; then export R_MAC_VERSION=3.6.3 export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/R-${R_MAC_VERSION}.nn.pkg export R_LINUX_VERSION="3.6.3-1bionic" export R_APT_REPO="bionic-cran35/" -elif [[ "${R_MAJOR_VERSION[0]}" == "4" ]]; then +elif [[ "${R_MAJOR_VERSION}" == "4" ]]; then export R_MAC_VERSION=4.3.1 export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/big-sur-${ARCH}/base/R-${R_MAC_VERSION}-${ARCH}.pkg export R_LINUX_VERSION="4.3.1-1.2204.0" @@ -110,7 +110,7 @@ fi # fix for issue where CRAN was not returning {lattice} and {evaluate} when using R 3.6 # "Warning: dependency ‘lattice’ is not available" -if [[ "${R_MAJOR_VERSION[0]}" == "3" ]]; then +if [[ "${R_MAJOR_VERSION}" == "3" ]]; then Rscript --vanilla -e "install.packages(c('https://cran.r-project.org/src/contrib/Archive/lattice/lattice_0.20-41.tar.gz', 'https://cran.r-project.org/src/contrib/Archive/evaluate/evaluate_0.23.tar.gz'), repos = NULL, lib = '${R_LIB_PATH}')" else # {Matrix} needs {lattice}, so this needs to run before manually installing {Matrix}. From 6da33046b1616415bfc30761fb5a4e2fe07edf16 Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Mon, 16 Sep 2024 10:58:25 +0100 Subject: [PATCH 13/22] adressing PR comments --- .ci/test-r-package.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index 66c832d01d95..9a11faecc1d5 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -19,7 +19,7 @@ if [[ $R_BUILD_TYPE != "cran" ]]; then fi # Get details needed for installing R components -IFS='.' read -ra R_MAJOR_VERSION <<< "$R_VERSION" +R_MAJOR_VERSION="${R_VERSION%.*.*}" if [[ "${R_MAJOR_VERSION}" == "3" ]]; then export R_MAC_VERSION=3.6.3 export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/R-${R_MAC_VERSION}.nn.pkg @@ -171,11 +171,11 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then mkdir -p packages cp "${PKG_TARBALL}" packages RCHK_LOG_FILE="rchk-logs.txt" - { docker run \ + docker run \ -v "$(pwd)/packages:/rchk/packages" \ kalibera/rchk:latest \ "/rchk/packages/${PKG_TARBALL}" \ - > ${RCHK_LOG_FILE}; } 2>&1 \ + > ${RCHK_LOG_FILE} 2>&1 \ || (cat ${RCHK_LOG_FILE} && exit 1) cat ${RCHK_LOG_FILE} From a7222103eabee13c4d1fa9ce54befc6b3d61d31c Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Mon, 16 Sep 2024 14:46:10 +0100 Subject: [PATCH 14/22] rename variable --- .ci/test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.ci/test.sh b/.ci/test.sh index 26706d312d5f..a7f66e45f214 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -155,17 +155,17 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then fi if [[ $PYTHON_VERSION == "3.7" ]]; then - CONDA_REQUIREMENT_FILES="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py37.txt" + CONDA_REQUIREMENT_FILE="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py37.txt" elif [[ $PYTHON_VERSION == "3.8" ]]; then - CONDA_REQUIREMENT_FILES="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py38.txt" + CONDA_REQUIREMENT_FILE="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py38.txt" else - CONDA_REQUIREMENT_FILES="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt" + CONDA_REQUIREMENT_FILE="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt" fi mamba create \ -y \ -n $CONDA_ENV \ - --file "${CONDA_REQUIREMENT_FILES}" \ + --file "${CONDA_REQUIREMENT_FILE}" \ "${CONDA_PYTHON_REQUIREMENT}" \ || exit 1 From 1ec9e93b4bae749038905c98d30cf31052a3dc46 Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Mon, 16 Sep 2024 21:24:33 +0100 Subject: [PATCH 15/22] Update .ci/test-r-package.sh Co-authored-by: James Lamb --- .ci/test-r-package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index 9a11faecc1d5..6cdd250e6372 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -19,7 +19,7 @@ if [[ $R_BUILD_TYPE != "cran" ]]; then fi # Get details needed for installing R components -R_MAJOR_VERSION="${R_VERSION%.*.*}" +R_MAJOR_VERSION="${R_VERSION%.*}" if [[ "${R_MAJOR_VERSION}" == "3" ]]; then export R_MAC_VERSION=3.6.3 export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/R-${R_MAC_VERSION}.nn.pkg From 2302dcf57a5a898c0ad6e511a865579be791b873 Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Mon, 16 Sep 2024 21:41:08 +0100 Subject: [PATCH 16/22] adress PR --- .ci/test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/test.sh b/.ci/test.sh index a7f66e45f214..982cace6cff1 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -233,7 +233,7 @@ if [[ $TASK == "gpu" ]]; then elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --gpu || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install "$(echo ./dist/lightgbm-${LGB_VER}*.whl)" -v || exit 1 + pip install "$(echo ./dist/lightgbm-"${LGB_VER}"*.whl)" -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then @@ -258,7 +258,7 @@ elif [[ $TASK == "cuda" ]]; then elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --cuda || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install "$(echo ./dist/lightgbm-${LGB_VER}*.whl)" -v || exit 1 + pip install "$(echo ./dist/lightgbm-"${LGB_VER}"*.whl)" -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then @@ -278,7 +278,7 @@ elif [[ $TASK == "mpi" ]]; then elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --mpi || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install "$(echo ./dist/lightgbm-${LGB_VER}*.whl)" -v || exit 1 + pip install "$(echo ./dist/lightgbm-"${LGB_VER}"*.whl)" -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then From 43736d5d5c17c9b23c0ce07fba4e1a3d55fd0b79 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 16 Sep 2024 22:22:49 -0500 Subject: [PATCH 17/22] fix quoting, simplify rstcheck usage --- .ci/test.sh | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/.ci/test.sh b/.ci/test.sh index 982cace6cff1..9c584ab9501f 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -117,10 +117,9 @@ if [[ $TASK == "lint" ]]; then fi if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then - cd "${BUILD_DIRECTORY}/docs" mamba env create \ -n $CONDA_ENV \ - --file ./env.yml || exit 1 + --file ./docs/env.yml || exit 1 mamba install \ -q \ -y \ @@ -130,22 +129,19 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then # shellcheck disable=SC1091 source activate $CONDA_ENV # check reStructuredText formatting - cd "${BUILD_DIRECTORY}/python-package" - find . -type f -name "*.rst" -exec \ - rstcheck --report-level warning {} + || exit 1 - cd "${BUILD_DIRECTORY}/docs" - find . -type f -name "*.rst" -exec \ - rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile {} + || exit 1 + find "${BUILD_DIRECTORY}/python-package" -type f -name "*.rst" \ + -exec rstcheck {} \+ + find "${BUILD_DIRECTORY}/docs" -type f -name "*.rst" \ + -exec rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile {} \+ # build docs - make html || exit 1 + make -C docs html || exit 1 if [[ $TASK == "check-links" ]]; then # check docs for broken links pip install linkchecker - linkchecker --config=.linkcheckerrc ./_build/html/*.html || exit 1 + linkchecker --config=.linkcheckerrc ./docs/_build/html/*.html || exit 1 exit 0 fi # check the consistency of parameters' descriptions and other stuff - cd "${BUILD_DIRECTORY}" cp ./docs/Parameters.rst ./docs/Parameters-backup.rst cp ./src/io/config_auto.cpp ./src/io/config_auto-backup.cpp python ./.ci/parameter-generator.py || exit 1 @@ -188,7 +184,7 @@ elif [[ $TASK == "bdist" ]]; then sh ./build-python.sh bdist_wheel || exit 1 sh .ci/check-python-dists.sh ./dist || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp dist/lightgbm-"${LGB_VER}"-py3-none-macosx*.whl "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1 + cp "$(echo "dist/lightgbm-${LGB_VER}-py3-none-macosx"*.whl)" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1 fi else if [[ $ARCH == "x86_64" ]]; then From 5f080c9146b28db603217f69e366ae6727fe6a93 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 16 Sep 2024 22:27:20 -0500 Subject: [PATCH 18/22] more quoting --- .ci/test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/test.sh b/.ci/test.sh index 9c584ab9501f..a0910eb3933c 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -66,9 +66,9 @@ fi CONDA_PYTHON_REQUIREMENT="python=${PYTHON_VERSION}[build=*cpython]" if [[ $TASK == "if-else" ]]; then - mamba create -q -y -n $CONDA_ENV "${CONDA_PYTHON_REQUIREMENT}" numpy + mamba create -q -y -n "${CONDA_ENV}" "${CONDA_PYTHON_REQUIREMENT}" numpy # shellcheck disable=SC1091 - source activate $CONDA_ENV + source activate "${CONDA_ENV}" cmake -B build -S . || exit 1 cmake --build build --target lightgbm -j4 || exit 1 cd "$BUILD_DIRECTORY/tests/cpp_tests" From d90accb97721a64519d610d0dbc7158616bdfa94 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 8 Oct 2024 23:08:47 -0500 Subject: [PATCH 19/22] use bash array for cpp-tests --- .ci/test.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/.ci/test.sh b/.ci/test.sh index 71021c50a0f2..bd356b109428 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -48,15 +48,18 @@ if [[ "${TASK}" == "r-package" ]] || [[ "${TASK}" == "r-rchk" ]]; then fi if [[ "$TASK" == "cpp-tests" ]]; then + cmake_args=( + -DBUILD_CPP_TEST=ON + -DUSE_OPENMP=OFF + -DUSE_DEBUG=ON + ) if [[ $METHOD == "with-sanitizers" ]]; then - extra_cmake_opts="-DUSE_SANITIZER=ON" + cmake_args+=("-DUSE_SANITIZER=ON") if [[ -n $SANITIZERS ]]; then - extra_cmake_opts="$extra_cmake_opts -DENABLED_SANITIZERS=$SANITIZERS" + cmake_args+=("-DENABLED_SANITIZERS=$SANITIZERS") fi - else - extra_cmake_opts="" fi - cmake -B build -S . -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON "${extra_cmake_opts}" + cmake -B build -S . "${cmake_args[@]}" cmake --build build --target testlightgbm -j4 || exit 1 ./testlightgbm || exit 1 exit 0 From 7e81ca9dcc1febbccd755f63cd49ed9d9f0292de Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Fri, 11 Oct 2024 12:08:04 +0100 Subject: [PATCH 20/22] bringing back flag and exit code --- .ci/test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.ci/test.sh b/.ci/test.sh index e21ace441004..7bf049bcb44f 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -134,9 +134,9 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then source activate $CONDA_ENV # check reStructuredText formatting find "${BUILD_DIRECTORY}/python-package" -type f -name "*.rst" \ - -exec rstcheck {} \+ + -exec rstcheck --report-level warning {} \+ || exit 1 find "${BUILD_DIRECTORY}/docs" -type f -name "*.rst" \ - -exec rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile {} \+ + -exec rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile {} \+ || exit 1 # build docs make -C docs html || exit 1 if [[ $TASK == "check-links" ]]; then @@ -164,7 +164,7 @@ fi conda create \ -y \ - -n $CONDA_ENV \ + -n "${CONDA_ENV}" \ --file "${CONDA_REQUIREMENT_FILE}" \ "${CONDA_PYTHON_REQUIREMENT}" \ || exit 1 From f6e469e62918a8f9549efbf3088fd730eb6cd182 Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Fri, 11 Oct 2024 12:09:28 +0100 Subject: [PATCH 21/22] Apply suggestions from code review Co-authored-by: Nikita Titov --- .ci/test-r-package.sh | 5 ++--- .ci/test.sh | 12 ++++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index bc3a4af82903..9525a9bf35ea 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -175,15 +175,14 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then -v "$(pwd)/packages:/rchk/packages" \ kalibera/rchk:latest \ "/rchk/packages/${PKG_TARBALL}" \ - > ${RCHK_LOG_FILE} 2>&1 \ + > "${RCHK_LOG_FILE}" 2>&1 \ || (cat ${RCHK_LOG_FILE} && exit 1) cat ${RCHK_LOG_FILE} # the exceptions below are from R itself and not LightGBM: # https://github.com/kalibera/rchk/issues/22#issuecomment-656036156 - # shellcheck disable=SC2046 exit $( - cat ${RCHK_LOG_FILE} \ + cat "${RCHK_LOG_FILE}" \ | grep -v "in function strptime_internal" \ | grep -v "in function RunGenCollect" \ | grep --count -E '\[PB\]|ERROR' diff --git a/.ci/test.sh b/.ci/test.sh index 7bf049bcb44f..238997e8d313 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -110,7 +110,7 @@ if [[ $TASK == "lint" ]]; then 'scikit-learn>=1.5.2' \ 'r-lintr>=3.1.2' # shellcheck disable=SC1091 - source activate $CONDA_ENV + source activate "${CONDA_ENV}" echo "Linting Python code" bash ./.ci/lint-python.sh || exit 1 echo "Linting R code" @@ -127,11 +127,11 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then conda install \ -q \ -y \ - -n $CONDA_ENV \ + -n "${CONDA_ENV}" \ 'doxygen>=1.10.0' \ 'rstcheck>=6.2.4' || exit 1 # shellcheck disable=SC1091 - source activate $CONDA_ENV + source activate "${CONDA_ENV}" # check reStructuredText formatting find "${BUILD_DIRECTORY}/python-package" -type f -name "*.rst" \ -exec rstcheck --report-level warning {} \+ || exit 1 @@ -233,7 +233,7 @@ if [[ $TASK == "gpu" ]]; then elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --gpu || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install "$(echo ./dist/lightgbm-"${LGB_VER}"*.whl)" -v || exit 1 + pip install "$(echo "./dist/lightgbm-${LGB_VER}"*.whl)" -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then @@ -258,7 +258,7 @@ elif [[ $TASK == "cuda" ]]; then elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --cuda || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install "$(echo ./dist/lightgbm-"${LGB_VER}"*.whl)" -v || exit 1 + pip install "$(echo "./dist/lightgbm-${LGB_VER}"*.whl)" -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then @@ -278,7 +278,7 @@ elif [[ $TASK == "mpi" ]]; then elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --mpi || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install "$(echo ./dist/lightgbm-"${LGB_VER}"*.whl)" -v || exit 1 + pip install "$(echo "./dist/lightgbm-${LGB_VER}"*.whl)" -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then From d723749f4830e4678733358f08d7ca47a0ed7d20 Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Fri, 11 Oct 2024 12:16:50 +0100 Subject: [PATCH 22/22] fixing two shellcheck warnings --- .ci/test-r-package.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index 9525a9bf35ea..7d821676bb71 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -181,12 +181,11 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then # the exceptions below are from R itself and not LightGBM: # https://github.com/kalibera/rchk/issues/22#issuecomment-656036156 - exit $( - cat "${RCHK_LOG_FILE}" \ - | grep -v "in function strptime_internal" \ + exit "$( + grep "${RCHK_LOG_FILE}" -v "in function strptime_internal" \ | grep -v "in function RunGenCollect" \ | grep --count -E '\[PB\]|ERROR' - ) + )" fi # Test CRAN source .tar.gz in a directory that is not this repo or below it.