From 562157ecbd241d9e06b8b5c484216356e479ef61 Mon Sep 17 00:00:00 2001 From: vnherdeiro Date: Sun, 13 Oct 2024 21:56:16 +0100 Subject: [PATCH] [ci] fix shellcheck warnings in CI scripts (#6646) Co-authored-by: James Lamb Co-authored-by: Nikita Titov --- .ci/setup.sh | 10 ++--- .ci/test-r-package.sh | 29 +++++++------ .ci/test.sh | 95 +++++++++++++++++++++++-------------------- 3 files changed, 69 insertions(+), 65 deletions(-) diff --git a/.ci/setup.sh b/.ci/setup.sh index d3b4fffb5ca5..e551b1683aef 100755 --- a/.ci/setup.sh +++ b/.ci/setup.sh @@ -29,7 +29,7 @@ if [[ $OS_NAME == "macos" ]]; then brew install swig fi else # Linux - if type -f apt 2>&1 > /dev/null; then + if type -f apt > /dev/null 2>&1; then sudo apt-get update sudo apt-get install --no-install-recommends -y \ ca-certificates \ @@ -42,10 +42,10 @@ else # Linux fi CMAKE_VERSION="3.30.0" curl -O -L \ - https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${ARCH}.sh \ + "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-${ARCH}.sh" \ || exit 1 sudo mkdir /opt/cmake || exit 1 - sudo sh cmake-${CMAKE_VERSION}-linux-${ARCH}.sh --skip-license --prefix=/opt/cmake || exit 1 + sudo sh "cmake-${CMAKE_VERSION}-linux-${ARCH}.sh" --skip-license --prefix=/opt/cmake || exit 1 sudo ln -sf /opt/cmake/bin/cmake /usr/local/bin/cmake || exit 1 if [[ $IN_UBUNTU_BASE_CONTAINER == "true" ]]; then @@ -147,8 +147,8 @@ if [[ "${TASK}" != "r-package" ]] && [[ "${TASK}" != "r-rchk" ]]; then curl \ -sL \ -o miniforge.sh \ - https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-${ARCH}.sh - sh miniforge.sh -b -p $CONDA + "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-$(uname)-${ARCH}.sh" + sh miniforge.sh -b -p "${CONDA}" fi conda config --set always_yes yes --set changeps1 no conda update -q -y conda diff --git a/.ci/test-r-package.sh b/.ci/test-r-package.sh index 6618d5450db4..7d821676bb71 100755 --- a/.ci/test-r-package.sh +++ b/.ci/test-r-package.sh @@ -19,7 +19,7 @@ if [[ $R_BUILD_TYPE != "cran" ]]; then fi # Get details needed for installing R components -R_MAJOR_VERSION=( ${R_VERSION//./ } ) +R_MAJOR_VERSION="${R_VERSION%.*}" if [[ "${R_MAJOR_VERSION}" == "3" ]]; then export R_MAC_VERSION=3.6.3 export R_MAC_PKG_URL=${CRAN_MIRROR}/bin/macosx/R-${R_MAC_VERSION}.nn.pkg @@ -69,7 +69,7 @@ if [[ $OS_NAME == "linux" ]]; then sudo apt-get install \ --no-install-recommends \ -y \ - autoconf=$(cat R-package/AUTOCONF_UBUNTU_VERSION) \ + "autoconf=$(cat R-package/AUTOCONF_UBUNTU_VERSION)" \ automake \ || exit 1 fi @@ -90,9 +90,9 @@ if [[ $OS_NAME == "macos" ]]; then sudo tlmgr --verify-repo=none update --self || exit 1 sudo tlmgr --verify-repo=none install inconsolata helvetic rsfs || exit 1 - curl -sL ${R_MAC_PKG_URL} -o R.pkg || exit 1 + curl -sL "${R_MAC_PKG_URL}" -o R.pkg || exit 1 sudo installer \ - -pkg $(pwd)/R.pkg \ + -pkg "$(pwd)/R.pkg" \ -target / || exit 1 # install tidy v5.8.0 @@ -100,7 +100,7 @@ if [[ $OS_NAME == "macos" ]]; then TIDY_URL=https://github.com/htacg/tidy-html5/releases/download/5.8.0/tidy-5.8.0-macos-x86_64+arm64.pkg curl -sL ${TIDY_URL} -o tidy.pkg sudo installer \ - -pkg $(pwd)/tidy.pkg \ + -pkg "$(pwd)/tidy.pkg" \ -target / # ensure that this newer version of 'tidy' is used by 'R CMD check' @@ -169,24 +169,23 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then if [[ "${TASK}" == "r-rchk" ]]; then echo "Checking R-package with rchk" mkdir -p packages - cp ${PKG_TARBALL} packages + cp "${PKG_TARBALL}" packages RCHK_LOG_FILE="rchk-logs.txt" docker run \ - -v $(pwd)/packages:/rchk/packages \ + -v "$(pwd)/packages:/rchk/packages" \ kalibera/rchk:latest \ "/rchk/packages/${PKG_TARBALL}" \ - 2>&1 > ${RCHK_LOG_FILE} \ + > "${RCHK_LOG_FILE}" 2>&1 \ || (cat ${RCHK_LOG_FILE} && exit 1) cat ${RCHK_LOG_FILE} # the exceptions below are from R itself and not LightGBM: # https://github.com/kalibera/rchk/issues/22#issuecomment-656036156 - exit $( - cat ${RCHK_LOG_FILE} \ - | grep -v "in function strptime_internal" \ + exit "$( + grep "${RCHK_LOG_FILE}" -v "in function strptime_internal" \ | grep -v "in function RunGenCollect" \ | grep --count -E '\[PB\]|ERROR' - ) + )" fi # Test CRAN source .tar.gz in a directory that is not this repo or below it. @@ -194,9 +193,9 @@ elif [[ $R_BUILD_TYPE == "cran" ]]; then # git repo around. This is to protect against the use of relative paths # like ../../CMakeLists.txt that would only work if you are in the repo R_CMD_CHECK_DIR="${HOME}/tmp-r-cmd-check/" - mkdir -p ${R_CMD_CHECK_DIR} - mv ${PKG_TARBALL} ${R_CMD_CHECK_DIR} - cd ${R_CMD_CHECK_DIR} + mkdir -p "${R_CMD_CHECK_DIR}" + mv "${PKG_TARBALL}" "${R_CMD_CHECK_DIR}" + cd "${R_CMD_CHECK_DIR}" fi declare -i allowed_notes=0 diff --git a/.ci/test.sh b/.ci/test.sh index 78ba3020c1d1..238997e8d313 100755 --- a/.ci/test.sh +++ b/.ci/test.sh @@ -48,26 +48,30 @@ if [[ "${TASK}" == "r-package" ]] || [[ "${TASK}" == "r-rchk" ]]; then fi if [[ "$TASK" == "cpp-tests" ]]; then + cmake_args=( + -DBUILD_CPP_TEST=ON + -DUSE_OPENMP=OFF + -DUSE_DEBUG=ON + ) if [[ $METHOD == "with-sanitizers" ]]; then - extra_cmake_opts="-DUSE_SANITIZER=ON" + cmake_args+=("-DUSE_SANITIZER=ON") if [[ -n $SANITIZERS ]]; then - extra_cmake_opts="$extra_cmake_opts -DENABLED_SANITIZERS=$SANITIZERS" + cmake_args+=("-DENABLED_SANITIZERS=$SANITIZERS") fi - else - extra_cmake_opts="" fi - cmake -B build -S . -DBUILD_CPP_TEST=ON -DUSE_OPENMP=OFF -DUSE_DEBUG=ON $extra_cmake_opts + cmake -B build -S . "${cmake_args[@]}" cmake --build build --target testlightgbm -j4 || exit 1 ./testlightgbm || exit 1 exit 0 fi # including python=version[build=*cpython] to ensure that conda doesn't fall back to pypy -CONDA_PYTHON_REQUIREMENT="python=$PYTHON_VERSION[build=*cpython]" +CONDA_PYTHON_REQUIREMENT="python=${PYTHON_VERSION}[build=*cpython]" if [[ $TASK == "if-else" ]]; then - conda create -q -y -n $CONDA_ENV ${CONDA_PYTHON_REQUIREMENT} numpy - source activate $CONDA_ENV + conda create -q -y -n "${CONDA_ENV}" "${CONDA_PYTHON_REQUIREMENT}" numpy + # shellcheck disable=SC1091 + source activate "${CONDA_ENV}" cmake -B build -S . || exit 1 cmake --build build --target lightgbm -j4 || exit 1 cd "$BUILD_DIRECTORY/tests/cpp_tests" @@ -89,14 +93,14 @@ if [[ $TASK == "swig" ]]; then python ./.ci/check-dynamic-dependencies.py ./objdump.log || exit 1 fi if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp ./build/lightgbmlib.jar $BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_$OS_NAME.jar + cp ./build/lightgbmlib.jar "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lightgbmlib_${OS_NAME}.jar" fi exit 0 fi if [[ $TASK == "lint" ]]; then - conda create -q -y -n $CONDA_ENV \ - ${CONDA_PYTHON_REQUIREMENT} \ + conda create -q -y -n "${CONDA_ENV}" \ + "${CONDA_PYTHON_REQUIREMENT}" \ 'cmakelint>=1.4.3' \ 'cpplint>=1.6.0' \ 'matplotlib-base>=3.9.1' \ @@ -105,7 +109,8 @@ if [[ $TASK == "lint" ]]; then 'pyarrow-core>=17.0' \ 'scikit-learn>=1.5.2' \ 'r-lintr>=3.1.2' - source activate $CONDA_ENV + # shellcheck disable=SC1091 + source activate "${CONDA_ENV}" echo "Linting Python code" bash ./.ci/lint-python.sh || exit 1 echo "Linting R code" @@ -116,32 +121,31 @@ if [[ $TASK == "lint" ]]; then fi if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then - cd "${BUILD_DIRECTORY}/docs" conda env create \ - -n $CONDA_ENV \ - --file ./env.yml || exit 1 + -n "${CONDA_ENV}" \ + --file ./docs/env.yml || exit 1 conda install \ -q \ -y \ - -n $CONDA_ENV \ + -n "${CONDA_ENV}" \ 'doxygen>=1.10.0' \ 'rstcheck>=6.2.4' || exit 1 - source activate $CONDA_ENV + # shellcheck disable=SC1091 + source activate "${CONDA_ENV}" # check reStructuredText formatting - cd "${BUILD_DIRECTORY}/python-package" - rstcheck --report-level warning $(find . -type f -name "*.rst") || exit 1 - cd "${BUILD_DIRECTORY}/docs" - rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile $(find . -type f -name "*.rst") || exit 1 + find "${BUILD_DIRECTORY}/python-package" -type f -name "*.rst" \ + -exec rstcheck --report-level warning {} \+ || exit 1 + find "${BUILD_DIRECTORY}/docs" -type f -name "*.rst" \ + -exec rstcheck --report-level warning --ignore-directives=autoclass,autofunction,autosummary,doxygenfile {} \+ || exit 1 # build docs - make html || exit 1 + make -C docs html || exit 1 if [[ $TASK == "check-links" ]]; then # check docs for broken links pip install linkchecker - linkchecker --config=.linkcheckerrc ./_build/html/*.html || exit 1 + linkchecker --config=.linkcheckerrc ./docs/_build/html/*.html || exit 1 exit 0 fi # check the consistency of parameters' descriptions and other stuff - cd "${BUILD_DIRECTORY}" cp ./docs/Parameters.rst ./docs/Parameters-backup.rst cp ./src/io/config_auto.cpp ./src/io/config_auto-backup.cpp python ./.ci/parameter-generator.py || exit 1 @@ -151,20 +155,21 @@ if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then fi if [[ $PYTHON_VERSION == "3.7" ]]; then - CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py37.txt" + CONDA_REQUIREMENT_FILE="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py37.txt" elif [[ $PYTHON_VERSION == "3.8" ]]; then - CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py38.txt" + CONDA_REQUIREMENT_FILE="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core-py38.txt" else - CONDA_REQUIREMENT_FILES="--file ${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt" + CONDA_REQUIREMENT_FILE="${BUILD_DIRECTORY}/.ci/conda-envs/ci-core.txt" fi conda create \ -y \ - -n $CONDA_ENV \ - ${CONDA_REQUIREMENT_FILES} \ - ${CONDA_PYTHON_REQUIREMENT} \ + -n "${CONDA_ENV}" \ + --file "${CONDA_REQUIREMENT_FILE}" \ + "${CONDA_PYTHON_REQUIREMENT}" \ || exit 1 +# shellcheck disable=SC1091 source activate $CONDA_ENV cd "${BUILD_DIRECTORY}" @@ -172,9 +177,9 @@ cd "${BUILD_DIRECTORY}" if [[ $TASK == "sdist" ]]; then sh ./build-python.sh sdist || exit 1 sh .ci/check-python-dists.sh ./dist || exit 1 - pip install ./dist/lightgbm-$LGB_VER.tar.gz -v || exit 1 + pip install "./dist/lightgbm-${LGB_VER}.tar.gz" -v || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp ./dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1 + cp "./dist/lightgbm-${LGB_VER}.tar.gz" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1 fi pytest ./tests/python_package_test || exit 1 exit 0 @@ -183,7 +188,7 @@ elif [[ $TASK == "bdist" ]]; then sh ./build-python.sh bdist_wheel || exit 1 sh .ci/check-python-dists.sh ./dist || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp dist/lightgbm-$LGB_VER-py3-none-macosx*.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1 + cp "$(echo "dist/lightgbm-${LGB_VER}-py3-none-macosx"*.whl)" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1 fi else if [[ $ARCH == "x86_64" ]]; then @@ -199,10 +204,10 @@ elif [[ $TASK == "bdist" ]]; then ./dist/tmp.whl || exit 1 mv \ ./dist/tmp.whl \ - ./dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl || exit 1 + "./dist/lightgbm-${LGB_VER}-py3-none-${PLATFORM}.whl" || exit 1 sh .ci/check-python-dists.sh ./dist || exit 1 if [[ $PRODUCES_ARTIFACTS == "true" ]]; then - cp dist/lightgbm-$LGB_VER-py3-none-$PLATFORM.whl $BUILD_ARTIFACTSTAGINGDIRECTORY || exit 1 + cp "dist/lightgbm-${LGB_VER}-py3-none-${PLATFORM}.whl" "${BUILD_ARTIFACTSTAGINGDIRECTORY}" || exit 1 fi # Make sure we can do both CPU and GPU; see tests/python_package_test/test_dual.py export LIGHTGBM_TEST_DUAL_CPU_GPU=1 @@ -221,14 +226,14 @@ if [[ $TASK == "gpu" ]]; then pip install \ -v \ --config-settings=cmake.define.USE_GPU=ON \ - ./dist/lightgbm-$LGB_VER.tar.gz \ + "./dist/lightgbm-${LGB_VER}.tar.gz" \ || exit 1 pytest ./tests/python_package_test || exit 1 exit 0 elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --gpu || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install ./dist/lightgbm-$LGB_VER*.whl -v || exit 1 + pip install "$(echo "./dist/lightgbm-${LGB_VER}"*.whl)" -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then @@ -246,14 +251,14 @@ elif [[ $TASK == "cuda" ]]; then pip install \ -v \ --config-settings=cmake.define.USE_CUDA=ON \ - ./dist/lightgbm-$LGB_VER.tar.gz \ + "./dist/lightgbm-${LGB_VER}.tar.gz" \ || exit 1 pytest ./tests/python_package_test || exit 1 exit 0 elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --cuda || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install ./dist/lightgbm-$LGB_VER*.whl -v || exit 1 + pip install "$(echo "./dist/lightgbm-${LGB_VER}"*.whl)" -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then @@ -266,14 +271,14 @@ elif [[ $TASK == "mpi" ]]; then pip install \ -v \ --config-settings=cmake.define.USE_MPI=ON \ - ./dist/lightgbm-$LGB_VER.tar.gz \ + "./dist/lightgbm-${LGB_VER}.tar.gz" \ || exit 1 pytest ./tests/python_package_test || exit 1 exit 0 elif [[ $METHOD == "wheel" ]]; then sh ./build-python.sh bdist_wheel --mpi || exit 1 sh ./.ci/check-python-dists.sh ./dist || exit 1 - pip install ./dist/lightgbm-$LGB_VER*.whl -v || exit 1 + pip install "$(echo "./dist/lightgbm-${LGB_VER}"*.whl)" -v || exit 1 pytest ./tests || exit 1 exit 0 elif [[ $METHOD == "source" ]]; then @@ -291,13 +296,13 @@ pytest ./tests || exit 1 if [[ $TASK == "regular" ]]; then if [[ $PRODUCES_ARTIFACTS == "true" ]]; then if [[ $OS_NAME == "macos" ]]; then - cp ./lib_lightgbm.dylib $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.dylib + cp ./lib_lightgbm.dylib "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.dylib" else if [[ $COMPILER == "gcc" ]]; then objdump -T ./lib_lightgbm.so > ./objdump.log || exit 1 python ./.ci/check-dynamic-dependencies.py ./objdump.log || exit 1 fi - cp ./lib_lightgbm.so $BUILD_ARTIFACTSTAGINGDIRECTORY/lib_lightgbm.so + cp ./lib_lightgbm.so "${BUILD_ARTIFACTSTAGINGDIRECTORY}/lib_lightgbm.so" fi fi cd "$BUILD_DIRECTORY/examples/python-guide" @@ -311,10 +316,10 @@ matplotlib.use\(\"Agg\"\)\ 'h5py>=3.10' \ 'ipywidgets>=8.1.2' \ 'notebook>=7.1.2' - for f in *.py **/*.py; do python $f || exit 1; done # run all examples + for f in *.py **/*.py; do python "${f}" || exit 1; done # run all examples cd "$BUILD_DIRECTORY/examples/python-guide/notebooks" sed -i'.bak' 's/INTERACTIVE = False/assert False, \\"Interactive mode disabled\\"/' interactive_plot_example.ipynb - jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb || exit 1 # run all notebooks + jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace ./*.ipynb || exit 1 # run all notebooks # importing the library should succeed even if all optional dependencies are not present conda uninstall -n $CONDA_ENV --force --yes \