diff --git a/.editorconfig b/.editorconfig index 1d2259154e486..4b1382dd6de3f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -39,6 +39,15 @@ indent_size = 2 [*.{pl,pm,py,sh}] indent_size = 4 +# Picked up by shfmt, too. +[*.sh] +indent_style = space +indent_size = 4 +shell_variant = bash +binary_next_line = true +switch_case_indent = true +space_redirects = true + # Match gemfiles, set indent to spaces with width of two [Gemfile] indent_size = 2 diff --git a/.github/workflows/check-bash-scripts.yml b/.github/workflows/check-bash-scripts.yml new file mode 100644 index 0000000000000..351d148ea5bd4 --- /dev/null +++ b/.github/workflows/check-bash-scripts.yml @@ -0,0 +1,105 @@ +# This file was copied from check-nix-format.yml for the logic to apply +# this only to those files which are already conforming on the target branch. +name: Check that shell scripts are formatted and shellcheck-ed + +on: + pull_request_target: + # See the comment at the same location in ./nixpkgs-vet.yml + types: [opened, synchronize, reopened, edited] +permissions: + contents: read + +jobs: + nixos: + name: shell-check + runs-on: ubuntu-latest + if: "!contains(github.event.pull_request.title, '[skip treewide]')" + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + # pull_request_target checks out the base branch by default + ref: refs/pull/${{ github.event.pull_request.number }}/merge + # Fetches the merge commit and its parents + fetch-depth: 2 + - name: Checking out base branch + run: | + base=$(mktemp -d) + baseRev=$(git rev-parse HEAD^1) + git worktree add "$base" "$baseRev" + echo "baseRev=$baseRev" >> "$GITHUB_ENV" + echo "base=$base" >> "$GITHUB_ENV" + - name: Get Nixpkgs revision for shellcheck and shfmt + run: | + # Pin to a commit from nixpkgs-unstable to avoid e.g. building shellcheck or shfmt + # from staging. + # This should not be a URL, because it would allow PRs to run arbitrary code in CI! + rev=$(jq -r .rev ci/pinned-nixpkgs.json) + echo "url=https://github.com/NixOS/nixpkgs/archive/$rev.tar.gz" >> "$GITHUB_ENV" + - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 + with: + # explicitly enable sandbox + extra_nix_config: sandbox = true + nix_path: nixpkgs=${{ env.url }} + - name: Install shellcheck and shfmt + run: "nix-env -f '' -iAP shellcheck shfmt" + - name: Check that shell scripts are conforming + run: | + shfmtFiles=() + shellcheckFiles=() + + # TODO: Make this more parallel + + # Loop through all .sh files touched by the PR + while readarray -d '' -n 2 entry && (( ${#entry[@]} != 0 )); do + type=${entry[0]} + file=${entry[1]} + case $type in + A*) + source="" + dest=$file + ;; + M*) + source=$file + dest=$file + ;; + C*|R*) + source=$file + read -r -d '' dest + ;; + *) + echo "Ignoring file $file with type $type" + continue + esac + + # Ignore files that weren't already formatted + if [[ -n "$source" ]] && ! shfmt --diff ${{ env.base }}/"$source" >/dev/null; then + echo "Ignoring file $file because it's not formatted in the base commit" + elif ! shfmt --diff "$dest"; then + shfmtFiles+=("$dest") + fi + + # Ignore files that weren't already shellcheck-ed + if [[ -n "$source" ]] && ! shellcheck ${{ env.base }}/"$source" >/dev/null; then + echo "Ignoring file $file because it's not shellcheck-ed in the base commit" + elif ! shellcheck "$dest"; then + shellcheckFiles+=("$dest") + fi + done < <(git diff -z --name-status ${{ env.baseRev }} -- '*.sh') + + if (( "${#shfmtFiles[@]}" > 0 )); then + echo "Some new/changed shell scripts are not properly formatted" + echo "Please go to the Nixpkgs root directory, run \`nix-shell\`, then:" + echo "shfmt -w ${shfmtFiles[*]@Q}" + echo + fi + + if (( "${#shellcheckFiles[@]}" > 0 )); then + echo "Some new/changed shell scripts are not properly shellcheck-ed" + echo "Please go to the Nixpkgs root directory, run \`nix-shell\`, then:" + echo "shellcheck ${shellcheckFiles[*]@Q}" + echo + fi + + if (( "${#shfmtFiles[@]}" + "${#shellcheckFiles[@]}" > 0 )) + exit 1 + fi diff --git a/.github/workflows/check-shell.yml b/.github/workflows/check-shell.yml index 82bc43fb92941..0d3d90d803208 100644 --- a/.github/workflows/check-shell.yml +++ b/.github/workflows/check-shell.yml @@ -1,4 +1,4 @@ -name: "Check shell" +name: "Check shell.nix" on: pull_request_target: @@ -7,7 +7,7 @@ permissions: {} jobs: x86_64-linux: - name: shell-check-x86_64-linux + name: shell-nix-check-x86_64-linux runs-on: ubuntu-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -19,7 +19,7 @@ jobs: run: nix-build shell.nix aarch64-darwin: - name: shell-check-aarch64-darwin + name: shell-nix-check-aarch64-darwin runs-on: macos-latest steps: - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 0000000000000..fb2496fe6c135 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,8 @@ +# supports source=pkgs/stdenv/generic/setup.sh directives +external-sources=true +source-path=pkgs/stdenv/generic +shell=bash + +# Otherwise all xxxPhase=... will fail this rule: +# xxxPhase appears unused. Verify use (...). +disable=SC2034 diff --git a/pkgs/build-support/rust/hooks/cargo-build-hook.sh b/pkgs/build-support/rust/hooks/cargo-build-hook.sh index 749ebed8b01b7..83b1d4b7e0227 100644 --- a/pkgs/build-support/rust/hooks/cargo-build-hook.sh +++ b/pkgs/build-support/rust/hooks/cargo-build-hook.sh @@ -1,4 +1,7 @@ -# shellcheck shell=bash disable=SC2154,SC2164 +# shellcheck source=stdenv.sh +. /dev/null + +declare cargoBuildType cargoBuildHook() { echo "Executing cargoBuildHook" diff --git a/pkgs/build-support/rust/hooks/cargo-check-hook.sh b/pkgs/build-support/rust/hooks/cargo-check-hook.sh index 965e2b4514471..b0acfa00da148 100644 --- a/pkgs/build-support/rust/hooks/cargo-check-hook.sh +++ b/pkgs/build-support/rust/hooks/cargo-check-hook.sh @@ -1,4 +1,7 @@ -# shellcheck shell=bash disable=SC2154,SC2164 +# shellcheck source=stdenv.sh +. /dev/null + +declare cargoCheckType cargoCheckHook() { echo "Executing cargoCheckHook" diff --git a/pkgs/build-support/rust/hooks/cargo-nextest-hook.sh b/pkgs/build-support/rust/hooks/cargo-nextest-hook.sh index 4719d1bad8ae9..bc255329b3848 100644 --- a/pkgs/build-support/rust/hooks/cargo-nextest-hook.sh +++ b/pkgs/build-support/rust/hooks/cargo-nextest-hook.sh @@ -1,4 +1,7 @@ -# shellcheck shell=bash disable=SC2154,SC2164 +# shellcheck source=stdenv.sh +. /dev/null + +declare cargoCheckType cargoNextestHook() { echo "Executing cargoNextestHook" diff --git a/pkgs/build-support/rust/hooks/maturin-build-hook.sh b/pkgs/build-support/rust/hooks/maturin-build-hook.sh index 2eab2124e70be..edf3f4c0b652f 100644 --- a/pkgs/build-support/rust/hooks/maturin-build-hook.sh +++ b/pkgs/build-support/rust/hooks/maturin-build-hook.sh @@ -1,4 +1,5 @@ -# shellcheck shell=bash disable=SC2154,SC2164 +# shellcheck source=stdenv.sh +. /dev/null maturinBuildHook() { echo "Executing maturinBuildHook" diff --git a/pkgs/build-support/rust/hooks/rust-bindgen-hook.sh b/pkgs/build-support/rust/hooks/rust-bindgen-hook.sh index 53624b124f2b1..7d5422f985b59 100644 --- a/pkgs/build-support/rust/hooks/rust-bindgen-hook.sh +++ b/pkgs/build-support/rust/hooks/rust-bindgen-hook.sh @@ -1,3 +1,6 @@ +# shellcheck source=stdenv.sh +. /dev/null + # populates LIBCLANG_PATH and BINDGEN_EXTRA_CLANG_ARGS for rust projects that # depend on the bindgen crate diff --git a/pkgs/build-support/setup-hooks/multiple-outputs.sh b/pkgs/build-support/setup-hooks/multiple-outputs.sh index 45096d833b427..7fd6501d5e3f1 100644 --- a/pkgs/build-support/setup-hooks/multiple-outputs.sh +++ b/pkgs/build-support/setup-hooks/multiple-outputs.sh @@ -45,20 +45,30 @@ _overrideFirst() { # Setup chains of sane default values with easy overridability. # The variables are global to be usable anywhere during the build. # Typical usage in package is defining outputBin = "dev"; +# Each variable is declare'd to make shellcheck aware of it. +declare outputDev _overrideFirst outputDev "dev" "out" +declare outputBin _overrideFirst outputBin "bin" "out" +declare outputInclude _overrideFirst outputInclude "$outputDev" # so-libs are often among the main things to keep, and so go to $out +declare outputLib _overrideFirst outputLib "lib" "out" +declare outputDoc _overrideFirst outputDoc "doc" "out" +declare outputDevdoc _overrideFirst outputDevdoc "devdoc" REMOVE # documentation for developers # man and info pages are small and often useful to distribute with binaries +declare outputMan _overrideFirst outputMan "man" "$outputBin" +declare outputDevman _overrideFirst outputDevman "devman" "devdoc" "$outputMan" +declare outputInfo _overrideFirst outputInfo "info" "$outputBin" diff --git a/pkgs/by-name/ca/cargo-tauri/hook.sh b/pkgs/by-name/ca/cargo-tauri/hook.sh index adfa24b515b85..47dc5188f8eff 100644 --- a/pkgs/by-name/ca/cargo-tauri/hook.sh +++ b/pkgs/by-name/ca/cargo-tauri/hook.sh @@ -1,4 +1,5 @@ -# shellcheck shell=bash disable=SC2034,SC2154,SC2164 +# shellcheck source=stdenv.sh +. /dev/null # We replace these export dontCargoBuild=true diff --git a/pkgs/development/tools/build-managers/gn/setup-hook.sh b/pkgs/development/tools/build-managers/gn/setup-hook.sh index f459e979aa0fe..6e9dbf945231e 100644 --- a/pkgs/development/tools/build-managers/gn/setup-hook.sh +++ b/pkgs/development/tools/build-managers/gn/setup-hook.sh @@ -1,4 +1,5 @@ -# shellcheck shell=bash +# shellcheck source=stdenv.sh +. /dev/null gnConfigurePhase() { runHook preConfigure @@ -9,7 +10,6 @@ gnConfigurePhase() { echoCmd 'gn flags' "${flagsArray[@]}" gn gen out/Release --args="${flagsArray[*]}" - # shellcheck disable=SC2164 cd out/Release/ runHook postConfigure diff --git a/pkgs/kde/frameworks/extra-cmake-modules/ecm-hook.sh b/pkgs/kde/frameworks/extra-cmake-modules/ecm-hook.sh index c635816777f52..7eed38a6e8912 100644 --- a/pkgs/kde/frameworks/extra-cmake-modules/ecm-hook.sh +++ b/pkgs/kde/frameworks/extra-cmake-modules/ecm-hook.sh @@ -1,6 +1,5 @@ -# shellcheck shell=bash -# Variables we use here are set by the stdenv, no use complaining about them -# shellcheck disable=SC2164 +# shellcheck source=stdenv.sh +. /dev/null ecmEnvHook() { addToSearchPath XDG_DATA_DIRS "$1/share" diff --git a/pkgs/os-specific/bsd/setup-hook.sh b/pkgs/os-specific/bsd/setup-hook.sh index 08dfdd4b14b6f..bc084b6d54d17 100644 --- a/pkgs/os-specific/bsd/setup-hook.sh +++ b/pkgs/os-specific/bsd/setup-hook.sh @@ -1,4 +1,5 @@ -# shellcheck shell=bash disable=SC2154,SC2164 +# shellcheck source=stdenv.sh +. /dev/null # BSD makefiles should be able to detect this # but without they end up using gcc on Darwin stdenv diff --git a/pkgs/stdenv/generic/default.nix b/pkgs/stdenv/generic/default.nix index 2cbd2636a4630..204dbd8a60591 100644 --- a/pkgs/stdenv/generic/default.nix +++ b/pkgs/stdenv/generic/default.nix @@ -57,6 +57,7 @@ argsStdenv@{ name ? "stdenv", preHook ? "", initialPath let defaultNativeBuildInputs = extraNativeBuildInputs ++ + # Please match this list of .sh files with those in pkgs/stdenv/generic/stdenv.sh for shellcheck. [ ../../build-support/setup-hooks/audit-tmpdir.sh ../../build-support/setup-hooks/compress-man-pages.sh diff --git a/pkgs/stdenv/generic/setup.sh b/pkgs/stdenv/generic/setup.sh index 72fdbef981da8..2b56a44a1c7d8 100644 --- a/pkgs/stdenv/generic/setup.sh +++ b/pkgs/stdenv/generic/setup.sh @@ -1,4 +1,6 @@ -# shellcheck shell=bash +# shellcheck source=shellcheck.sh +. /dev/null + # shellcheck disable=1090,2154,2123,2034,2178,2048,2068,1091 __nixpkgs_setup_set_original=$- set -eu @@ -179,7 +181,6 @@ runHook() { return 0 } - # Run all hooks with the specified name, until one succeeds (returns a # zero exit code). If none succeed, return a non-zero exit code. runOneHook() { @@ -200,7 +201,6 @@ runOneHook() { return "$ret" } - # Run the named hook, either by calling the function with that name or # by evaluating the variable with that name. This allows convenient # setting of hooks both from Nix expressions (as attributes / @@ -226,7 +226,6 @@ _callImplicitHook() { # return trap is no good because it would affect nested returns. } - # A function wrapper around ‘eval’ that ensures that ‘return’ inside # hooks exits the hook, not the caller. Also will only pass args if # command can take them @@ -238,7 +237,6 @@ _eval() { fi } - ###################################################################### # Logging. @@ -253,7 +251,6 @@ echoCmd() { echo } - ###################################################################### # Error handling. @@ -270,7 +267,7 @@ exitHandler() { echo "system time for all child processes ${buildTimes[3]}" fi - if (( "$exitCode" != 0 )); then + if (("$exitCode" != 0)); then runHook failureHook # If the builder had a non-zero exit code and @@ -293,17 +290,14 @@ exitHandler() { trap "exitHandler" EXIT - ###################################################################### # Helper functions. - addToSearchPathWithCustomDelimiter() { local delimiter="$1" local varName="$2" local dir="$3" - if [[ -d "$dir" && "${!varName:+${delimiter}${!varName}${delimiter}}" \ - != *"${delimiter}${dir}${delimiter}"* ]]; then + if [[ -d "$dir" && "${!varName:+${delimiter}${!varName}${delimiter}}" != *"${delimiter}${dir}${delimiter}"* ]]; then export "${varName}=${!varName:+${!varName}${delimiter}}${dir}" fi } @@ -335,18 +329,21 @@ prependToVar() { case "${type#* }" in -A*) echo "prependToVar(): ERROR: trying to use prependToVar on an associative array." >&2 - return 1 ;; + return 1 + ;; -a*) - useArray=true ;; + useArray=true + ;; *) - useArray=false ;; + useArray=false + ;; esac fi shift if $useArray; then - nameref=( "$@" ${nameref+"${nameref[@]}"} ) + nameref=("$@" ${nameref+"${nameref[@]}"}) else nameref="$* ${nameref-}" fi @@ -368,18 +365,21 @@ appendToVar() { case "${type#* }" in -A*) echo "appendToVar(): ERROR: trying to use appendToVar on an associative array, use variable+=([\"X\"]=\"Y\") instead." >&2 - return 1 ;; + return 1 + ;; -a*) - useArray=true ;; + useArray=true + ;; *) - useArray=false ;; + useArray=false + ;; esac fi shift if $useArray; then - nameref=( ${nameref+"${nameref[@]}"} "$@" ) + nameref=(${nameref+"${nameref[@]}"} "$@") else nameref="${nameref-} $*" fi @@ -392,28 +392,31 @@ appendToVar() { concatTo() { local - set -o noglob - local -n targetref="$1"; shift + local -n targetref="$1" + shift local arg default name type for arg in "$@"; do IFS="=" read -r name default <<< "$arg" local -n nameref="$name" if [[ -z "${nameref[*]}" && -n "$default" ]]; then - targetref+=( "$default" ) + targetref+=("$default") elif type=$(declare -p "$name" 2> /dev/null); then case "${type#* }" in -A*) echo "concatTo(): ERROR: trying to use concatTo on an associative array." >&2 - return 1 ;; + return 1 + ;; -a*) - targetref+=( "${nameref[@]}" ) ;; + targetref+=("${nameref[@]}") + ;; *) if [[ "$name" = *"Array" ]]; then nixErrorLog "concatTo(): $name is not declared as array, treating as a singleton. This will become an error in future" # Reproduces https://github.com/NixOS/nixpkgs/pull/318614/files#diff-7c7ca80928136cfc73a02d5b28350bd900e331d6d304857053ffc9f7beaad576L359 - targetref+=( ${nameref+"${nameref[@]}"} ) + targetref+=(${nameref+"${nameref[@]}"}) else # shellcheck disable=SC2206 - targetref+=( ${nameref-} ) + targetref+=(${nameref-}) fi ;; esac @@ -441,12 +444,15 @@ concatStringsSep() { case "${type#* }" in -A*) echo "concatStringsSep(): ERROR: trying to use concatStringsSep on an associative array." >&2 - return 1 ;; + return 1 + ;; -a*) local IFS="$sep" - echo -n "${nameref[*]}" ;; + echo -n "${nameref[*]}" + ;; *) - echo -n "${nameref// /"${sep}"}" ;; + echo -n "${nameref// /"${sep}"}" + ;; esac fi } @@ -487,15 +493,15 @@ isMachO() { # https://opensource.apple.com/source/lldb/lldb-310.2.36/examples/python/mach_o.py.auto.html if [[ "$magic" = $(echo -ne "\xfe\xed\xfa\xcf") || "$magic" = $(echo -ne "\xcf\xfa\xed\xfe") ]]; then # MH_MAGIC_64 || MH_CIGAM_64 - return 0; + return 0 elif [[ "$magic" = $(echo -ne "\xfe\xed\xfa\xce") || "$magic" = $(echo -ne "\xce\xfa\xed\xfe") ]]; then # MH_MAGIC || MH_CIGAM - return 0; + return 0 elif [[ "$magic" = $(echo -ne "\xca\xfe\xba\xbe") || "$magic" = $(echo -ne "\xbe\xba\xfe\xca") ]]; then # FAT_MAGIC || FAT_CIGAM - return 0; + return 0 else - return 1; + return 1 fi } @@ -513,12 +519,12 @@ isScript() { # printf unfortunately will print a trailing newline regardless printLines() { - (( "$#" > 0 )) || return 0 + (("$#" > 0)) || return 0 printf '%s\n' "$@" } printWords() { - (( "$#" > 0 )) || return 0 + (("$#" > 0)) || return 0 printf '%s ' "$@" } @@ -533,7 +539,6 @@ if [[ -n $__structuredAttrs ]]; then done fi - # Set a fallback default value for SOURCE_DATE_EPOCH, used by some build tools # to provide a deterministic substitute for the "current" time. Note that # 315532800 = 1980-01-01 12:00:00. We use this date because python's wheel @@ -542,13 +547,11 @@ fi export SOURCE_DATE_EPOCH : "${SOURCE_DATE_EPOCH:=315532800}" - # Wildcard expansions that don't match should expand to an empty list. # This ensures that, for instance, "for i in *; do ...; done" does the # right thing. shopt -s nullglob - # Set up the initial path. PATH= HOST_PATH= @@ -569,22 +572,22 @@ unset i nixWarnLog "initial path: $PATH" # Check that the pre-hook initialised SHELL. -if [ -z "${SHELL:-}" ]; then echo "SHELL not set"; exit 1; fi +if [ -z "${SHELL:-}" ]; then + echo "SHELL not set" + exit 1 +fi BASH="$SHELL" export CONFIG_SHELL="$SHELL" - # Execute the pre-hook. if [ -z "${shell:-}" ]; then export shell="$SHELL"; fi runHook preHook - # Allow the caller to augment buildInputs (it's not always possible to # do this before the call to setup.sh, since the PATH is empty at that # point; here we have a basic Unix environment). runHook addInputsHook - # Package accumulators declare -a pkgsBuildBuild pkgsBuildHost pkgsBuildTarget @@ -597,7 +600,6 @@ declare -a pkgTargetAccumVars=(pkgsTargetTarget) declare -a pkgAccumVarVars=(pkgBuildAccumVars pkgHostAccumVars pkgTargetAccumVars) - # Hooks declare -a envBuildBuildHooks envBuildHostHooks envBuildTargetHooks @@ -624,7 +626,6 @@ addEnvHooks() { done } - # Propagated dep files declare -a propagatedBuildDepFiles=( @@ -648,7 +649,6 @@ declare -a propagatedDepFilesVars=( # Platform offsets: build = -1, host = 0, target = 1 declare -a allPlatOffsets=(-1 0 1) - # Mutually-recursively find all build inputs. See the dependency section of the # stdenv chapter of the Nixpkgs manual for the specification this algorithm # implements. @@ -658,7 +658,7 @@ findInputs() { local -r targetOffset="$3" # Sanity check - (( hostOffset <= targetOffset )) || exit 1 + ((hostOffset <= targetOffset)) || exit 1 # shellcheck disable=SC1087 local varVar="${pkgAccumVarVars[hostOffset + 1]}" @@ -692,7 +692,7 @@ findInputs() { function mapOffset() { local -r inputOffset="$1" local -n outputOffset="$2" - if (( inputOffset <= 0 )); then + if ((inputOffset <= 0)); then outputOffset=$((inputOffset + hostOffset)) else outputOffset=$((inputOffset - 1 + targetOffset)) @@ -713,13 +713,13 @@ findInputs() { # Ensure we're in bounds relative to the package currently # being built. - (( -1 <= hostOffsetNext && hostOffsetNext <= 1 )) || continue + ((-1 <= hostOffsetNext && hostOffsetNext <= 1)) || continue # Target offset relative to the *host* offset of the package # whose immediate dependencies we are currently exploring. local relTargetOffset for relTargetOffset in "${allPlatOffsets[@]}"; do - (( "$relHostOffset" <= "$relTargetOffset" )) || continue + (("$relHostOffset" <= "$relTargetOffset")) || continue local fileRef="${files}[$relTargetOffset - $relHostOffset]" local file="${!fileRef}" @@ -732,7 +732,7 @@ findInputs() { # Once again, ensure we're in bounds relative to the # package currently being built. - (( -1 <= hostOffsetNext && hostOffsetNext <= 1 )) || continue + ((-1 <= hostOffsetNext && hostOffsetNext <= 1)) || continue [[ -f "$pkg/nix-support/$file" ]] || continue @@ -761,26 +761,26 @@ for pkg in ${depsBuildBuild[@]} ${depsBuildBuildPropagated[@]}; do findInputs "$pkg" -1 -1 done for pkg in ${nativeBuildInputs[@]} ${propagatedNativeBuildInputs[@]}; do - findInputs "$pkg" -1 0 + findInputs "$pkg" -1 0 done for pkg in ${depsBuildTarget[@]} ${depsBuildTargetPropagated[@]}; do - findInputs "$pkg" -1 1 + findInputs "$pkg" -1 1 done for pkg in ${depsHostHost[@]} ${depsHostHostPropagated[@]}; do - findInputs "$pkg" 0 0 + findInputs "$pkg" 0 0 done -for pkg in ${buildInputs[@]} ${propagatedBuildInputs[@]} ; do - findInputs "$pkg" 0 1 +for pkg in ${buildInputs[@]} ${propagatedBuildInputs[@]}; do + findInputs "$pkg" 0 1 done for pkg in ${depsTargetTarget[@]} ${depsTargetTargetPropagated[@]}; do - findInputs "$pkg" 1 1 + findInputs "$pkg" 1 1 done # Default inputs must be processed last for pkg in ${defaultNativeBuildInputs[@]}; do - findInputs "$pkg" -1 0 + findInputs "$pkg" -1 0 done for pkg in ${defaultBuildInputs[@]}; do - findInputs "$pkg" 0 1 + findInputs "$pkg" 0 1 done # Add package to the future PATH and run setup hooks @@ -790,7 +790,7 @@ activatePackage() { local -r targetOffset="$3" # Sanity check - (( hostOffset <= targetOffset )) || exit 1 + ((hostOffset <= targetOffset)) || exit 1 if [ -f "$pkg" ]; then nixTalkativeLog "sourcing setup hook '$pkg'" @@ -808,7 +808,7 @@ activatePackage() { addToSearchPath _PATH "$pkg/bin" fi - if (( hostOffset <= -1 )); then + if ((hostOffset <= -1)); then addToSearchPath _XDG_DATA_DIRS "$pkg/share" fi @@ -829,7 +829,7 @@ _activatePkgs() { for hostOffset in "${allPlatOffsets[@]}"; do local pkgsVar="${pkgAccumVarVars[hostOffset + 1]}" for targetOffset in "${allPlatOffsets[@]}"; do - (( hostOffset <= targetOffset )) || continue + ((hostOffset <= targetOffset)) || continue local pkgsRef="${pkgsVar}[$targetOffset - $hostOffset]" local pkgsSlice="${!pkgsRef}[@]" for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; do @@ -858,7 +858,7 @@ _addToEnv() { local hookVar="${pkgHookVarVars[depHostOffset + 1]}" local pkgsVar="${pkgAccumVarVars[depHostOffset + 1]}" for depTargetOffset in "${allPlatOffsets[@]}"; do - (( depHostOffset <= depTargetOffset )) || continue + ((depHostOffset <= depTargetOffset)) || continue local hookRef="${hookVar}[$depTargetOffset - $depHostOffset]" if [[ -z "${strictDeps-}" ]]; then @@ -875,8 +875,7 @@ _addToEnv() { "${pkgsBuildTarget[@]}" \ "${pkgsHostHost[@]}" \ "${pkgsHostTarget[@]}" \ - "${pkgsTargetTarget[@]}" - do + "${pkgsTargetTarget[@]}"; do if [[ "$visitedPkgs" = *"$pkg"* ]]; then continue fi @@ -897,35 +896,30 @@ _addToEnv() { # Run the package-specific hooks set by the setup-hook scripts. _addToEnv - # Unset setup-specific declared variables unset allPlatOffsets unset pkgBuildAccumVars pkgHostAccumVars pkgTargetAccumVars pkgAccumVarVars unset pkgBuildHookVars pkgHostHookVars pkgTargetHookVars pkgHookVarVars unset propagatedDepFilesVars - _addRpathPrefix "$out" - # Set the TZ (timezone) environment variable, otherwise commands like # `date' will complain (e.g., `Tue Mar 9 10:01:47 Local time zone must # be set--see zic manual page 2004'). export TZ=UTC - # Set the prefix. This is generally $out, but it can be overriden, # for instance if we just want to perform a test build/install to a # temporary location and write a build report to $out. if [ -z "${prefix:-}" ]; then - prefix="$out"; + prefix="$out" fi if [ "${useTempPrefix:-}" = 1 ]; then - prefix="$NIX_BUILD_TOP/tmp_prefix"; + prefix="$NIX_BUILD_TOP/tmp_prefix" fi - PATH="${_PATH-}${_PATH:+${PATH:+:}}$PATH" HOST_PATH="${_HOST_PATH-}${_HOST_PATH:+${HOST_PATH:+:}}$HOST_PATH" export XDG_DATA_DIRS="${_XDG_DATA_DIRS-}${_XDG_DATA_DIRS:+${XDG_DATA_DIRS:+:}}${XDG_DATA_DIRS-}" @@ -938,30 +932,27 @@ unset _PATH unset _HOST_PATH unset _XDG_DATA_DIRS - # Normalize the NIX_BUILD_CORES variable. The value might be 0, which # means that we're supposed to try and auto-detect the number of # available CPU cores at run-time. NIX_BUILD_CORES="${NIX_BUILD_CORES:-1}" if ((NIX_BUILD_CORES <= 0)); then - guess=$(nproc 2>/dev/null || true) - ((NIX_BUILD_CORES = guess <= 0 ? 1 : guess)) + guess=$(nproc 2> /dev/null || true) + ((NIX_BUILD_CORES = guess <= 0 ? 1 : guess)) fi export NIX_BUILD_CORES - # Prevent SSL libraries from using certificates in /etc/ssl, unless set explicitly. # Leave it in impure shells for convenience. if [[ -z "${NIX_SSL_CERT_FILE:-}" && "${IN_NIX_SHELL:-}" != "impure" ]]; then - export NIX_SSL_CERT_FILE=/no-cert-file.crt + export NIX_SSL_CERT_FILE=/no-cert-file.crt fi # Another variant left for compatibility. if [[ -z "${SSL_CERT_FILE:-}" && "${IN_NIX_SHELL:-}" != "impure" ]]; then - export SSL_CERT_FILE=/no-cert-file.crt + export SSL_CERT_FILE=/no-cert-file.crt fi - ###################################################################### # Textual substitution functions. @@ -973,7 +964,7 @@ substituteStream() { local description=$2 shift 2 - while (( "$#" )); do + while (("$#")); do local replace_mode="$1" case "$1" in --replace) @@ -985,7 +976,7 @@ substituteStream() { fi replace_mode='--replace-warn' ;& - --replace-quiet|--replace-warn|--replace-fail) + --replace-quiet | --replace-warn | --replace-fail) pattern="$2" replacement="$3" shift 3 @@ -1042,7 +1033,7 @@ substituteStream() { # fail loudly if provided with a binary (containing null bytes) consumeEntire() { # read returns non-0 on EOF, so we want read to fail - if IFS='' read -r -d '' "$1" ; then + if IFS='' read -r -d '' "$1"; then echo "consumeEntire(): ERROR: Input null bytes, won't process" >&2 return 1 fi @@ -1113,18 +1104,15 @@ substituteAll() { substitute "$input" "$output" "${args[@]}" } - substituteAllInPlace() { local fileName="$1" shift substituteAll "$fileName" "$fileName" "$@" } - ###################################################################### # What follows is the generic builder. - # This function is useful for debugging broken Nix builds. It dumps # all environment variables to a file `env-vars' in the build # directory. If the build fails and the `-K' option is used, you can @@ -1136,13 +1124,12 @@ dumpVars() { # so first we create the file and then write to it # See https://github.com/NixOS/nixpkgs/issues/335016 { - install -m 0600 /dev/null "$NIX_BUILD_TOP/env-vars" && - export 2>/dev/null >| "$NIX_BUILD_TOP/env-vars" + install -m 0600 /dev/null "$NIX_BUILD_TOP/env-vars" \ + && export 2> /dev/null >| "$NIX_BUILD_TOP/env-vars" } || true fi } - # Utility function: echo the base name of the given path, with the # prefix `HASH-' removed, if present. stripHash() { @@ -1156,10 +1143,9 @@ stripHash() { else echo "$strippedName" fi - if (( casematchOpt )); then shopt -s nocasematch; fi + if ((casematchOpt)); then shopt -s nocasematch; fi } - recordPropagatedDependencies() { # Propagate dependencies into the development output. declare -ra flatVars=( @@ -1192,7 +1178,6 @@ recordPropagatedDependencies() { done } - unpackCmdHooks+=(_defaultUnpack) _defaultUnpack() { local fn="$1" @@ -1226,7 +1211,10 @@ _defaultUnpack() { # disregard the error code from the xz invocation. Otherwise, # it can happen that tar exits earlier, causing xz to fail # from a SIGPIPE. - (XZ_OPT="--threads=$NIX_BUILD_CORES" xz -d < "$fn"; true) | tar xf - --mode=+w --warning=no-timestamp + ( + XZ_OPT="--threads=$NIX_BUILD_CORES" xz -d < "$fn" + true + ) | tar xf - --mode=+w --warning=no-timestamp ;; *.tar | *.tar.* | *.tgz | *.tbz2 | *.tbz) # GNU tar can automatically select the decompression method @@ -1241,7 +1229,6 @@ _defaultUnpack() { fi } - unpackFile() { curSrc="$1" echo "unpacking source archive $curSrc" @@ -1251,7 +1238,6 @@ unpackFile() { fi } - unpackPhase() { runHook preUnpack @@ -1294,8 +1280,7 @@ unpackPhase() { for i in *; do if [ -d "$i" ]; then case $dirsBefore in - *\ $i\ *) - ;; + *\ $i\ *) ;; *) if [ -n "$sourceRoot" ]; then echo "unpacker produced multiple directories" @@ -1325,7 +1310,6 @@ unpackPhase() { runHook postUnpack } - patchPhase() { runHook prePatch @@ -1360,7 +1344,6 @@ patchPhase() { runHook postPatch } - fixLibtool() { local search_path for flag in $NIX_LDFLAGS; do @@ -1376,7 +1359,6 @@ fixLibtool() { -e 's^eval sys_lib_.+search_path=.*^^' } - configurePhase() { runHook preConfigure @@ -1401,13 +1383,13 @@ configurePhase() { # autoreconf'ing themselves CONFIGURE_MTIME_REFERENCE=$(mktemp configure.mtime.reference.XXXXXX) find . \ - -executable \ - -type f \ - -name configure \ - -exec grep -l 'GNU Libtool is free software; you can redistribute it and/or modify' {} \; \ - -exec touch -r {} "$CONFIGURE_MTIME_REFERENCE" \; \ - -exec sed -i s_/usr/bin/file_file_g {} \; \ - -exec touch -r "$CONFIGURE_MTIME_REFERENCE" {} \; + -executable \ + -type f \ + -name configure \ + -exec grep -l 'GNU Libtool is free software; you can redistribute it and/or modify' {} \; \ + -exec touch -r {} "$CONFIGURE_MTIME_REFERENCE" \; \ + -exec sed -i s_/usr/bin/file_file_g {} \; \ + -exec touch -r "$CONFIGURE_MTIME_REFERENCE" {} \; rm -f "$CONFIGURE_MTIME_REFERENCE" fi @@ -1450,11 +1432,10 @@ configurePhase() { runHook postConfigure } - buildPhase() { runHook preBuild - if [[ -z "${makeFlags-}" && -z "${makefile:-}" && ! ( -e Makefile || -e makefile || -e GNUmakefile ) ]]; then + if [[ -z "${makeFlags-}" && -z "${makefile:-}" && ! (-e Makefile || -e makefile || -e GNUmakefile) ]]; then echo "no Makefile or custom buildPhase, doing nothing" else foundMakefile=1 @@ -1474,7 +1455,6 @@ buildPhase() { runHook postBuild } - checkPhase() { runHook preCheck @@ -1486,9 +1466,9 @@ checkPhase() { if [[ -z "${checkTarget:-}" ]]; then #TODO(@oxij): should flagsArray influence make -n? - if make -n ${makefile:+-f $makefile} check >/dev/null 2>&1; then + if make -n ${makefile:+-f $makefile} check > /dev/null 2>&1; then checkTarget="check" - elif make -n ${makefile:+-f $makefile} test >/dev/null 2>&1; then + elif make -n ${makefile:+-f $makefile} test > /dev/null 2>&1; then checkTarget="test" fi fi @@ -1514,12 +1494,11 @@ checkPhase() { runHook postCheck } - installPhase() { runHook preInstall # Dont reuse 'foundMakefile' set in buildPhase, a makefile may have been created in buildPhase - if [[ -z "${makeFlags-}" && -z "${makefile:-}" && ! ( -e Makefile || -e makefile || -e GNUmakefile ) ]]; then + if [[ -z "${makeFlags-}" && -z "${makefile:-}" && ! (-e Makefile || -e makefile || -e GNUmakefile) ]]; then echo "no Makefile or custom installPhase, doing nothing" runHook postInstall return @@ -1546,7 +1525,6 @@ installPhase() { runHook postInstall } - # The fixup phase performs generic, package-independent stuff, like # stripping binaries, running patchelf and setting # propagated-build-inputs. @@ -1566,7 +1544,6 @@ fixupPhase() { prefix="${!output}" runHook fixupOutput done - # record propagated dependencies & setup hook into the development output. recordPropagatedDependencies @@ -1602,7 +1579,6 @@ fixupPhase() { runHook postFixup } - installCheckPhase() { runHook preInstallCheck @@ -1610,7 +1586,7 @@ installCheckPhase() { echo "no Makefile or custom installCheckPhase, doing nothing" #TODO(@oxij): should flagsArray influence make -n? elif [[ -z "${installCheckTarget:-}" ]] \ - && ! make -n ${makefile:+-f $makefile} "${installCheckTarget:-installcheck}" >/dev/null 2>&1; then + && ! make -n ${makefile:+-f $makefile} "${installCheckTarget:-installcheck}" > /dev/null 2>&1; then echo "no installcheck target in ${makefile:-Makefile}, doing nothing" else # Old bash empty array hack @@ -1621,7 +1597,7 @@ installCheckPhase() { ) concatTo flagsArray makeFlags makeFlagsArray \ - installCheckFlags installCheckFlagsArray installCheckTarget=installcheck + installCheckFlags installCheckFlagsArray installCheckTarget=installcheck echoCmd 'installcheck flags' "${flagsArray[@]}" make ${makefile:+-f $makefile} "${flagsArray[@]}" @@ -1631,7 +1607,6 @@ installCheckPhase() { runHook postInstallCheck } - distPhase() { runHook preDist @@ -1653,7 +1628,6 @@ distPhase() { runHook postDist } - showPhaseHeader() { local phase="$1" echo "Running phase: $phase" @@ -1666,24 +1640,22 @@ showPhaseHeader() { printf "@nix { \"action\": \"setPhase\", \"phase\": \"%s\" }\n" "$phase" >&"$NIX_LOG_FD" } - showPhaseFooter() { local phase="$1" local startTime="$2" local endTime="$3" - local delta=$(( endTime - startTime )) - (( delta < 30 )) && return + local delta=$((endTime - startTime)) + ((delta < 30)) && return - local H=$((delta/3600)) - local M=$((delta%3600/60)) - local S=$((delta%60)) + local H=$((delta / 3600)) + local M=$((delta % 3600 / 60)) + local S=$((delta % 60)) echo -n "$phase completed in " - (( H > 0 )) && echo -n "$H hours " - (( M > 0 )) && echo -n "$M minutes " + ((H > 0)) && echo -n "$H hours " + ((M > 0)) && echo -n "$M minutes " echo "$S seconds" } - runPhase() { local curPhase="$*" if [[ "$curPhase" = unpackPhase && -n "${dontUnpack:-}" ]]; then return; fi @@ -1718,7 +1690,6 @@ runPhase() { fi } - genericBuild() { # variable used by our gzip wrapper to add -n. # gzip is in common-path.nix and is added to nix-shell but we only want to change its behaviour in nix builds. do not move to a setupHook in gzip. @@ -1737,7 +1708,7 @@ genericBuild() { phases="${prePhases[*]:-} unpackPhase patchPhase ${preConfigurePhases[*]:-} \ configurePhase ${preBuildPhases[*]:-} buildPhase checkPhase \ ${preInstallPhases[*]:-} installPhase ${preFixupPhases[*]:-} fixupPhase installCheckPhase \ - ${preDistPhases[*]:-} distPhase ${postPhases[*]:-}"; + ${preDistPhases[*]:-} distPhase ${postPhases[*]:-}" fi # The use of ${phases[*]} gives the correct behavior both with and @@ -1749,20 +1720,16 @@ genericBuild() { done } - # Execute the post-hooks. runHook postHook - # Execute the global user hook (defined through the Nixpkgs # configuration option ‘stdenv.userHook’). This can be used to set # global compiler optimisation flags, for instance. runHook userHook - dumpVars - # Restore the original options for nix-shell [[ $__nixpkgs_setup_set_original == *e* ]] || set +e [[ $__nixpkgs_setup_set_original == *u* ]] || set +u diff --git a/pkgs/stdenv/generic/stdenv.sh b/pkgs/stdenv/generic/stdenv.sh new file mode 100644 index 0000000000000..6845f14c69401 --- /dev/null +++ b/pkgs/stdenv/generic/stdenv.sh @@ -0,0 +1,63 @@ +# This file is only used for shellcheck, not actually sourced via bash. +# The name of the file makes it, so that it can be included via directive like this: +# # shellcheck source=stdenv.sh +# +# Shellcheck has rule SC2154 "var is referenced but not assigned": +# https://www.shellcheck.net/wiki/SC2154 +# Naturally, this rule is hard to validate for all our setup hooks, because they +# depend on variables: +# - set in stdenv/generic/setup.sh +# - set in default setup-hooks defined in stdenv/generic/default.nix (.../build-support/setup-hooks/...) +# - passed as derivation arguments +# +# We can tell shellcheck about "dependencies" indirectly, with the following construct: +# # shellcheck source=setup.sh +# . /dev/null +# This will source /dev/null, so nothing. The shellcheck meta command will replace this +# with sourcing the generic/setup.sh file. Thus, all variables defined in stdenv will be +# known to shellcheck. + +# All paths are relative to pkgs/stdenv/generic, thanks to .shellcheckrc. +# shellcheck source=setup.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/audit-tmpdir.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/compress-man-pages.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/make-symlinks-relative.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/move-docs.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/move-lib64.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/move-sbin.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/move-systemd-user-units.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/multiple-outputs.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/patch-shebangs.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/prune-libtool-files.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/reproducible-builds.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/set-source-date-epoch-to-latest.sh +. /dev/null +# shellcheck source=../../build-support/setup-hooks/strip.sh +. /dev/null + +# We can't do the same for derivation arguments. We are left with the following choices: +# - Putting shellcheck disable comments for that rule on each reference to a derivation argument. +# - Disable the rule in a broader scope (per file / globally). +# - "declare" the variables somewhere to have them be picked up by shellcheck. +# +# The last approach has the advantage that we don't need to miss out on the benefits of the rule, i.e. +# preventing spelling mistakes. More so, we get that check for those known derivation arguments as well. +# The best place to put those declaration is right in the file they are invented. Sometimes this might +# not be possible, in this case they can be redefined below. +# +# TLDR: The following code is **not** actually run by bash, only loaded by shellcheck. + +# Note: Keeping this list nicely sorted will make it easier to prevent duplicates. +# ... nothing here, yet ... diff --git a/shell.nix b/shell.nix index b8a9fe7df19fb..12f9a5df4a2f9 100644 --- a/shell.nix +++ b/shell.nix @@ -26,5 +26,8 @@ pkgs.mkShellNoCC { # Helper to review Nixpkgs PRs # See CONTRIBUTING.md nixpkgs-review + # Linter and Formatter for shell scripts + shellcheck + shfmt ]; }