From 7cb86c9eb6ad0b9b74de4980e429b32a903c84f9 Mon Sep 17 00:00:00 2001 From: Torsten Long Date: Mon, 15 Jan 2024 09:33:43 +0100 Subject: [PATCH] Minor release 0.3.0 Signed-off-by: Torsten Long --- .github/workflows/test-lint-deploy.yml | 22 +++--- bin/mock_exe.sh | 44 +++++++++--- docs/usage.md | 41 ++++++++++- lib/command_global_config.bash | 12 ++-- lib/mock_management.bash | 8 ++- lib/shellmock.bash | 95 ++++++++++++++++++++++++-- tests/extended.bats | 91 ++++++++++++++++++++++++ tests/main.bats | 3 +- tests/misc.bats | 1 + 9 files changed, 288 insertions(+), 29 deletions(-) create mode 100644 tests/extended.bats diff --git a/.github/workflows/test-lint-deploy.yml b/.github/workflows/test-lint-deploy.yml index cc308d5..9370739 100644 --- a/.github/workflows/test-lint-deploy.yml +++ b/.github/workflows/test-lint-deploy.yml @@ -66,29 +66,35 @@ jobs: chmod +x shfmt sudo mv shfmt /usr/local/bin/shfmt - - name: Install prettier + - name: Install NVM run: | - curl -sSfL -o install_nvm.sh https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh + curl -sSfL -o install_nvm.sh "${BASE_URL}/${NVM_VERSION}/install.sh" echo "${NVMINSTALL_SHA256} install_nvm.sh" | sha256sum --check bash install_nvm.sh + env: + NVM_VERSION: v0.39.5 + NVMINSTALL_SHA256: 69da4f89f430cd5d6e591c2ccfa2e9e3ad55564ba60f651f00da85e04010c640 + BASE_URL: "https://raw.githubusercontent.com/nvm-sh/nvm" + + - name: Install prettier + run: | source "${HOME}/.nvm/nvm.sh" nvm install "${NODE_VERSION}" - dirname "$(which npm)" >> "${GITHUB_PATH}" npm install -g prettier + dirname "$(which prettier)" >> "${GITHUB_PATH}" env: NODE_VERSION: "20" - NVM_VERSION: v0.39.5 - NVMINSTALL_SHA256: 69da4f89f430cd5d6e591c2ccfa2e9e3ad55564ba60f651f00da85e04010c640 - name: Install mdslw run: | - curl -sSfL -o mdslw https://github.com/razziel89/mdslw/releases/download/${MDSLW_VERSION}/mdslw_x86_64-unknown-linux-musl + curl -sSfL -o mdslw "${BASE_URL}/${MDSLW_VERSION}/mdslw_x86_64-unknown-linux-musl" echo "${MDSLW_SHA256} mdslw" | sha256sum --check chmod +x mdslw sudo mv mdslw /usr/local/bin env: - MDSLW_VERSION: 0.5.5 - MDSLW_SHA256: 04d944b8e9596b82b7511c29bbf5ff10ab3ca707772d6a3164acb087a7e3f02e + MDSLW_VERSION: 0.6.1 + MDSLW_SHA256: 20814371ec7c1801b995aa2e506a71ea1a3b23a4affa977ca75b02ad3dd5c562 + BASE_URL: "https://github.com/razziel89/mdslw/releases/download" - uses: actions/checkout@v3 with: diff --git a/bin/mock_exe.sh b/bin/mock_exe.sh index 9af1c17..5f3289a 100755 --- a/bin/mock_exe.sh +++ b/bin/mock_exe.sh @@ -140,12 +140,37 @@ _match_spec() { done < <(base64 --decode <<< "${full_spec}") && wait $! } +# Check whether the given process is a bats process. A bats process is a bash +# process with a script located in bats's libexec directory. If we are not +# being executed by bats at all, we consider all processes to be non-bats. +_is_bats_process() { + local process="$1" + if [[ -z ${BATS_LIBEXEC-} ]]; then + # Not using bats, process cannot be a bats one. + return 1 + fi + + local cmd_w_args + mapfile -t -d $'\0' cmd_w_args < "/proc/${process}/cmdline" + # The first entry in cmd_w_args would be "bash" and the second one the bats + # script if our parent process were a bats process. Such a bats script is in + # bats's libexec directory. + [[ ${#cmd_w_args[@]} -ge 2 ]] \ + && [[ ${cmd_w_args[0]} == "bash" && + ${cmd_w_args[1]} == "${BATS_LIBEXEC%%/}/"* ]] +} + _kill_parent() { local parent="$1" - if [[ ${__SHELLMOCK__KILLPARENT-} -ne 1 ]]; then - return + # Do not kill the parent process if it is a bats process. If we did, bats + # would no longer be able to track the test. + if + [[ ${__SHELLMOCK__KILLPARENT-} -ne 1 ]] || _is_bats_process "${parent}" + then + return 0 fi + errecho "Killing parent process with information:" # In case the `ps` command fails (e.g. because we mock it), don't fail this # mock. @@ -156,8 +181,10 @@ _kill_parent() { find_matching_argspec() { # Find arg specs for this command and determine whether a specification # matches. - local cmd_b32="${1}" - shift + local outdir="${1}" + local cmd="${2}" + local cmd_b32="${3}" + shift 3 local env_var while read -r env_var; do @@ -172,7 +199,7 @@ find_matching_argspec() { | grep -x "MOCK_ARGSPEC_BASE64_${cmd_b32}_[0-9][0-9]*" | sort -u ) && wait $! - errecho "SHELLMOCK: unexpected call to '$0 $*'" + errecho "SHELLMOCK: unexpected call '${cmd} $*'" _kill_parent "${PPID}" return 1 } @@ -203,8 +230,9 @@ main() { env_var_check # Determine our name. This assumes that the first value in argv is the name of # the command. This is almost always so. - local cmd_b32 - cmd_b32="$(basename "$0" | base32 -w0 | tr "=" "_")" + local cmd cmd_b32 args + cmd="$(basename "$0")" + cmd_b32="$(base32 -w0 <<< "${cmd}" | tr "=" "_")" local outdir outdir="$(get_and_ensure_outdir "${cmd_b32}")" declare -g STDERR="${outdir}/stderr" @@ -214,7 +242,7 @@ main() { # associated information to stdout and exit with the associated exit code. If # it cannot be found, either exit with an error or kill the parent process. local cmd_spec - cmd_spec="$(find_matching_argspec "${cmd_b32}" "$@")" + cmd_spec="$(find_matching_argspec "${outdir}" "${cmd}" "${cmd_b32}" "$@")" provide_output "${cmd_spec}" return_with_code "${cmd_spec}" } diff --git a/docs/usage.md b/docs/usage.md index f773267..3d9e0da 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -418,6 +418,7 @@ There are currently the following settings: - `checkpath` - `killparent` +- `ensure-assertions` @@ -429,7 +430,7 @@ However, the tested code can still make modifications to its `PATH`, which could cause Shellmock's mocks to not be called. Thus, every call to the `shellmock` command will check whether the `PATH` variable has been changed since Shellmock was loaded via `load shellmock`. -An error will be written to standard error in case such a change is detected. +A warning will be written to standard error in case such a change is detected. The default value is 1. Use `shellmock global-config setval checkpath 0` to disable. @@ -467,6 +468,44 @@ The default value is 1. Use `shellmock global-config setval killparent 0` to disable. Use `shellmock global-config getval killparent` to retrieve the current setting. +#### ensure-assertions + +When creating and configuring a mock using Shellmock, you have to make sure to +assert that your configured mock has been used as expected via the `shellmock +assert` command. +Otherwise, you might not detect unexpected calls to your mock, or even the fact +that your mock has not even been used! +By default, Shellmock will fail a test that creates a mock without also running +corresponding assertions. +Take the following test as an example: + +```bash +@test "without asserting expectations" { + shellmock new curl + shellmock config curl 0 + # Not calling the curl mock here deliberately. +} +``` + +Without the `ensure-assertions` feature, the above test would succeed even +though the mock for `curl` had not even been called. +With the `ensure-assertions` feature enabled, which is the default, you will +receive an error output like this (some line breaks added to make it easier to +read): + +``` + ✗ without asserting expectations + `@test "without asserting expectations" {' failed + ERROR: expectations for mock curl have not been asserted. + Consider adding 'shellmock assert expectations curl' to + the following test: without asserting expectations +``` + +The default value is 1. +Use `shellmock global-config setval ensure-assertions 0` to disable. +Use `shellmock global-config getval ensure-assertions` to retrieve the current +setting. + ### calls diff --git a/lib/command_global_config.bash b/lib/command_global_config.bash index 3b567d9..424c85c 100644 --- a/lib/command_global_config.bash +++ b/lib/command_global_config.bash @@ -19,20 +19,24 @@ # The global-config command that can be used to get and set some global options. __shellmock__global-config() { __shellmock_internal_pathcheck + __shellmock_internal_trapcheck local subcmd="$1" local arg="$2" local val="${3-}" + local replacement="${arg//-/_}" + replacement="${replacement^^}" + case ${subcmd} in setval) case ${arg} in - checkpath | killparent) + checkpath | killparent | ensure-assertions) if [[ -z ${val-} ]]; then echo >&2 "Value argument to setval must not be empty." return 1 fi - local varname="__SHELLMOCK__${arg^^}" + local varname="__SHELLMOCK__${replacement}" declare -gx "${varname}=${val}" ;; *) @@ -43,8 +47,8 @@ __shellmock__global-config() { ;; getval) case ${arg} in - checkpath | killparent) - local varname="__SHELLMOCK__${arg^^}" + checkpath | killparent | ensure-assertions) + local varname="__SHELLMOCK__${replacement}" echo "${!varname}" ;; *) diff --git a/lib/mock_management.bash b/lib/mock_management.bash index 5f1f028..8f1f5f4 100644 --- a/lib/mock_management.bash +++ b/lib/mock_management.bash @@ -20,6 +20,7 @@ # or they could not be mocked with executables. __shellmock__new() { __shellmock_internal_pathcheck + __shellmock_internal_trapcheck local cmd="$1" @@ -64,6 +65,7 @@ __shellmock_assert_no_duplicate_argspecs() { # code, as well as the desired argspecs. __shellmock__config() { __shellmock_internal_pathcheck + __shellmock_internal_trapcheck # Fake output is read from stdin. local cmd="$1" @@ -152,6 +154,7 @@ __shellmock__config() { # Assert whether the configured mocks have been called as expected. __shellmock__assert() { __shellmock_internal_pathcheck + __shellmock_internal_trapcheck local assert_type="$1" local cmd="$2" @@ -164,6 +167,8 @@ __shellmock__assert() { return 1 fi + touch "${__SHELLMOCK_EXPECTATIONS_DIR}/${cmd}" + case "${assert_type}" in # Make sure that no calls were issued to the mock that we did not expect. By # default, the mock will kill its parent process if an unexpected call @@ -201,7 +206,7 @@ __shellmock__assert() { mapfile -t actual_argspecs < <( if [[ -d "${__SHELLMOCK_OUTPUT}/${cmd_b32}" ]]; then find "${__SHELLMOCK_OUTPUT}/${cmd_b32}" -mindepth 2 -type f \ - -name argspec -print0 | xargs -0 cat | sort -u + -name argspec -print0 | xargs -r -0 cat | sort -u fi ) && wait $! @@ -269,6 +274,7 @@ __shellmock_jsonify_array() { __shellmock__calls() { __shellmock_internal_pathcheck + __shellmock_internal_trapcheck local cmd="$1" local format="${2-"--plain"}" diff --git a/lib/shellmock.bash b/lib/shellmock.bash index d77076a..ac90f20 100644 --- a/lib/shellmock.bash +++ b/lib/shellmock.bash @@ -28,18 +28,21 @@ __shellmock_internal_init() { return 1 fi local has_bats=1 - if [[ -z ${BATS_RUN_TMPDIR} ]]; then + if [[ -z ${BATS_TEST_TMPDIR} ]]; then has_bats=0 fi # Modify PATH to permit injecting executables. declare -gx __SHELLMOCK_MOCKBIN - __SHELLMOCK_MOCKBIN="$(mktemp -d -p "${BATS_RUN_TMPDIR-${TMPDIR-/tmp}}")" - mkdir -p "${__SHELLMOCK_MOCKBIN}" + __SHELLMOCK_MOCKBIN="$(mktemp -d -p "${BATS_TEST_TMPDIR-${TMPDIR-/tmp}}")" export PATH="${__SHELLMOCK_MOCKBIN}:${PATH}" declare -gx __SHELLMOCK_OUTPUT - __SHELLMOCK_OUTPUT="$(mktemp -d -p "${BATS_RUN_TMPDIR-${TMPDIR-/tmp}}")" - mkdir -p "${__SHELLMOCK_OUTPUT}" + __SHELLMOCK_OUTPUT="$(mktemp -d -p "${BATS_TEST_TMPDIR-${TMPDIR-/tmp}}")" + + declare -gx __SHELLMOCK_EXPECTATIONS_DIR + __SHELLMOCK_EXPECTATIONS_DIR="$( + mktemp -d -p "${BATS_TEST_TMPDIR-${TMPDIR-/tmp}}" + )" if [[ ${has_bats} -eq 0 ]]; then echo >&2 "Running outside of bats, temporary directories will be kept." @@ -57,6 +60,32 @@ __shellmock_internal_init() { # By default, we kill a mock's parent process in case there is an unexpected # call. declare -gx __SHELLMOCK__KILLPARENT=1 + + # By default, we assert that all mocks have had their expectations asserted. + # We do so only when running inside of bats because, otherwise, we cannot + # easily determine the function at whose end we shall execute the trap. + declare -gx __SHELLMOCK__ENSURE_ASSERTIONS + declare -gx __SHELLMOCK_TRAP + local return_trap + return_trap="$(trap -p -- RETURN)" + if [[ ${has_bats} -eq 1 ]] && [[ -z ${return_trap} ]]; then + trap -- "__shellmock_internal_trap" RETURN + __SHELLMOCK__ENSURE_ASSERTIONS=1 + __SHELLMOCK_TRAP="$(trap -p -- RETURN)" + else + local reason + if [[ -n ${return_trap} ]]; then + reason="Detected existing trap '${return_trap}' for RETURN signal." + else + reason="Not using bats to run tests." + fi + echo >&2 "${reason}" \ + "Shellmock will be unable to automatically ensure that" \ + "expectations have been asserted. Make sure to assert expectations" \ + "manually for every test." + __SHELLMOCK__ENSURE_ASSERTIONS=0 + __SHELLMOCK_TRAP= + fi } __shellmock_internal_bash_version_check() { @@ -77,7 +106,7 @@ __shellmock_internal_bash_version_check() { } # Check whether PATH changed since shellmock has been initialised. If it has -# changed, the shellmock's mocks might no longer be used preferentially. +# changed, then shellmock's mocks might no longer be used preferentially. __shellmock_internal_pathcheck() { if [[ ${__SHELLMOCK__CHECKPATH} -eq 1 ]] \ && [[ ${PATH} != "${__SHELLMOCK_PATH}" ]]; then @@ -87,6 +116,60 @@ __shellmock_internal_pathcheck() { fi } +# Check whether the pre-configured trap changed since shellmock has been +# initialised. If it has changed, then shellmock's automatic assertion detection +# will likely not work anymore. +__shellmock_internal_trapcheck() { + if [[ ${__SHELLMOCK__ENSURE_ASSERTIONS} -eq 1 ]] \ + && [[ "$(trap -p -- RETURN)" != "${__SHELLMOCK_TRAP}" ]]; then + + echo >&2 "WARNING: RETURN trap has changed since loading shellmock," \ + "we will not be able to automatically ensure that expectations have" \ + "been asserted." + fi +} + +# This function is called as a trap (signal handler) for the RETURN signal. Due +# to the way bats works, it will be called pretty often at the end of many of +# bats's helper functions. Thus, we have to determine whether we are being +# called at the end of the actual test function because that is when we can test +# whether all expectations have been asserted. +__shellmock_internal_trap() { + # Do not perform any actions if the auto-assert feature has been deactivated. + # Do not perform any actions if we are not being called by the expected bats + # test function. + if + [[ ${__SHELLMOCK__ENSURE_ASSERTIONS} -eq 1 && + "$(caller 0)" == *" ${BATS_TEST_NAME-} "* ]] + then + local defined_cmds + readarray -d $'\n' -t defined_cmds < <( + find "${__SHELLMOCK_MOCKBIN}" -type f -print0 | xargs -r -0 -I{} basename {} + ) && wait $! + + local cmd has_err=0 + for cmd in "${defined_cmds[@]}"; do + if ! [[ -e "${__SHELLMOCK_EXPECTATIONS_DIR}/${cmd}" ]]; then + local cmd_quoted + cmd_quoted=$(printf "%q" "${cmd}") + echo >&2 "ERROR: expectations for mock ${cmd} have not been asserted." \ + "Consider adding 'shellmock assert expectations ${cmd_quoted}' to" \ + "the following test: ${BATS_TEST_DESCRIPTION-}" + has_err=1 + fi + done + # Exit the current process to indicate a test failure. This is how we can + # signal a test failure from within a return trap. When running tests, we + # only return, though, because bats would be unable to track the test if we + # were to call exit here. + if [[ ${__SHELLMOCK_TESTING_TRAP-0} -eq 1 ]]; then + return "${has_err}" + elif [[ ${has_err} -ne 0 ]]; then + exit "${has_err}" + fi + fi +} + # Main shellmock command. Subcommands can be added by creating a shell function # following a specific naming scheme. We avoid complex parsing of arguments with # a tool such as getopt or getopts. diff --git a/tests/extended.bats b/tests/extended.bats new file mode 100644 index 0000000..786d4bc --- /dev/null +++ b/tests/extended.bats @@ -0,0 +1,91 @@ +#!/usr/bin/env bats + +# Copyright (c) 2022 - for information on the respective copyright owner +# see the NOTICE file or the repository +# https://github.com/boschresearch/shellmock +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +setup_file() { + # Ensure we use the minimum required bats version and fail with a nice error + # if not. + bats_require_minimum_version 1.5.0 +} + +setup() { + load ../shellmock +} + +@test "auto-detection of forgotten assertions" { + # Clear shellmock's own RETURN trap. Instead, we call the trap manually for + # these tests. There is no way to tell bats that this test function is + # supposed to fail inside its RETURN trap. + trap -- - RETURN + shellmock global-config setval ensure-assertions 1 + shellmock new my_exe + shellmock config my_exe 0 + my_exe + local stderr + if stderr=$(__SHELLMOCK_TESTING_TRAP=1 __shellmock_internal_trap 2>&1); then + echo >&2 "Expected manual trap call to fail." + exit 1 + fi + [[ ${stderr} == *"ERROR: expectations for mock my_exe have not been asserted."* ]] +} + +@test "asserting expectations works" { + trap -- - RETURN + shellmock global-config setval ensure-assertions 1 + shellmock new my_exe + shellmock config my_exe 0 + my_exe + shellmock assert expectations my_exe + __SHELLMOCK_TESTING_TRAP=1 __shellmock_internal_trap +} + +@test "deactivating auto-detection of forgotten assertions does not error out" { + trap -- - RETURN + shellmock global-config setval ensure-assertions 0 + shellmock new my_exe + shellmock config my_exe 0 + my_exe + __SHELLMOCK_TESTING_TRAP=1 __shellmock_internal_trap +} + +@test "changing the RETURN trap is detected" { + # Clear the RETURN trap set by shellmock, triggering the warning. + trap -- - RETURN + stderr="$(__shellmock_internal_trapcheck 2>&1)" + echo "${stderr}" + [[ ${stderr} == "WARNING: RETURN trap has changed since loading shellmock"* ]] +} + +@test "an empty trap is overwritten" { + trap -- - RETURN + __shellmock_internal_init + [[ -n $(trap -p -- RETURN) ]] +} + +@test "an existing RETURN trap is kept" { + trap "echo some trap" RETURN + __shellmock_internal_init + [[ $(trap -p -- RETURN) == "trap -- 'echo some trap' RETURN" ]] + trap -- - RETURN +} + +@test "not setting a trap outside of bats" { + trap -- - RETURN + tmpdir="${BATS_TEST_TMPDIR}" + TMPDIR="${tmpdir}" BATS_TEST_TMPDIR="" __shellmock_internal_init + [[ -z $(trap -p -- RETURN) ]] +} diff --git a/tests/main.bats b/tests/main.bats index e18d317..cef6ef6 100644 --- a/tests/main.bats +++ b/tests/main.bats @@ -31,6 +31,7 @@ setup_file() { setup() { #shellcheck disable=SC2317 load ../shellmock + shellmock global-config setval ensure-assertions 0 } @test "we can mock an executable" { @@ -180,7 +181,7 @@ setup() { # stderr, which means we have to redirect to stdout to capture it. report="$(shellmock assert expectations my_exe 2>&1 || :)" - grep "^SHELLMOCK: unexpected call to '.*my_exe asdf'" <<< "${report}" + grep "^SHELLMOCK: unexpected call 'my_exe asdf'" <<< "${report}" grep -x "SHELLMOCK: got at least one unexpected call for .*my_exe\." <<< "${report}" } diff --git a/tests/misc.bats b/tests/misc.bats index 0ef0f8f..1fa774b 100644 --- a/tests/misc.bats +++ b/tests/misc.bats @@ -24,6 +24,7 @@ setup_file() { setup() { load ../shellmock + shellmock global-config setval ensure-assertions 0 } @test "incorrect argspecs fail the configuration" {