diff --git a/docs/source/reference-testing.rst b/docs/source/reference-testing.rst index 3b061a32d..8ad62669c 100644 --- a/docs/source/reference-testing.rst +++ b/docs/source/reference-testing.rst @@ -231,8 +231,12 @@ ExceptionGroup helpers .. autoclass:: RaisesGroup :members: + .. autoattribute:: fail_reason + .. autoclass:: Matcher :members: + .. autoattribute:: fail_reason + .. autoclass:: trio.testing._raises_group._ExceptionInfo :members: diff --git a/newsfragments/3145.feature.rst b/newsfragments/3145.feature.rst new file mode 100644 index 000000000..e04959191 --- /dev/null +++ b/newsfragments/3145.feature.rst @@ -0,0 +1 @@ +If `trio.testing.RaisesGroup` does not get the expected exceptions it now raises an `AssertionError` with a helpful message, instead of letting the raised exception/group fall through. The raised exception is available in the ``__context__`` of the `AssertionError` and can be seen in the traceback. diff --git a/src/trio/_core/_run.py b/src/trio/_core/_run.py index 718dc181d..543009f62 100644 --- a/src/trio/_core/_run.py +++ b/src/trio/_core/_run.py @@ -76,6 +76,7 @@ PosArgT = TypeVarTuple("PosArgT") StatusT = TypeVar("StatusT", default=None) StatusT_contra = TypeVar("StatusT_contra", contravariant=True, default=None) + BaseExcT = TypeVar("BaseExcT", bound=BaseException) else: from typing import TypeVar @@ -123,6 +124,21 @@ def _hypothesis_plugin_setup() -> None: # pragma: no cover _ALLOW_DETERMINISTIC_SCHEDULING = True # type: ignore register_random(_r) + # monkeypatch repr_callable to make repr's way better + # requires importing hypothesis (in the test file or in conftest.py) + try: + from hypothesis.internal.reflection import get_pretty_function_description + + import trio.testing._raises_group + + def repr_callable(fun: Callable[[BaseExcT], bool]) -> str: + # add quotes around the signature + return repr(get_pretty_function_description(fun)) + + trio.testing._raises_group.repr_callable = repr_callable + except ImportError: + pass + def _count_context_run_tb_frames() -> int: """Count implementation dependent traceback frames from Context.run() diff --git a/src/trio/_tests/test_exports.py b/src/trio/_tests/test_exports.py index bad9ebec3..0bf619e3b 100644 --- a/src/trio/_tests/test_exports.py +++ b/src/trio/_tests/test_exports.py @@ -318,8 +318,11 @@ def lookup_symbol(symbol: str) -> dict[str, str]: if module_name == "trio.socket" and class_name in dir(stdlib_socket): continue - # ignore class that does dirty tricks - if class_ is trio.testing.RaisesGroup: + # Ignore classes that don't use attrs, they only define their members once + # __init__ is called (and reason they don't use attrs is because they're going + # to be reimplemented in pytest). + # Not 100% that's the case, and it works locally, so whatever /shrug + if class_ is trio.testing.RaisesGroup or class_ is trio.testing.Matcher: continue # dir() and inspect.getmembers doesn't display properties from the metaclass @@ -460,11 +463,6 @@ def lookup_symbol(symbol: str) -> dict[str, str]: "send_all_hook", "wait_send_all_might_not_block_hook", }, - trio.testing.Matcher: { - "exception_type", - "match", - "check", - }, } if tool == "mypy" and class_ in EXTRAS: before = len(extra) diff --git a/src/trio/_tests/test_testing_raisesgroup.py b/src/trio/_tests/test_testing_raisesgroup.py index 4efd5d06f..b3fe2ae75 100644 --- a/src/trio/_tests/test_testing_raisesgroup.py +++ b/src/trio/_tests/test_testing_raisesgroup.py @@ -3,20 +3,35 @@ import re import sys from types import TracebackType +from typing import TYPE_CHECKING import pytest import trio from trio.testing import Matcher, RaisesGroup +from trio.testing._raises_group import repr_callable if sys.version_info < (3, 11): - from exceptiongroup import ExceptionGroup + from exceptiongroup import BaseExceptionGroup, ExceptionGroup + +if TYPE_CHECKING: + from _pytest.python_api import RaisesContext def wrap_escape(s: str) -> str: return "^" + re.escape(s) + "$" +def fails_raises_group( + msg: str, add_prefix: bool = True +) -> RaisesContext[AssertionError]: + assert ( + msg[-1] != "\n" + ), "developer error, expected string should not end with newline" + prefix = "Raised exception group did not match: " if add_prefix else "" + return pytest.raises(AssertionError, match=wrap_escape(prefix + msg)) + + def test_raises_group() -> None: with pytest.raises( ValueError, @@ -25,13 +40,14 @@ def test_raises_group() -> None: ), ): RaisesGroup(TypeError()) # type: ignore[call-overload] - with RaisesGroup(ValueError): raise ExceptionGroup("foo", (ValueError(),)) - with RaisesGroup(SyntaxError): - with RaisesGroup(ValueError): - raise ExceptionGroup("foo", (SyntaxError(),)) + with ( + fails_raises_group("'SyntaxError' is not of type 'ValueError'"), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("foo", (SyntaxError(),)) # multiple exceptions with RaisesGroup(ValueError, SyntaxError): @@ -59,23 +75,67 @@ def test_raises_group() -> None: ), ) - # will error if there's excess exceptions - with pytest.raises(ExceptionGroup): - with RaisesGroup(ValueError): - raise ExceptionGroup("", (ValueError(), ValueError())) - with pytest.raises(ExceptionGroup): - with RaisesGroup(ValueError): - raise ExceptionGroup("", (RuntimeError(), ValueError())) +def test_incorrect_number_exceptions() -> None: + # We previously gave an error saying the number of exceptions was wrong, + # but we now instead indicate excess/missing exceptions + with ( + fails_raises_group( + "1 matched exception. Unexpected exception(s): [RuntimeError()]" + ), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", (RuntimeError(), ValueError())) # will error if there's missing exceptions - with pytest.raises(ExceptionGroup): - with RaisesGroup(ValueError, ValueError): - raise ExceptionGroup("", (ValueError(),)) + with ( + fails_raises_group( + "1 matched exception. Too few exceptions raised, found no match for: ['SyntaxError']" + ), + RaisesGroup(ValueError, SyntaxError), + ): + raise ExceptionGroup("", (ValueError(),)) + + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "Too few exceptions raised!\n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " It matches ValueError() which was paired with 'ValueError'" + ), + RaisesGroup(ValueError, ValueError), + ): + raise ExceptionGroup("", (ValueError(),)) - with pytest.raises(ExceptionGroup): - with RaisesGroup(ValueError, SyntaxError): - raise ExceptionGroup("", (ValueError(),)) + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "Unexpected exception(s)!\n" + "The following raised exceptions did not find a match\n" + " ValueError():\n" + " It matches 'ValueError' which was paired with ValueError()" + ), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", (ValueError(), ValueError())) + + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " It matches ValueError() which was paired with 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " SyntaxError():\n" + " 'SyntaxError' is not of type 'ValueError'" + ), + RaisesGroup(ValueError, ValueError), + ): + raise ExceptionGroup("", [ValueError(), SyntaxError()]) def test_flatten_subgroups() -> None: @@ -96,9 +156,6 @@ def test_flatten_subgroups() -> None: "", (ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)),), ) - with pytest.raises(ExceptionGroup): - with RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True)): - raise ExceptionGroup("", (ValueError(),)) # but not the other way around with pytest.raises( @@ -107,6 +164,102 @@ def test_flatten_subgroups() -> None: ): RaisesGroup(RaisesGroup(ValueError), flatten_subgroups=True) # type: ignore[call-overload] + # flatten_subgroups is not sufficient to catch fully unwrapped + with ( + fails_raises_group( + "'ValueError' is not an exception group, but would match with `allow_unwrapped=True`" + ), + RaisesGroup(ValueError, flatten_subgroups=True), + ): + raise ValueError + with ( + fails_raises_group( + "RaisesGroup(ValueError, flatten_subgroups=True): 'ValueError' is not an exception group, but would match with `allow_unwrapped=True`" + ), + RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True)), + ): + raise ExceptionGroup("", (ValueError(),)) + + # helpful suggestion if flatten_subgroups would make it pass + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'TypeError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), TypeError()]):\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'TypeError'\n" + "Did you mean to use `flatten_subgroups=True`?", + add_prefix=False, + ), + RaisesGroup(ValueError, TypeError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError(), TypeError()])]) + # but doesn't consider check (otherwise we'd break typing guarantees) + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'TypeError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), TypeError()]):\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'TypeError'\n" + "Did you mean to use `flatten_subgroups=True`?", + add_prefix=False, + ), + RaisesGroup( + ValueError, + TypeError, + check=lambda eg: len(eg.exceptions) == 1, + ), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError(), TypeError()])]) + # correct number of exceptions, and flatten_subgroups would make it pass + # This now doesn't print a repr of the caught exception at all, but that can be found in the traceback + with ( + fails_raises_group( + "Raised exception group did not match: Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?", + add_prefix=False, + ), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError()])]) + # correct number of exceptions, but flatten_subgroups wouldn't help, so we don't suggest it + with ( + fails_raises_group("Unexpected nested 'ExceptionGroup', expected 'ValueError'"), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [TypeError()])]) + + # flatten_subgroups can be suggested if nested. This will implicitly ask the user to + # do `RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True))` which is unlikely + # to be what they actually want - but I don't think it's worth trying to special-case + with ( + fails_raises_group( + "RaisesGroup(ValueError): Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?", + ), + RaisesGroup(RaisesGroup(ValueError)), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [ExceptionGroup("", [ValueError()])])], + ) + + # Don't mention "unexpected nested" if expecting an ExceptionGroup. + # Although it should perhaps be an error to specify `RaisesGroup(ExceptionGroup)` in + # favor of doing `RaisesGroup(RaisesGroup(...))`. + with ( + fails_raises_group("'BaseExceptionGroup' is not of type 'ExceptionGroup'"), + RaisesGroup(ExceptionGroup), + ): + raise BaseExceptionGroup("", [BaseExceptionGroup("", [KeyboardInterrupt()])]) + def test_catch_unwrapped_exceptions() -> None: # Catches lone exceptions with strict=False @@ -140,23 +293,52 @@ def test_catch_unwrapped_exceptions() -> None: raise ExceptionGroup("", [ValueError()]) # with allow_unwrapped=False (default) it will not be caught - with pytest.raises(ValueError, match=r"^value error text$"): - with RaisesGroup(ValueError): - raise ValueError("value error text") + with ( + fails_raises_group( + "'ValueError' is not an exception group, but would match with `allow_unwrapped=True`" + ), + RaisesGroup(ValueError), + ): + raise ValueError("value error text") - # allow_unwrapped on it's own won't match against nested groups - with pytest.raises(ExceptionGroup): - with RaisesGroup(ValueError, allow_unwrapped=True): - raise ExceptionGroup("", [ExceptionGroup("", [ValueError()])]) + # allow_unwrapped on its own won't match against nested groups + with ( + fails_raises_group( + "Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?", + ), + RaisesGroup(ValueError, allow_unwrapped=True), + ): + raise ExceptionGroup("foo", [ExceptionGroup("bar", [ValueError()])]) - # for that you need both allow_unwrapped and flatten_subgroups + # you need both allow_unwrapped and flatten_subgroups to fully emulate except* with RaisesGroup(ValueError, allow_unwrapped=True, flatten_subgroups=True): raise ExceptionGroup("", [ExceptionGroup("", [ValueError()])]) # code coverage - with pytest.raises(TypeError): - with RaisesGroup(ValueError, allow_unwrapped=True): - raise TypeError + with ( + fails_raises_group( + "Raised exception (group) did not match: 'TypeError' is not of type 'ValueError'", + add_prefix=False, + ), + RaisesGroup(ValueError, allow_unwrapped=True), + ): + raise TypeError("this text doesn't show up in the error message") + with ( + fails_raises_group( + "Raised exception (group) did not match: Matcher(ValueError): 'TypeError' is not of type 'ValueError'", + add_prefix=False, + ), + RaisesGroup(Matcher(ValueError), allow_unwrapped=True), + ): + raise TypeError + + # check we don't suggest unwrapping with nested RaisesGroup + with ( + fails_raises_group("'ValueError' is not an exception group"), + RaisesGroup(RaisesGroup(ValueError)), + ): + raise ValueError def test_match() -> None: @@ -181,18 +363,42 @@ def test_match() -> None: e.add_note("my note") raise e - with pytest.raises(ExceptionGroup): - with RaisesGroup(ValueError, match="foo"): - raise ExceptionGroup("bar", (ValueError(),)) + with ( + fails_raises_group( + "Regex pattern 'foo' did not match 'bar' of 'ExceptionGroup'" + ), + RaisesGroup(ValueError, match="foo"), + ): + raise ExceptionGroup("bar", (ValueError(),)) + + # Suggest a fix for easy pitfall of adding match to the RaisesGroup instead of + # using a Matcher. + # This requires a single expected & raised exception, the expected is a type, + # and `isinstance(raised, expected_type)`. + with ( + fails_raises_group( + "Regex pattern 'foo' did not match 'bar' of 'ExceptionGroup', but matched the expected 'ValueError'. You might want RaisesGroup(Matcher(ValueError, match='foo'))" + ), + RaisesGroup(ValueError, match="foo"), + ): + raise ExceptionGroup("bar", [ValueError("foo")]) def test_check() -> None: exc = ExceptionGroup("", (ValueError(),)) - with RaisesGroup(ValueError, check=lambda x: x is exc): + + def is_exc(e: ExceptionGroup[ValueError]) -> bool: + return e is exc + + is_exc_repr = repr_callable(is_exc) + with RaisesGroup(ValueError, check=is_exc): raise exc - with pytest.raises(ExceptionGroup): - with RaisesGroup(ValueError, check=lambda x: x is exc): - raise ExceptionGroup("", (ValueError(),)) + + with ( + fails_raises_group(f"check {is_exc_repr} did not return True"), + RaisesGroup(ValueError, check=is_exc), + ): + raise ExceptionGroup("", (ValueError(),)) def test_unwrapped_match_check() -> None: @@ -238,12 +444,14 @@ def check_message( message: str, body: RaisesGroup[BaseException], ) -> None: - with pytest.raises( - AssertionError, - match=f"^DID NOT RAISE any exception, expected {re.escape(message)}$", + with ( + pytest.raises( + AssertionError, + match=f"^DID NOT RAISE any exception, expected {re.escape(message)}$", + ), + body, ): - with body: - ... + ... # basic check_message("ExceptionGroup(ValueError)", RaisesGroup(ValueError)) @@ -290,6 +498,492 @@ def check_message( ) +def test_assert_message() -> None: + # the message does not need to list all parameters to RaisesGroup, nor all exceptions + # in the exception group, as those are both visible in the traceback. + # first fails to match + with ( + fails_raises_group("'TypeError' is not of type 'ValueError'"), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("a", [TypeError()]) + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " RaisesGroup(ValueError, match='a')\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [RuntimeError()]):\n" + " RaisesGroup(ValueError): 'RuntimeError' is not of type 'ValueError'\n" + " RaisesGroup(ValueError, match='a'): Regex pattern 'a' did not match '' of 'ExceptionGroup'\n" + " RuntimeError():\n" + " RaisesGroup(ValueError): 'RuntimeError' is not an exception group\n" + " RaisesGroup(ValueError, match='a'): 'RuntimeError' is not an exception group", + add_prefix=False, # to see the full structure + ), + RaisesGroup(RaisesGroup(ValueError), RaisesGroup(ValueError, match="a")), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [RuntimeError()]), RuntimeError()], + ) + + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "2 matched exceptions. \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(RuntimeError)\n" + " RaisesGroup(ValueError)\n" + "The following raised exceptions did not find a match\n" + " RuntimeError():\n" + # " 'RuntimeError' is not of type 'ValueError'\n" + # " Matcher(TypeError): 'RuntimeError' is not of type 'TypeError'\n" + " RaisesGroup(RuntimeError): 'RuntimeError' is not an exception group, but would match with `allow_unwrapped=True`\n" + " RaisesGroup(ValueError): 'RuntimeError' is not an exception group\n" + " ValueError('bar'):\n" + " It matches 'ValueError' which was paired with ValueError('foo')\n" + " RaisesGroup(RuntimeError): 'ValueError' is not an exception group\n" + " RaisesGroup(ValueError): 'ValueError' is not an exception group, but would match with `allow_unwrapped=True`", + add_prefix=False, # to see the full structure + ), + RaisesGroup( + ValueError, + Matcher(TypeError), + RaisesGroup(RuntimeError), + RaisesGroup(ValueError), + ), + ): + raise ExceptionGroup( + "a", + [RuntimeError(), TypeError(), ValueError("foo"), ValueError("bar")], + ) + + with ( + fails_raises_group( + "1 matched exception. 'AssertionError' is not of type 'TypeError'" + ), + RaisesGroup(ValueError, TypeError), + ): + raise ExceptionGroup("a", [ValueError(), AssertionError()]) + + with ( + fails_raises_group( + "Matcher(ValueError): 'TypeError' is not of type 'ValueError'" + ), + RaisesGroup(Matcher(ValueError)), + ): + raise ExceptionGroup("a", [TypeError()]) + + # suggest escaping + with ( + fails_raises_group( + "Raised exception group did not match: Regex pattern 'h(ell)o' did not match 'h(ell)o' of 'ExceptionGroup'\n" + " Did you mean to `re.escape()` the regex?", + add_prefix=False, # to see the full structure + ), + RaisesGroup(ValueError, match="h(ell)o"), + ): + raise ExceptionGroup("h(ell)o", [ValueError()]) + with ( + fails_raises_group( + "Matcher(match='h(ell)o'): Regex pattern 'h(ell)o' did not match 'h(ell)o'\n" + " Did you mean to `re.escape()` the regex?", + ), + RaisesGroup(Matcher(match="h(ell)o")), + ): + raise ExceptionGroup("", [ValueError("h(ell)o")]) + + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'ValueError'\n" + " 'ValueError'\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), TypeError()]):\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'", + add_prefix=False, # to see the full structure + ), + RaisesGroup(ValueError, ValueError, ValueError, ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError(), TypeError()])]) + + +def test_message_indent() -> None: + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError, ValueError)\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [TypeError(), RuntimeError()]):\n" + " RaisesGroup(ValueError, ValueError): \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " TypeError():\n" + " 'TypeError' is not of type 'ValueError'\n" + " 'TypeError' is not of type 'ValueError'\n" + " RuntimeError():\n" + " 'RuntimeError' is not of type 'ValueError'\n" + " 'RuntimeError' is not of type 'ValueError'\n" + # TODO: this line is not great, should maybe follow the same format as the other and say + # 'ValueError': Unexpected nested 'ExceptionGroup' (?) + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " TypeError():\n" + " RaisesGroup(ValueError, ValueError): 'TypeError' is not an exception group\n" + " 'TypeError' is not of type 'ValueError'", + add_prefix=False, + ), + RaisesGroup( + RaisesGroup(ValueError, ValueError), + ValueError, + ), + ): + raise ExceptionGroup( + "", + [ + ExceptionGroup("", [TypeError(), RuntimeError()]), + TypeError(), + ], + ) + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "RaisesGroup(ValueError, ValueError): \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " TypeError():\n" + " 'TypeError' is not of type 'ValueError'\n" + " 'TypeError' is not of type 'ValueError'\n" + " RuntimeError():\n" + " 'RuntimeError' is not of type 'ValueError'\n" + " 'RuntimeError' is not of type 'ValueError'", + add_prefix=False, + ), + RaisesGroup( + RaisesGroup(ValueError, ValueError), + ), + ): + raise ExceptionGroup( + "", + [ + ExceptionGroup("", [TypeError(), RuntimeError()]), + ], + ) + + +def test_suggestion_on_nested_and_brief_error() -> None: + # Make sure "Did you mean" suggestion gets indented iff it follows a single-line error + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ExceptionGroup('', [ValueError()])]):\n" + " RaisesGroup(ValueError): Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'", + ), + RaisesGroup(RaisesGroup(ValueError), ValueError), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [ExceptionGroup("", [ValueError()])])], + ) + # if indented here it would look like another raised exception + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError, ValueError)\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), ExceptionGroup('', [ValueError()])]):\n" + " RaisesGroup(ValueError, ValueError): \n" + " 1 matched exception. \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " It matches ValueError() which was paired with 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError()]):\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'\n" + " Did you mean to use `flatten_subgroups=True`?\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'" + ), + RaisesGroup(RaisesGroup(ValueError, ValueError), ValueError), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [ValueError(), ExceptionGroup("", [ValueError()])])], + ) + + # re.escape always comes after single-line errors + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(Exception, match='^hello')\n" + " 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('^hello', [Exception()]):\n" + " RaisesGroup(Exception, match='^hello'): Regex pattern '^hello' did not match '^hello' of 'ExceptionGroup'\n" + " Did you mean to `re.escape()` the regex?\n" + " Unexpected nested 'ExceptionGroup', expected 'ValueError'" + ), + RaisesGroup(RaisesGroup(Exception, match="^hello"), ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("^hello", [Exception()])]) + + +def test_assert_message_nested() -> None: + # we only get one instance of aaaaaaaaaa... and bbbbbb..., but we do get multiple instances of ccccc... and dddddd.. + # but I think this now only prints the full repr when that is necessary to disambiguate exceptions + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " RaisesGroup(RaisesGroup(ValueError))\n" + " RaisesGroup(Matcher(TypeError, match='foo'))\n" + " RaisesGroup(TypeError, ValueError)\n" + "The following raised exceptions did not find a match\n" + " TypeError('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'):\n" + " RaisesGroup(ValueError): 'TypeError' is not an exception group\n" + " RaisesGroup(RaisesGroup(ValueError)): 'TypeError' is not an exception group\n" + " RaisesGroup(Matcher(TypeError, match='foo')): 'TypeError' is not an exception group\n" + " RaisesGroup(TypeError, ValueError): 'TypeError' is not an exception group\n" + " ExceptionGroup('Exceptions from Trio nursery', [TypeError('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb')]):\n" + " RaisesGroup(ValueError): 'TypeError' is not of type 'ValueError'\n" + " RaisesGroup(RaisesGroup(ValueError)): RaisesGroup(ValueError): 'TypeError' is not an exception group\n" + " RaisesGroup(Matcher(TypeError, match='foo')): Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'\n" + " RaisesGroup(TypeError, ValueError): 1 matched exception. Too few exceptions raised, found no match for: ['ValueError']\n" + " ExceptionGroup('Exceptions from Trio nursery', [TypeError('cccccccccccccccccccccccccccccccccccccccc'), TypeError('dddddddddddddddddddddddddddddddddddddddd')]):\n" + " RaisesGroup(ValueError): \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " TypeError('cccccccccccccccccccccccccccccccccccccccc'):\n" + " 'TypeError' is not of type 'ValueError'\n" + " TypeError('dddddddddddddddddddddddddddddddddddddddd'):\n" + " 'TypeError' is not of type 'ValueError'\n" + " RaisesGroup(RaisesGroup(ValueError)): \n" + " The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " The following raised exceptions did not find a match\n" + " TypeError('cccccccccccccccccccccccccccccccccccccccc'):\n" + " RaisesGroup(ValueError): 'TypeError' is not an exception group\n" + " TypeError('dddddddddddddddddddddddddddddddddddddddd'):\n" + " RaisesGroup(ValueError): 'TypeError' is not an exception group\n" + " RaisesGroup(Matcher(TypeError, match='foo')): \n" + " The following expected exceptions did not find a match:\n" + " Matcher(TypeError, match='foo')\n" + " The following raised exceptions did not find a match\n" + " TypeError('cccccccccccccccccccccccccccccccccccccccc'):\n" + " Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'cccccccccccccccccccccccccccccccccccccccc'\n" + " TypeError('dddddddddddddddddddddddddddddddddddddddd'):\n" + " Matcher(TypeError, match='foo'): Regex pattern 'foo' did not match 'dddddddddddddddddddddddddddddddddddddddd'\n" + " RaisesGroup(TypeError, ValueError): \n" + " 1 matched exception. \n" + " The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " The following raised exceptions did not find a match\n" + " TypeError('dddddddddddddddddddddddddddddddddddddddd'):\n" + " It matches 'TypeError' which was paired with TypeError('cccccccccccccccccccccccccccccccccccccccc')\n" + " 'TypeError' is not of type 'ValueError'", + add_prefix=False, # to see the full structure + ), + RaisesGroup( + RaisesGroup(ValueError), + RaisesGroup(RaisesGroup(ValueError)), + RaisesGroup(Matcher(TypeError, match="foo")), + RaisesGroup(TypeError, ValueError), + ), + ): + raise ExceptionGroup( + "", + [ + TypeError("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ExceptionGroup( + "Exceptions from Trio nursery", + [TypeError("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")], + ), + ExceptionGroup( + "Exceptions from Trio nursery", + [ + TypeError("cccccccccccccccccccccccccccccccccccccccc"), + TypeError("dddddddddddddddddddddddddddddddddddddddd"), + ], + ), + ], + ) + + +@pytest.mark.skipif( + "hypothesis" in sys.modules, + reason="hypothesis may have monkeypatched _check_repr", +) +def test_check_no_patched_repr() -> None: + # We make `_check_repr` monkeypatchable to avoid this very ugly and verbose + # repr. The other tests that use `check` make use of `_check_repr` so they'll + # continue passing in case it is patched - but we have this one test that + # demonstrates just how nasty it gets otherwise. + with ( + pytest.raises( + AssertionError, + match=( + r"^Raised exception group did not match: \n" + r"The following expected exceptions did not find a match:\n" + r" Matcher\(check=. at .*>\)\n" + r" 'TypeError'\n" + r"The following raised exceptions did not find a match\n" + r" ValueError\('foo'\):\n" + r" Matcher\(check=. at .*>\): check did not return True\n" + r" 'ValueError' is not of type 'TypeError'\n" + r" ValueError\('bar'\):\n" + r" Matcher\(check=. at .*>\): check did not return True\n" + r" 'ValueError' is not of type 'TypeError'$" + ), + ), + RaisesGroup(Matcher(check=lambda x: False), TypeError), + ): + raise ExceptionGroup("", [ValueError("foo"), ValueError("bar")]) + + +def test_misordering_example() -> None: + with ( + fails_raises_group( + "\n" + "3 matched exceptions. \n" + "The following expected exceptions did not find a match:\n" + " Matcher(ValueError, match='foo')\n" + " It matches ValueError('foo') which was paired with 'ValueError'\n" + " It matches ValueError('foo') which was paired with 'ValueError'\n" + " It matches ValueError('foo') which was paired with 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " ValueError('bar'):\n" + " It matches 'ValueError' which was paired with ValueError('foo')\n" + " It matches 'ValueError' which was paired with ValueError('foo')\n" + " It matches 'ValueError' which was paired with ValueError('foo')\n" + " Matcher(ValueError, match='foo'): Regex pattern 'foo' did not match 'bar'\n" + "There exist a possible match when attempting an exhaustive check, but RaisesGroup uses a greedy algorithm. Please make your expected exceptions more stringent with `Matcher` etc so the greedy algorithm can function." + ), + RaisesGroup( + ValueError, ValueError, ValueError, Matcher(ValueError, match="foo") + ), + ): + raise ExceptionGroup( + "", + [ + ValueError("foo"), + ValueError("foo"), + ValueError("foo"), + ValueError("bar"), + ], + ) + + +def test_brief_error_on_one_fail() -> None: + """if only one raised and one expected fail to match up, we print a full table iff + the raised exception would match one of the expected that previously got matched""" + # no also-matched + with ( + fails_raises_group( + "1 matched exception. 'TypeError' is not of type 'RuntimeError'" + ), + RaisesGroup(ValueError, RuntimeError), + ): + raise ExceptionGroup("", [ValueError(), TypeError()]) + + # raised would match an expected + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "The following expected exceptions did not find a match:\n" + " 'RuntimeError'\n" + "The following raised exceptions did not find a match\n" + " TypeError():\n" + " It matches 'Exception' which was paired with ValueError()\n" + " 'TypeError' is not of type 'RuntimeError'" + ), + RaisesGroup(Exception, RuntimeError), + ): + raise ExceptionGroup("", [ValueError(), TypeError()]) + + # expected would match a raised + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "The following expected exceptions did not find a match:\n" + " 'ValueError'\n" + " It matches ValueError() which was paired with 'ValueError'\n" + "The following raised exceptions did not find a match\n" + " TypeError():\n" + " 'TypeError' is not of type 'ValueError'" + ), + RaisesGroup(ValueError, ValueError), + ): + raise ExceptionGroup("", [ValueError(), TypeError()]) + + +def test_identity_oopsies() -> None: + # it's both possible to have several instances of the same exception in the same group + # and to expect multiple of the same type + # this previously messed up the logic + + with ( + fails_raises_group( + "3 matched exceptions. 'RuntimeError' is not of type 'TypeError'" + ), + RaisesGroup(ValueError, ValueError, ValueError, TypeError), + ): + raise ExceptionGroup( + "", [ValueError(), ValueError(), ValueError(), RuntimeError()] + ) + + e = ValueError("foo") + m = Matcher(match="bar") + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " Matcher(match='bar')\n" + " Matcher(match='bar')\n" + " Matcher(match='bar')\n" + "The following raised exceptions did not find a match\n" + " ValueError('foo'):\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " ValueError('foo'):\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " ValueError('foo'):\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'\n" + " Matcher(match='bar'): Regex pattern 'bar' did not match 'foo'" + ), + RaisesGroup(m, m, m), + ): + raise ExceptionGroup("", [e, e, e]) + + def test_matcher() -> None: with pytest.raises( ValueError, @@ -304,31 +998,47 @@ def test_matcher() -> None: with RaisesGroup(Matcher(ValueError)): raise ExceptionGroup("", (ValueError(),)) - with pytest.raises(ExceptionGroup): - with RaisesGroup(Matcher(TypeError)): - raise ExceptionGroup("", (ValueError(),)) + with ( + fails_raises_group( + "Matcher(TypeError): 'ValueError' is not of type 'TypeError'" + ), + RaisesGroup(Matcher(TypeError)), + ): + raise ExceptionGroup("", (ValueError(),)) def test_matcher_match() -> None: with RaisesGroup(Matcher(ValueError, "foo")): raise ExceptionGroup("", (ValueError("foo"),)) - with pytest.raises(ExceptionGroup): - with RaisesGroup(Matcher(ValueError, "foo")): - raise ExceptionGroup("", (ValueError("bar"),)) + with ( + fails_raises_group( + "Matcher(ValueError, match='foo'): Regex pattern 'foo' did not match 'bar'" + ), + RaisesGroup(Matcher(ValueError, "foo")), + ): + raise ExceptionGroup("", (ValueError("bar"),)) # Can be used without specifying the type with RaisesGroup(Matcher(match="foo")): raise ExceptionGroup("", (ValueError("foo"),)) - with pytest.raises(ExceptionGroup): - with RaisesGroup(Matcher(match="foo")): - raise ExceptionGroup("", (ValueError("bar"),)) + with ( + fails_raises_group( + "Matcher(match='foo'): Regex pattern 'foo' did not match 'bar'" + ), + RaisesGroup(Matcher(match="foo")), + ): + raise ExceptionGroup("", (ValueError("bar"),)) # check ^$ with RaisesGroup(Matcher(ValueError, match="^bar$")): raise ExceptionGroup("", [ValueError("bar")]) - with pytest.raises(ExceptionGroup): - with RaisesGroup(Matcher(ValueError, match="^bar$")): - raise ExceptionGroup("", [ValueError("barr")]) + with ( + fails_raises_group( + "Matcher(ValueError, match='^bar$'): Regex pattern '^bar$' did not match 'barr'" + ), + RaisesGroup(Matcher(ValueError, match="^bar$")), + ): + raise ExceptionGroup("", [ValueError("barr")]) def test_Matcher_check() -> None: @@ -345,9 +1055,24 @@ def check_errno_is_5(e: OSError) -> bool: with RaisesGroup(Matcher(OSError, check=check_errno_is_5)): raise ExceptionGroup("", (OSError(5, ""),)) - with pytest.raises(ExceptionGroup): - with RaisesGroup(Matcher(OSError, check=check_errno_is_5)): - raise ExceptionGroup("", (OSError(6, ""),)) + # avoid printing overly verbose repr multiple times + with ( + fails_raises_group( + f"Matcher(OSError, check={check_errno_is_5!r}): check did not return True" + ), + RaisesGroup(Matcher(OSError, check=check_errno_is_5)), + ): + raise ExceptionGroup("", (OSError(6, ""),)) + + # in nested cases you still get it multiple times though + # to address this you'd need logic in Matcher.__repr__ and RaisesGroup.__repr__ + with ( + fails_raises_group( + f"RaisesGroup(Matcher(OSError, check={check_errno_is_5!r})): Matcher(OSError, check={check_errno_is_5!r}): check did not return True" + ), + RaisesGroup(RaisesGroup(Matcher(OSError, check=check_errno_is_5))), + ): + raise ExceptionGroup("", [ExceptionGroup("", [OSError(6, "")])]) def test_matcher_tostring() -> None: @@ -363,6 +1088,23 @@ def test_matcher_tostring() -> None: ) +def test_raisesgroup_tostring() -> None: + def check_str_and_repr(s: str) -> None: + evaled = eval(s) + assert s == str(evaled) == repr(evaled) + + check_str_and_repr("RaisesGroup(ValueError)") + check_str_and_repr("RaisesGroup(RaisesGroup(ValueError))") + check_str_and_repr("RaisesGroup(Matcher(ValueError))") + check_str_and_repr("RaisesGroup(ValueError, allow_unwrapped=True)") + check_str_and_repr("RaisesGroup(ValueError, match='aoeu')") + + assert ( + str(RaisesGroup(ValueError, match="[a-z]", check=bool)) + == f"RaisesGroup(ValueError, match='[a-z]', check={bool!r})" + ) + + def test__ExceptionInfo(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr( trio.testing._raises_group, diff --git a/src/trio/_tests/type_tests/raisesgroup.py b/src/trio/_tests/type_tests/raisesgroup.py index 4d5ed4882..75c56c053 100644 --- a/src/trio/_tests/type_tests/raisesgroup.py +++ b/src/trio/_tests/type_tests/raisesgroup.py @@ -12,12 +12,11 @@ # split into functions to isolate the different scopes -def check_matcher_typevar_default(e: Matcher) -> object: +def check_matcher_typevar_default(e: Matcher) -> None: assert e.exception_type is not None - exc: type[BaseException] = e.exception_type + _exc: type[BaseException] = e.exception_type # this would previously pass, as the type would be `Any` e.exception_type().blah() # type: ignore - return exc # Silence Pyright unused var warning def check_basic_contextmanager() -> None: @@ -77,10 +76,7 @@ def check_filenotfound(exc: FileNotFoundError) -> bool: def raisesgroup_check_type_narrowing() -> None: """Check type narrowing on the `check` argument to `RaisesGroup`. - All `type: ignore`s are correctly pointing out type errors, except - where otherwise noted. - - + All `type: ignore`s are correctly pointing out type errors. """ def handle_exc(e: BaseExceptionGroup[BaseException]) -> bool: @@ -223,17 +219,11 @@ def check_triple_nested_raisesgroup() -> None: def check_check_typing() -> None: - # mypy issue is https://github.com/python/mypy/issues/18185 - - # fmt: off - # mypy raises an error on `assert_type` - # pyright raises an error on `RaisesGroup(ValueError).check` - # to satisfy both, need to disable formatting and put it on one line - assert_type(RaisesGroup(ValueError).check, # type: ignore + # `BaseExceptiongroup` should perhaps be `ExceptionGroup`, but close enough + assert_type( + RaisesGroup(ValueError).check, Union[ - Callable[[BaseExceptionGroup[ValueError]], None], - Callable[[ExceptionGroup[ValueError]], None], + Callable[[BaseExceptionGroup[ValueError]], bool], None, ], ) - # fmt: on diff --git a/src/trio/testing/_raises_group.py b/src/trio/testing/_raises_group.py index 700c16ca6..9c2cb857f 100644 --- a/src/trio/testing/_raises_group.py +++ b/src/trio/testing/_raises_group.py @@ -2,7 +2,9 @@ import re import sys +from abc import ABC, abstractmethod from re import Pattern +from textwrap import indent from typing import ( TYPE_CHECKING, Generic, @@ -154,11 +156,117 @@ def _stringify_exception(exc: BaseException) -> str: _REGEX_NO_FLAGS = re.compile(r"").flags +def _match_pattern(match: Pattern[str]) -> str | Pattern[str]: + """helper function to remove redundant `re.compile` calls when printing regex""" + return match.pattern if match.flags == _REGEX_NO_FLAGS else match + + +def repr_callable(fun: Callable[[BaseExcT_1], bool]) -> str: + """Get the repr of a ``check`` parameter. + + Split out so it can be monkeypatched (e.g. by our hypothesis plugin) + """ + return repr(fun) + + +def _exception_type_name(e: type[BaseException]) -> str: + return repr(e.__name__) + + +def _check_raw_type( + expected_type: type[BaseException] | None, + exception: BaseException, +) -> str | None: + if expected_type is None: + return None + + if not isinstance( + exception, + expected_type, + ): + actual_type_str = _exception_type_name(type(exception)) + expected_type_str = _exception_type_name(expected_type) + if isinstance(exception, BaseExceptionGroup) and not issubclass( + expected_type, BaseExceptionGroup + ): + return f"Unexpected nested {actual_type_str}, expected {expected_type_str}" + return f"{actual_type_str} is not of type {expected_type_str}" + return None + + +class AbstractMatcher(ABC, Generic[BaseExcT_co]): + """ABC with common functionality shared between Matcher and RaisesGroup""" + + def __init__( + self, + match: str | Pattern[str] | None, + check: Callable[[BaseExcT_co], bool] | None, + ) -> None: + if isinstance(match, str): + self.match: Pattern[str] | None = re.compile(match) + else: + self.match = match + self.check = check + self._fail_reason: str | None = None + + # used to suppress repeated printing of `repr(self.check)` + self._nested: bool = False + + @property + def fail_reason(self) -> str | None: + """Set after a call to `matches` to give a human-readable + reason for why the match failed. + When used as a context manager the string will be given as the text of an + `AssertionError`""" + return self._fail_reason + + def _check_check( + self: AbstractMatcher[BaseExcT_1], + exception: BaseExcT_1, + ) -> bool: + if self.check is None: + return True + + if self.check(exception): + return True + + check_repr = "" if self._nested else " " + repr_callable(self.check) + self._fail_reason = f"check{check_repr} did not return True" + return False + + def _check_match(self, e: BaseException) -> bool: + if self.match is None or re.search( + self.match, + stringified_exception := _stringify_exception(e), + ): + return True + + maybe_specify_type = ( + f" of {_exception_type_name(type(e))}" + if isinstance(e, BaseExceptionGroup) + else "" + ) + self._fail_reason = f"Regex pattern {_match_pattern(self.match)!r} did not match {stringified_exception!r}{maybe_specify_type}" + if _match_pattern(self.match) == stringified_exception: + self._fail_reason += "\n Did you mean to `re.escape()` the regex?" + return False + + # TODO: when transitioning to pytest, harmonize Matcher and RaisesGroup + # signatures. One names the parameter `exc_val` and the other `exception` + @abstractmethod + def matches( + self: AbstractMatcher[BaseExcT_1], exc_val: BaseException + ) -> TypeGuard[BaseExcT_1]: + """Check if an exception matches the requirements of this AbstractMatcher. + If it fails, `AbstractMatcher.fail_reason` should be set. + """ + + @final -class Matcher(Generic[MatchE]): +class Matcher(AbstractMatcher[MatchE]): """Helper class to be used together with RaisesGroups when you want to specify requirements on sub-exceptions. Only specifying the type is redundant, and it's also unnecessary when the type is a nested `RaisesGroup` since it supports the same arguments. The type is checked with `isinstance`, and does not need to be an exact match. If that is wanted you can use the ``check`` parameter. - :meth:`trio.testing.Matcher.matches` can also be used standalone to check individual exceptions. + :meth:`Matcher.matches` can also be used standalone to check individual exceptions. Examples:: @@ -169,12 +277,14 @@ class Matcher(Generic[MatchE]): with RaisesGroups(Matcher(check=lambda x: type(x) is ValueError)): ... + Tip: if you install ``hypothesis`` and import it in ``conftest.py`` you will get + readable ``repr``s of ``check`` callables in the output. """ # At least one of the three parameters must be passed. @overload def __init__( - self: Matcher[MatchE], + self, exception_type: type[MatchE], match: str | Pattern[str] = ..., check: Callable[[MatchE], bool] = ..., @@ -198,6 +308,7 @@ def __init__( match: str | Pattern[str] | None = None, check: Callable[[MatchE], bool] | None = None, ): + super().__init__(match, check) if exception_type is None and match is None and check is None: raise ValueError("You must specify at least one parameter to match on.") if exception_type is not None and not issubclass(exception_type, BaseException): @@ -205,15 +316,13 @@ def __init__( f"exception_type {exception_type} must be a subclass of BaseException", ) self.exception_type = exception_type - self.match: Pattern[str] | None - if isinstance(match, str): - self.match = re.compile(match) - else: - self.match = match - self.check = check - def matches(self, exception: BaseException) -> TypeGuard[MatchE]: + def matches( + self, + exception: BaseException, + ) -> TypeGuard[MatchE]: """Check if an exception matches the requirements of this Matcher. + If it fails, `Matcher.fail_reason` will be set. Examples:: @@ -230,36 +339,34 @@ def matches(self, exception: BaseException) -> TypeGuard[MatchE]: assert re.search("foo", str(excinfo.value.__cause__) """ - if self.exception_type is not None and not isinstance( - exception, - self.exception_type, - ): + if not self._check_type(exception): return False - if self.match is not None and not re.search( - self.match, - _stringify_exception(exception), - ): + + if not self._check_match(exception): return False - # If exception_type is None check() accepts BaseException. - # If non-none, we have done an isinstance check above. - return self.check is None or self.check(cast("MatchE", exception)) - def __str__(self) -> str: - reqs = [] + return self._check_check(exception) + + def __repr__(self) -> str: + parameters = [] if self.exception_type is not None: - reqs.append(self.exception_type.__name__) - if (match := self.match) is not None: + parameters.append(self.exception_type.__name__) + if self.match is not None: # If no flags were specified, discard the redundant re.compile() here. - reqs.append( - f"match={match.pattern if match.flags == _REGEX_NO_FLAGS else match!r}", + parameters.append( + f"match={_match_pattern(self.match)!r}", ) if self.check is not None: - reqs.append(f"check={self.check!r}") - return f'Matcher({", ".join(reqs)})' + parameters.append(f"check={repr_callable(self.check)}") + return f'Matcher({", ".join(parameters)})' + + def _check_type(self, exception: BaseException) -> TypeGuard[MatchE]: + self._fail_reason = _check_raw_type(self.exception_type, exception) + return self._fail_reason is None @final -class RaisesGroup(Generic[BaseExcT_co]): +class RaisesGroup(AbstractMatcher[BaseExceptionGroup[BaseExcT_co]]): """Contextmanager for checking for an expected `ExceptionGroup`. This works similar to ``pytest.raises``, and a version of it will hopefully be added upstream, after which this can be deprecated and removed. See https://github.com/pytest-dev/pytest/issues/11538 @@ -278,9 +385,7 @@ class RaisesGroup(Generic[BaseExcT_co]): * With ``flatten_subgroups=True`` it will "flatten" the raised `ExceptionGroup`, extracting all exceptions inside any nested :class:`ExceptionGroup`, before matching. - It currently does not care about the order of the exceptions, so ``RaisesGroups(ValueError, TypeError)`` is equivalent to ``RaisesGroups(TypeError, ValueError)``. - - This class is not as polished as ``pytest.raises``, and is currently not as helpful in e.g. printing diffs when strings don't match, suggesting you use ``re.escape``, etc. + It does not care about the order of the exceptions, so ``RaisesGroups(ValueError, TypeError)`` is equivalent to ``RaisesGroups(TypeError, ValueError)``. Examples:: @@ -312,6 +417,9 @@ class RaisesGroup(Generic[BaseExcT_co]): even though it generally does not care about the order of the exceptions in the group. To avoid the above you should specify the first ValueError with a Matcher as well. + + Tip: if you install ``hypothesis`` and import it in ``conftest.py`` you will get + readable ``repr``s of ``check`` callables in the output. """ # allow_unwrapped=True requires: singular exception, exception not being @@ -423,17 +531,24 @@ def __init__( | None ) = None, ): + # The type hint on the `self` and `check` parameters uses different formats + # that are *very* hard to reconcile while adhering to the overloads, so we cast + # it to avoid an error when passing it to super().__init__ + check = cast( + "Callable[[" + "BaseExceptionGroup[ExcT_1|BaseExcT_1|BaseExceptionGroup[BaseExcT_2]]" + "], bool]", + check, + ) + super().__init__(match, check) self.expected_exceptions: tuple[ - type[BaseExcT_co] | Matcher[BaseExcT_co] | RaisesGroup[BaseException], - ..., + type[BaseExcT_co] | Matcher[BaseExcT_co] | RaisesGroup[BaseException], ... ] = ( exception, *other_exceptions, ) - self.flatten_subgroups: bool = flatten_subgroups self.allow_unwrapped = allow_unwrapped - self.match_expr = match - self.check = check + self.flatten_subgroups: bool = flatten_subgroups self.is_baseexceptiongroup = False if allow_unwrapped and other_exceptions: @@ -470,16 +585,15 @@ def __init__( " match a nested structure.", ) self.is_baseexceptiongroup |= exc.is_baseexceptiongroup + exc._nested = True elif isinstance(exc, Matcher): - # The Matcher could match BaseExceptions through the other arguments - # but `self.is_baseexceptiongroup` is only used for printing. - if exc.exception_type is None: - continue - # Matcher __init__ assures it's a subclass of BaseException - self.is_baseexceptiongroup |= not issubclass( - exc.exception_type, - Exception, - ) + if exc.exception_type is not None: + # Matcher __init__ assures it's a subclass of BaseException + self.is_baseexceptiongroup |= not issubclass( + exc.exception_type, + Exception, + ) + exc._nested = True elif isinstance(exc, type) and issubclass(exc, BaseException): self.is_baseexceptiongroup |= not issubclass(exc, Exception) else: @@ -503,6 +617,22 @@ def __enter__(self) -> ExceptionInfo[BaseExceptionGroup[BaseException]]: ) return self.excinfo + def __repr__(self) -> str: + parameters = [ + e.__name__ if isinstance(e, type) else repr(e) + for e in self.expected_exceptions + ] + if self.allow_unwrapped: + parameters.append(f"allow_unwrapped={self.allow_unwrapped}") + if self.flatten_subgroups: + parameters.append(f"flatten_subgroups={self.flatten_subgroups}") + if self.match is not None: + # If no flags were specified, discard the redundant re.compile() here. + parameters.append(f"match={_match_pattern(self.match)!r}") + if self.check is not None: + parameters.append(f"check={repr_callable(self.check)}") + return f"RaisesGroup({', '.join(parameters)})" + def _unroll_exceptions( self, exceptions: Sequence[BaseException], @@ -533,6 +663,7 @@ def matches( exc_val: BaseException | None, ) -> TypeGuard[BaseExceptionGroup[BaseExcT_co]]: """Check if an exception matches the requirements of this RaisesGroup. + If it fails, `RaisesGroup.fail_reason` will be set. Example:: @@ -545,50 +676,245 @@ def matches( assert len(myexc.exceptions) == 1 assert isinstance(myexc.exceptions[0], ValueError) """ + self._fail_reason = None if exc_val is None: + self._fail_reason = "exception is None" return False - # TODO: print/raise why a match fails, in a way that works properly in nested cases - # maybe have a list of strings logging failed matches, that __exit__ can - # recursively step through and print on a failing match. if not isinstance(exc_val, BaseExceptionGroup): - if self.allow_unwrapped: - exp_exc = self.expected_exceptions[0] - if isinstance(exp_exc, Matcher) and exp_exc.matches(exc_val): - return True - if isinstance(exp_exc, type) and isinstance(exc_val, exp_exc): - return True - return False - - if self.match_expr is not None and not re.search( - self.match_expr, - _stringify_exception(exc_val), - ): + # we opt to only print type of the exception here, as the repr would + # likely be quite long + not_group_msg = f"{type(exc_val).__name__!r} is not an exception group" + if len(self.expected_exceptions) > 1: + self._fail_reason = not_group_msg + return False + # if we have 1 expected exception, check if it would work even if + # allow_unwrapped is not set + res = self._check_expected(self.expected_exceptions[0], exc_val) + if res is None and self.allow_unwrapped: + return True + + if res is None: + self._fail_reason = ( + f"{not_group_msg}, but would match with `allow_unwrapped=True`" + ) + elif self.allow_unwrapped: + self._fail_reason = res + else: + self._fail_reason = not_group_msg return False - remaining_exceptions = list(self.expected_exceptions) actual_exceptions: Sequence[BaseException] = exc_val.exceptions if self.flatten_subgroups: actual_exceptions = self._unroll_exceptions(actual_exceptions) - # important to check the length *after* flattening subgroups - if len(actual_exceptions) != len(self.expected_exceptions): + if not self._check_match(exc_val): + old_reason = self._fail_reason + if ( + len(actual_exceptions) == len(self.expected_exceptions) == 1 + and isinstance(expected := self.expected_exceptions[0], type) + and isinstance(actual := actual_exceptions[0], expected) + and self._check_match(actual) + ): + assert self.match is not None, "can't be None if _check_match failed" + assert self._fail_reason is old_reason is not None + self._fail_reason += f", but matched the expected {self._repr_expected(expected)}. You might want RaisesGroup(Matcher({expected.__name__}, match={_match_pattern(self.match)!r}))" + else: + self._fail_reason = old_reason + return False + + # do the full check on expected exceptions + if not self._check_exceptions( + exc_val, + actual_exceptions, + ): + assert self._fail_reason is not None + old_reason = self._fail_reason + # if we're not expecting a nested structure, and there is one, do a second + # pass where we try flattening it + if ( + not self.flatten_subgroups + and not any( + isinstance(e, RaisesGroup) for e in self.expected_exceptions + ) + and any(isinstance(e, BaseExceptionGroup) for e in actual_exceptions) + and self._check_exceptions( + exc_val, + self._unroll_exceptions(exc_val.exceptions), + ) + ): + # only indent if it's a single-line reason. In a multi-line there's already + # indented lines that this does not belong to. + indent = " " if "\n" not in self._fail_reason else "" + self._fail_reason = ( + old_reason + + f"\n{indent}Did you mean to use `flatten_subgroups=True`?" + ) + else: + self._fail_reason = old_reason return False - for e in actual_exceptions: - for rem_e in remaining_exceptions: - if ( - (isinstance(rem_e, type) and isinstance(e, rem_e)) - or (isinstance(rem_e, RaisesGroup) and rem_e.matches(e)) - or (isinstance(rem_e, Matcher) and rem_e.matches(e)) - ): - remaining_exceptions.remove(rem_e) + # Only run `self.check` once we know `exc_val` is of the correct type. + # TODO: if this fails, we should say the *group* did not match + return self._check_check(exc_val) + + @staticmethod + def _check_expected( + expected_type: ( + type[BaseException] | Matcher[BaseException] | RaisesGroup[BaseException] + ), + exception: BaseException, + ) -> str | None: + """Helper method for `RaisesGroup.matches` and `RaisesGroup._check_exceptions` + to check one of potentially several expected exceptions.""" + if isinstance(expected_type, type): + return _check_raw_type(expected_type, exception) + res = expected_type.matches(exception) + if res: + return None + assert expected_type.fail_reason is not None + if expected_type.fail_reason.startswith("\n"): + return f"\n{expected_type!r}: {indent(expected_type.fail_reason, ' ')}" + return f"{expected_type!r}: {expected_type.fail_reason}" + + @staticmethod + def _repr_expected(e: type[BaseException] | AbstractMatcher[BaseException]) -> str: + """Get the repr of an expected type/Matcher/RaisesGroup, but we only want + the name if it's a type""" + if isinstance(e, type): + return _exception_type_name(e) + return repr(e) + + @overload + def _check_exceptions( + self: RaisesGroup[ExcT_1], + _exc_val: Exception, + actual_exceptions: Sequence[Exception], + ) -> TypeGuard[ExceptionGroup[ExcT_1]]: ... + @overload + def _check_exceptions( + self: RaisesGroup[BaseExcT_1], + _exc_val: BaseException, + actual_exceptions: Sequence[BaseException], + ) -> TypeGuard[BaseExceptionGroup[BaseExcT_1]]: ... + + def _check_exceptions( + self, + _exc_val: BaseException, + actual_exceptions: Sequence[BaseException], + ) -> TypeGuard[BaseExceptionGroup[BaseExcT_co]]: + """helper method for RaisesGroup.matches that attempts to pair up expected and actual exceptions""" + # full table with all results + results = ResultHolder(self.expected_exceptions, actual_exceptions) + + # (indexes of) raised exceptions that haven't (yet) found an expected + remaining_actual = list(range(len(actual_exceptions))) + # (indexes of) expected exceptions that haven't found a matching raised + failed_expected: list[int] = [] + # successful greedy matches + matches: dict[int, int] = {} + + # loop over expected exceptions first to get a more predictable result + for i_exp, expected in enumerate(self.expected_exceptions): + for i_rem in remaining_actual: + res = self._check_expected(expected, actual_exceptions[i_rem]) + results.set_result(i_exp, i_rem, res) + if res is None: + remaining_actual.remove(i_rem) + matches[i_exp] = i_rem break else: - return False + failed_expected.append(i_exp) + + # All exceptions matched up successfully + if not remaining_actual and not failed_expected: + return True + + # in case of a single expected and single raised we simplify the output + if 1 == len(actual_exceptions) == len(self.expected_exceptions): + assert not matches + self._fail_reason = res + return False + + # The test case is failing, so we can do a slow and exhaustive check to find + # duplicate matches etc that will be helpful in debugging + for i_exp, expected in enumerate(self.expected_exceptions): + for i_actual, actual in enumerate(actual_exceptions): + if results.has_result(i_exp, i_actual): + continue + results.set_result( + i_exp, i_actual, self._check_expected(expected, actual) + ) + + successful_str = ( + f"{len(matches)} matched exception{'s' if len(matches) > 1 else ''}. " + if matches + else "" + ) - # only run `self.check` once we know `exc_val` is correct. (see the types) - # unfortunately mypy isn't smart enough to recognize the above `for`s as narrowing. - return self.check is None or self.check(exc_val) # type: ignore[arg-type] + # all expected were found + if not failed_expected and results.no_match_for_actual(remaining_actual): + self._fail_reason = f"{successful_str}Unexpected exception(s): {[actual_exceptions[i] for i in remaining_actual]!r}" + return False + # all raised exceptions were expected + if not remaining_actual and results.no_match_for_expected(failed_expected): + self._fail_reason = f"{successful_str}Too few exceptions raised, found no match for: [{', '.join(self._repr_expected(self.expected_exceptions[i]) for i in failed_expected)}]" + return False + + # if there's only one remaining and one failed, and the unmatched didn't match anything else, + # we elect to only print why the remaining and the failed didn't match. + if ( + 1 == len(remaining_actual) == len(failed_expected) + and results.no_match_for_actual(remaining_actual) + and results.no_match_for_expected(failed_expected) + ): + self._fail_reason = f"{successful_str}{results.get_result(failed_expected[0], remaining_actual[0])}" + return False + + # there's both expected and raised exceptions without matches + s = "" + if matches: + s += f"\n{successful_str}" + indent_1 = " " * 2 + indent_2 = " " * 4 + + if not remaining_actual: + s += "\nToo few exceptions raised!" + elif not failed_expected: + s += "\nUnexpected exception(s)!" + + if failed_expected: + s += "\nThe following expected exceptions did not find a match:" + rev_matches = {v: k for k, v in matches.items()} + for i_failed in failed_expected: + s += ( + f"\n{indent_1}{self._repr_expected(self.expected_exceptions[i_failed])}" + ) + for i_actual, actual in enumerate(actual_exceptions): + if results.get_result(i_exp, i_actual) is None: + # we print full repr of match target + s += f"\n{indent_2}It matches {actual!r} which was paired with {self._repr_expected(self.expected_exceptions[rev_matches[i_actual]])}" + + if remaining_actual: + s += "\nThe following raised exceptions did not find a match" + for i_actual in remaining_actual: + s += f"\n{indent_1}{actual_exceptions[i_actual]!r}:" + for i_exp, expected in enumerate(self.expected_exceptions): + res = results.get_result(i_exp, i_actual) + if i_exp in failed_expected: + assert res is not None + if res[0] != "\n": + s += "\n" + s += indent(res, indent_2) + if res is None: + # we print full repr of match target + s += f"\n{indent_2}It matches {self._repr_expected(expected)} which was paired with {actual_exceptions[matches[i_exp]]!r}" + + if len(self.expected_exceptions) == len(actual_exceptions) and possible_match( + results + ): + s += "\nThere exist a possible match when attempting an exhaustive check, but RaisesGroup uses a greedy algorithm. Please make your expected exceptions more stringent with `Matcher` etc so the greedy algorithm can function." + self._fail_reason = s + return False def __exit__( self, @@ -604,8 +930,15 @@ def __exit__( self.excinfo is not None ), "Internal error - should have been constructed in __enter__" - if not self.matches(exc_val): - return False + group_str = ( + "(group)" + if self.allow_unwrapped and not issubclass(exc_type, BaseExceptionGroup) + else "group" + ) + + assert self.matches( + exc_val, + ), f"Raised exception {group_str} did not match: {self._fail_reason}" # Cast to narrow the exception type now that it's verified. exc_info = cast( @@ -628,3 +961,61 @@ def expected_type(self) -> str: raise AssertionError("unknown type") group_type = "Base" if self.is_baseexceptiongroup else "" return f"{group_type}ExceptionGroup({', '.join(subexcs)})" + + +@final +class NotChecked: ... + + +class ResultHolder: + def __init__( + self, + expected_exceptions: tuple[ + type[BaseException] | AbstractMatcher[BaseException], ... + ], + actual_exceptions: Sequence[BaseException], + ) -> None: + self.results: list[list[str | type[NotChecked] | None]] = [ + [NotChecked for _ in expected_exceptions] for _ in actual_exceptions + ] + + def set_result(self, expected: int, actual: int, result: str | None) -> None: + self.results[actual][expected] = result + + def get_result(self, expected: int, actual: int) -> str | None: + res = self.results[actual][expected] + # mypy doesn't support `assert res is not NotChecked` + assert not isinstance(res, type) + return res + + def has_result(self, expected: int, actual: int) -> bool: + return self.results[actual][expected] is not NotChecked + + def no_match_for_expected(self, expected: list[int]) -> bool: + for i in expected: + for actual_results in self.results: + assert actual_results[i] is not NotChecked + if actual_results[i] is None: + return False + return True + + def no_match_for_actual(self, actual: list[int]) -> bool: + for i in actual: + for res in self.results[i]: + assert res is not NotChecked + if res is None: + return False + return True + + +def possible_match(results: ResultHolder, used: set[int] | None = None) -> bool: + if used is None: + used = set() + curr_row = len(used) + if curr_row == len(results.results): + return True + + for i, val in enumerate(results.results[curr_row]): + if val is None and i not in used and possible_match(results, used | {i}): + return True + return False