diff --git a/playwright/_impl/_assertions.py b/playwright/_impl/_assertions.py index 8ec657531..87163fd04 100644 --- a/playwright/_impl/_assertions.py +++ b/playwright/_impl/_assertions.py @@ -13,7 +13,19 @@ # limitations under the License. import collections.abc -from typing import Any, List, Optional, Pattern, Sequence, Union +from types import TracebackType +from typing import ( + TYPE_CHECKING, + Any, + Generic, + List, + Optional, + Pattern, + Sequence, + Type, + TypeVar, + Union, +) from urllib.parse import urljoin from playwright._impl._api_structures import ( @@ -29,6 +41,10 @@ from playwright._impl._page import Page from playwright._impl._str_utils import escape_regex_flags +if TYPE_CHECKING: + from ..async_api import Expect as AsyncExpect + from ..sync_api import Expect as SyncExpect + class AssertionsBase: def __init__( @@ -37,6 +53,7 @@ def __init__( timeout: float = None, is_not: bool = False, message: Optional[str] = None, + soft_context: Optional["SoftAssertionContext"] = None, ) -> None: self._actual_locator = locator self._loop = locator._loop @@ -44,6 +61,7 @@ def __init__( self._timeout = timeout self._is_not = is_not self._custom_message = message + self._soft_context = soft_context async def _expect_impl( self, @@ -71,9 +89,13 @@ async def _expect_impl( out_message = ( f"{message} '{expected}'" if expected is not None else f"{message}" ) - raise AssertionError( + error = AssertionError( f"{out_message}\nActual value: {actual} {format_call_log(result.get('log'))}" ) + if self._soft_context is not None: + self._soft_context.add_failure(error) + else: + raise error class PageAssertions(AssertionsBase): @@ -83,14 +105,19 @@ def __init__( timeout: float = None, is_not: bool = False, message: Optional[str] = None, + soft_context: Optional["SoftAssertionContext"] = None, ) -> None: - super().__init__(page.locator(":root"), timeout, is_not, message) + super().__init__(page.locator(":root"), timeout, is_not, message, soft_context) self._actual_page = page @property def _not(self) -> "PageAssertions": return PageAssertions( - self._actual_page, self._timeout, not self._is_not, self._custom_message + self._actual_page, + self._timeout, + not self._is_not, + self._custom_message, + self._soft_context, ) async def to_have_title( @@ -148,14 +175,19 @@ def __init__( timeout: float = None, is_not: bool = False, message: Optional[str] = None, + soft_context: Optional["SoftAssertionContext"] = None, ) -> None: - super().__init__(locator, timeout, is_not, message) + super().__init__(locator, timeout, is_not, message, soft_context) self._actual_locator = locator @property def _not(self) -> "LocatorAssertions": return LocatorAssertions( - self._actual_locator, self._timeout, not self._is_not, self._custom_message + self._actual_locator, + self._timeout, + not self._is_not, + self._custom_message, + self._soft_context, ) async def to_contain_text( @@ -848,6 +880,7 @@ def __init__( timeout: float = None, is_not: bool = False, message: Optional[str] = None, + soft_context: Optional["SoftAssertionContext"] = None, ) -> None: self._loop = response._loop self._dispatcher_fiber = response._dispatcher_fiber @@ -855,11 +888,16 @@ def __init__( self._is_not = is_not self._actual = response self._custom_message = message + self._soft_context = soft_context @property def _not(self) -> "APIResponseAssertions": return APIResponseAssertions( - self._actual, self._timeout, not self._is_not, self._custom_message + self._actual, + self._timeout, + not self._is_not, + self._custom_message, + self._soft_context, ) async def to_be_ok( @@ -880,7 +918,11 @@ async def to_be_ok( if text is not None: out_message += f"\n Response Text:\n{text[:1000]}" - raise AssertionError(out_message) + error = AssertionError(out_message) + if self._soft_context is not None: + self._soft_context.add_failure(error) + else: + raise error async def not_to_be_ok(self) -> None: __tracebackhide__ = True @@ -933,3 +975,57 @@ def to_expected_text_values( else: raise Error("value must be a string or regular expression") return out + + +class SoftAssertionContext: + def __init__(self) -> None: + self._failures: List[Exception] = [] + + def __repr__(self) -> str: + return f"" + + def add_failure(self, error: Exception) -> None: + self._failures.append(error) + + def has_failures(self) -> bool: + return bool(self._failures) + + def get_failure_messages(self) -> str: + return "\n".join( + f"{i}. {str(error)}" for i, error in enumerate(self._failures, 1) + ) + + +E = TypeVar("E", "SyncExpect", "AsyncExpect") + + +class SoftAssertionContextManager(Generic[E]): + def __init__(self, expect: E, context: SoftAssertionContext) -> None: + self._expect: E = expect + self._context = context + + def __enter__(self) -> E: + self._expect._soft_context = self._context + return self._expect + + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType], + ) -> None: + __tracebackhide__ = True + + if self._context.has_failures(): + if exc_type is not None and exc_val is not None: + failure_message = ( + f"{str(exc_val)}" + f"\n\nThe above exception occurred within soft assertion block." + f"\n\nSoft assertion failures:\n{self._context.get_failure_messages()}" + ) + exc_val.args = (failure_message,) + exc_val.args[1:] + return + + raise AssertionError( + f"Soft assertion failures\n{self._context.get_failure_messages()}" + ) diff --git a/playwright/async_api/__init__.py b/playwright/async_api/__init__.py index be918f53c..16c618c6c 100644 --- a/playwright/async_api/__init__.py +++ b/playwright/async_api/__init__.py @@ -28,6 +28,10 @@ ) from playwright._impl._assertions import LocatorAssertions as LocatorAssertionsImpl from playwright._impl._assertions import PageAssertions as PageAssertionsImpl +from playwright._impl._assertions import ( + SoftAssertionContext, + SoftAssertionContextManager, +) from playwright.async_api._context_manager import PlaywrightContextManager from playwright.async_api._generated import ( Accessibility, @@ -94,6 +98,7 @@ class Expect: def __init__(self) -> None: self._timeout: Optional[float] = None + self._soft_context: Optional[SoftAssertionContext] = None def set_options(self, timeout: Optional[float] = _unset) -> None: """ @@ -108,6 +113,11 @@ def set_options(self, timeout: Optional[float] = _unset) -> None: if timeout is not self._unset: self._timeout = timeout + def soft(self) -> SoftAssertionContextManager: + expect = Expect() + expect._timeout = self._timeout + return SoftAssertionContextManager(expect, SoftAssertionContext()) + @overload def __call__( self, actual: Page, message: Optional[str] = None @@ -128,16 +138,29 @@ def __call__( ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]: if isinstance(actual, Page): return PageAssertions( - PageAssertionsImpl(actual._impl_obj, self._timeout, message=message) + PageAssertionsImpl( + actual._impl_obj, + self._timeout, + message=message, + soft_context=self._soft_context, + ) ) elif isinstance(actual, Locator): return LocatorAssertions( - LocatorAssertionsImpl(actual._impl_obj, self._timeout, message=message) + LocatorAssertionsImpl( + actual._impl_obj, + self._timeout, + message=message, + soft_context=self._soft_context, + ) ) elif isinstance(actual, APIResponse): return APIResponseAssertions( APIResponseAssertionsImpl( - actual._impl_obj, self._timeout, message=message + actual._impl_obj, + self._timeout, + message=message, + soft_context=self._soft_context, ) ) raise ValueError(f"Unsupported type: {type(actual)}") diff --git a/playwright/sync_api/__init__.py b/playwright/sync_api/__init__.py index 136433982..65250805a 100644 --- a/playwright/sync_api/__init__.py +++ b/playwright/sync_api/__init__.py @@ -28,6 +28,10 @@ ) from playwright._impl._assertions import LocatorAssertions as LocatorAssertionsImpl from playwright._impl._assertions import PageAssertions as PageAssertionsImpl +from playwright._impl._assertions import ( + SoftAssertionContext, + SoftAssertionContextManager, +) from playwright.sync_api._context_manager import PlaywrightContextManager from playwright.sync_api._generated import ( Accessibility, @@ -94,6 +98,7 @@ class Expect: def __init__(self) -> None: self._timeout: Optional[float] = None + self._soft_context: Optional[SoftAssertionContext] = None def set_options(self, timeout: Optional[float] = _unset) -> None: """ @@ -108,6 +113,11 @@ def set_options(self, timeout: Optional[float] = _unset) -> None: if timeout is not self._unset: self._timeout = timeout + def soft(self) -> SoftAssertionContextManager: + expect = Expect() + expect._timeout = self._timeout + return SoftAssertionContextManager(expect, SoftAssertionContext()) + @overload def __call__( self, actual: Page, message: Optional[str] = None @@ -128,16 +138,29 @@ def __call__( ) -> Union[PageAssertions, LocatorAssertions, APIResponseAssertions]: if isinstance(actual, Page): return PageAssertions( - PageAssertionsImpl(actual._impl_obj, self._timeout, message=message) + PageAssertionsImpl( + actual._impl_obj, + self._timeout, + message=message, + soft_context=self._soft_context, + ) ) elif isinstance(actual, Locator): return LocatorAssertions( - LocatorAssertionsImpl(actual._impl_obj, self._timeout, message=message) + LocatorAssertionsImpl( + actual._impl_obj, + self._timeout, + message=message, + soft_context=self._soft_context, + ) ) elif isinstance(actual, APIResponse): return APIResponseAssertions( APIResponseAssertionsImpl( - actual._impl_obj, self._timeout, message=message + actual._impl_obj, + self._timeout, + message=message, + soft_context=self._soft_context, ) ) raise ValueError(f"Unsupported type: {type(actual)}") diff --git a/tests/async/test_assertions.py b/tests/async/test_assertions.py index 06292aa9b..c96611fdc 100644 --- a/tests/async/test_assertions.py +++ b/tests/async/test_assertions.py @@ -1079,3 +1079,172 @@ async def test_to_have_role(page: Page) -> None: with pytest.raises(Error) as excinfo: await expect(page.locator("div")).to_have_role(re.compile(r"button|checkbox")) # type: ignore assert '"role" argument in to_have_role must be a string' in str(excinfo.value) + + +async def test_should_collect_soft_failures(page: Page) -> None: + await page.set_content( + """ +
Text1
+
Text2
+ """ + ) + + with pytest.raises(AssertionError) as excinfo: + with expect.soft() as soft_expect: + await soft_expect(page.locator("#div1")).to_have_text("wrong", timeout=1000) + await soft_expect(page.locator("#div2")).to_have_text( + "wrong2", timeout=1000 + ) + assert "1. Locator expected to have text 'wrong'" in str(excinfo.value) + assert "2. Locator expected to have text 'wrong2'" in str(excinfo.value) + + +async def test_should_support_custom_message_in_soft_assertions(page: Page) -> None: + await page.set_content('
Text1
') + + with pytest.raises(AssertionError) as excinfo: + with expect.soft() as soft_expect: + await soft_expect(page.locator("#div1"), "Custom message 1").to_have_text( + "wrong", timeout=1000 + ) + await soft_expect(page.locator("#div1"), "Custom message 2").to_have_text( + "also wrong", timeout=1000 + ) + assert "1. Custom message 1" in str(excinfo.value) + assert "2. Custom message 2" in str(excinfo.value) + + +async def test_should_work_with_different_assertion_types( + page: Page, server: Server +) -> None: + await page.set_content( + """ +
Text1
+ Page Title + """ + ) + + with pytest.raises(AssertionError) as excinfo: + with expect.soft() as soft_expect: + await soft_expect(page.locator("#div1")).to_have_text("wrong", timeout=1000) + await soft_expect(page).to_have_title("wrong title", timeout=1000) + response = await page.request.get(server.PREFIX + "/non-existent") + await soft_expect(response).to_be_ok() + + error_text = str(excinfo.value) + assert "1. Locator expected to have text 'wrong'" in error_text + assert "2. Page title expected to be 'wrong title'" in error_text + assert "3. Response status expected to be within [200..299] range" in error_text + + +async def test_should_report_soft_failures_in_order(page: Page) -> None: + await page.set_content( + """ +
First
+
Second
+
Third
+ """ + ) + + with pytest.raises(AssertionError) as excinfo: + with expect.soft() as soft_expect: + await soft_expect(page.locator("#third")).to_have_text( + "wrong3", timeout=1000 + ) + await soft_expect(page.locator("#first")).to_have_text( + "wrong1", timeout=1000 + ) + await soft_expect(page.locator("#second")).to_have_text( + "wrong2", timeout=1000 + ) + + error_text = str(excinfo.value) + first_pos = error_text.find("wrong3") + second_pos = error_text.find("wrong1") + third_pos = error_text.find("wrong2") + assert ( + first_pos < second_pos < third_pos + ), "Errors should be reported in order of occurrence" + + +async def test_should_pass_successful_soft_assertions(page: Page) -> None: + await page.set_content("
Text
") + + try: + with expect.soft() as soft_expect: + await soft_expect(page.locator("div")).to_have_text("Text") + await soft_expect(page.locator("div")).to_be_visible() + except Exception as e: + pytest.fail(f"Should not have raised an exception, but got: {e}") + + +async def test_should_respect_timeout_in_soft_assertions(page: Page) -> None: + await page.set_content("
Text
") + + original_timeout = expect._timeout + try: + expect.set_options(timeout=2000) + with pytest.raises(AssertionError) as excinfo: + with expect.soft() as soft_expect: + await soft_expect(page.locator("div")).to_have_text( + "wrong", timeout=100 + ) + await soft_expect(page.locator("div")).to_have_text("also wrong") + + error_text = str(excinfo.value) + assert "timeout 100ms" in error_text + assert "timeout 2000ms" in error_text + finally: + expect.set_options(timeout=original_timeout) + + +async def test_should_include_soft_failures_in_error_message(page: Page) -> None: + await page.set_content("
Text
") + + with pytest.raises(ValueError) as excinfo: + with expect.soft() as soft_expect: + await soft_expect(page.locator("div")).to_have_text("wrong", timeout=1000) + raise ValueError("Some error") + + error_text = str(excinfo.value) + assert "Some error" in error_text + assert "expected to have text 'wrong'" in error_text + + +async def test_should_collect_negated_soft_failures(page: Page) -> None: + await page.set_content( + """ +
Text1
+
Text2
+
Text3
+ """ + ) + + with pytest.raises(AssertionError) as excinfo: + with expect.soft() as soft_expect: + await soft_expect(page.locator("#div1")).not_to_have_text( + "Text1", timeout=1000 + ) + await soft_expect(page.locator("#div2")).not_to_be_visible(timeout=1000) + await soft_expect(page.locator("#div3")).not_to_be_attached(timeout=1000) + + error_text = str(excinfo.value) + assert "1. Locator expected not to have text 'Text1'" in error_text + assert "2. Locator expected not to be visible" in error_text + assert "3. Locator expected not to be attached" in error_text + + +async def test_should_pass_negated_soft_assertions(page: Page) -> None: + await page.set_content( + """ +
Text1
+ """ + ) + + try: + with expect.soft() as soft_expect: + await soft_expect(page.locator("#div1")).not_to_have_text("wrong") + await soft_expect(page.locator("#not-exists")).not_to_be_visible() + await soft_expect(page.locator("#not-exists")).not_to_be_attached() + except Exception as e: + pytest.fail(f"Should not have raised an exception, but got: {e}") diff --git a/tests/sync/test_assertions.py b/tests/sync/test_assertions.py index 6aaffd49b..2f15d6a60 100644 --- a/tests/sync/test_assertions.py +++ b/tests/sync/test_assertions.py @@ -1043,3 +1043,158 @@ def test_to_have_role(page: Page) -> None: with pytest.raises(Error) as excinfo: expect(page.locator("div")).to_have_role(re.compile(r"button|checkbox")) # type: ignore assert '"role" argument in to_have_role must be a string' in str(excinfo.value) + + +def test_should_collect_soft_failures(page: Page) -> None: + page.set_content( + """ +
Text1
+
Text2
+ """ + ) + + with pytest.raises(AssertionError) as excinfo: + with expect.soft() as soft_expect: + soft_expect(page.locator("#div1")).to_have_text("wrong", timeout=1000) + soft_expect(page.locator("#div2")).to_have_text("wrong2", timeout=1000) + assert "1. Locator expected to have text 'wrong'" in str(excinfo.value) + assert "2. Locator expected to have text 'wrong2'" in str(excinfo.value) + + +def test_should_support_custom_message_in_soft_assertions(page: Page) -> None: + page.set_content('
Text1
') + + with pytest.raises(AssertionError) as excinfo: + with expect.soft() as soft_expect: + soft_expect(page.locator("#div1"), "Custom message 1").to_have_text( + "wrong", timeout=1000 + ) + soft_expect(page.locator("#div1"), "Custom message 2").to_have_text( + "also wrong", timeout=1000 + ) + assert "1. Custom message 1" in str(excinfo.value) + assert "2. Custom message 2" in str(excinfo.value) + + +def test_should_work_with_different_assertion_types(page: Page, server: Server) -> None: + page.set_content( + """ +
Text1
+ Page Title + """ + ) + + with pytest.raises(AssertionError) as excinfo: + with expect.soft() as soft_expect: + soft_expect(page.locator("#div1")).to_have_text("wrong", timeout=1000) + soft_expect(page).to_have_title("wrong title", timeout=1000) + response = page.request.get(server.PREFIX + "/non-existent") + soft_expect(response).to_be_ok() + + error_text = str(excinfo.value) + assert "1. Locator expected to have text 'wrong'" in error_text + assert "2. Page title expected to be 'wrong title'" in error_text + assert "3. Response status expected to be within [200..299] range" in error_text + + +def test_should_report_soft_failures_in_order(page: Page) -> None: + page.set_content( + """ +
First
+
Second
+
Third
+ """ + ) + + with pytest.raises(AssertionError) as excinfo: + with expect.soft() as soft_expect: + soft_expect(page.locator("#third")).to_have_text("wrong3", timeout=1000) + soft_expect(page.locator("#first")).to_have_text("wrong1", timeout=1000) + soft_expect(page.locator("#second")).to_have_text("wrong2", timeout=1000) + + error_text = str(excinfo.value) + first_pos = error_text.find("wrong3") + second_pos = error_text.find("wrong1") + third_pos = error_text.find("wrong2") + assert ( + first_pos < second_pos < third_pos + ), "Errors should be reported in order of occurrence" + + +def test_should_pass_successful_soft_assertions(page: Page) -> None: + page.set_content("
Text
") + + try: + with expect.soft() as soft_expect: + soft_expect(page.locator("div")).to_have_text("Text") + soft_expect(page.locator("div")).to_be_visible() + except Exception as e: + pytest.fail(f"Should not have raised an exception, but got: {e}") + + +def test_should_respect_timeout_in_soft_assertions(page: Page) -> None: + page.set_content("
Text
") + + original_timeout = expect._timeout + try: + expect.set_options(timeout=2000) + with pytest.raises(AssertionError) as excinfo: + with expect.soft() as soft_expect: + soft_expect(page.locator("div")).to_have_text("wrong", timeout=100) + soft_expect(page.locator("div")).to_have_text("also wrong") + + error_text = str(excinfo.value) + assert "timeout 100ms" in error_text + assert "timeout 2000ms" in error_text + finally: + expect.set_options(timeout=original_timeout) + + +def test_should_include_soft_failures_in_error_message(page: Page) -> None: + page.set_content("
Text
") + + with pytest.raises(ValueError) as excinfo: + with expect.soft() as soft_expect: + soft_expect(page.locator("div")).to_have_text("wrong", timeout=1000) + raise ValueError("Some error") + + error_text = str(excinfo.value) + assert "Some error" in error_text + assert "expected to have text 'wrong'" in error_text + + +def test_should_collect_negated_soft_failures(page: Page) -> None: + page.set_content( + """ +
Text1
+
Text2
+
Text3
+ """ + ) + + with pytest.raises(AssertionError) as excinfo: + with expect.soft() as soft_expect: + soft_expect(page.locator("#div1")).not_to_have_text("Text1", timeout=1000) + soft_expect(page.locator("#div2")).not_to_be_visible(timeout=1000) + soft_expect(page.locator("#div3")).not_to_be_attached(timeout=1000) + + error_text = str(excinfo.value) + assert "1. Locator expected not to have text 'Text1'" in error_text + assert "2. Locator expected not to be visible" in error_text + assert "3. Locator expected not to be attached" in error_text + + +def test_should_pass_negated_soft_assertions(page: Page) -> None: + page.set_content( + """ +
Text1
+ """ + ) + + try: + with expect.soft() as soft_expect: + soft_expect(page.locator("#div1")).not_to_have_text("wrong") + soft_expect(page.locator("#not-exists")).not_to_be_visible() + soft_expect(page.locator("#not-exists")).not_to_be_attached() + except Exception as e: + pytest.fail(f"Should not have raised an exception, but got: {e}")