diff --git a/README.md b/README.md index e307b4c3..d8452a44 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Slash | | | |-----------------------|------------------------------------------------------------------------------------| | Build Status | ![Build Status](https://secure.travis-ci.org/getslash/slash.png?branch=master,dev) | -| Supported Versions | ![Supported Versions](https://img.shields.io/badge/python-2.7%2C3.3%2C3.4%2C3.5%2C3.6-green.svg) | +| Supported Versions | ![Supported Versions](https://img.shields.io/badge/python-2.7%2C3.5%2C3.6%2C3.7-green.svg) | | Latest Version | ![Latest Version](https://img.shields.io/pypi/v/slash.svg) | | Test Coverage | ![Coverage Status](https://img.shields.io/coveralls/getslash/slash/develop.svg) | diff --git a/doc/changelog.rst b/doc/changelog.rst index d195ad40..f174c351 100644 --- a/doc/changelog.rst +++ b/doc/changelog.rst @@ -1,6 +1,9 @@ Changelog ========= +* :bug:`930` Restore behavior of exceptions propagating out of the test_start or test_end hooks. Correct behavior is for those to fail the test (thanks @pierreluctg) +* :bug:`934` Parallel sessions now honor fatal exceptions encountered in worker sessions +* :bug:`928` Fixed a bug causing requirements to leak across sibling test classes * :release:`1.7.9 <09-03-2019>` * :bug:`-` Revert console coloring change, as it does not behave consistently across different terminals * :release:`1.7.8 <04-03-2019>` diff --git a/slash/core/fixtures/fixture_store.py b/slash/core/fixtures/fixture_store.py index 85cb4533..31efcb66 100644 --- a/slash/core/fixtures/fixture_store.py +++ b/slash/core/fixtures/fixture_store.py @@ -72,14 +72,13 @@ def call_with_fixtures(self, test_func, namespace, trigger_test_start=False, tri if trigger_test_start: for fixture in self.iter_active_fixtures(): fixture.call_test_start() + + return test_func(**kwargs) finally: - try: - return test_func(**kwargs) # pylint: disable=lost-exception - finally: - if trigger_test_end: - for fixture in self.iter_active_fixtures(): - with handling_exceptions(swallow=True): - fixture.call_test_end() + if trigger_test_end: + for fixture in self.iter_active_fixtures(): + with handling_exceptions(swallow=True): + fixture.call_test_end() def get_required_fixture_names(self, test_func): """Returns a list of fixture names needed by test_func. diff --git a/slash/core/requirements.py b/slash/core/requirements.py index affaff29..d1a41a08 100644 --- a/slash/core/requirements.py +++ b/slash/core/requirements.py @@ -1,3 +1,5 @@ +from ..utils.python import resolve_underlying_function + _SLASH_REQUIRES_KEY_NAME = '__slash_requirements__' @@ -12,18 +14,34 @@ def requires(req, message=None): else: assert message is None, 'Cannot specify message when passing Requirement objects to slash.requires' - def decorator(func): - reqs = getattr(func, _SLASH_REQUIRES_KEY_NAME, None) - if reqs is None: - reqs = [] - setattr(func, _SLASH_REQUIRES_KEY_NAME, reqs) + def decorator(func_or_class): + reqs = _get_requirements_list(func_or_class) reqs.append(req) - return func + return func_or_class return decorator +def _get_requirements_list(thing, create=True): + + thing = resolve_underlying_function(thing) + existing = getattr(thing, _SLASH_REQUIRES_KEY_NAME, None) + + key = id(thing) + + + if existing is None or key != existing[0]: + new_reqs = (key, [] if existing is None else existing[1][:]) + if create: + setattr(thing, _SLASH_REQUIRES_KEY_NAME, new_reqs) + assert thing.__slash_requirements__ is new_reqs + returned = new_reqs[1] + else: + returned = existing[1] + + return returned + def get_requirements(test): - return list(getattr(test, _SLASH_REQUIRES_KEY_NAME, [])) + return list(_get_requirements_list(test, create=False)) class Requirement(object): diff --git a/slash/core/test.py b/slash/core/test.py index e7bf3ffd..8c45234e 100644 --- a/slash/core/test.py +++ b/slash/core/test.py @@ -118,7 +118,7 @@ def run(self): # pylint: disable=E0202 method = self.get_test_function() with bound_parametrizations_context(self._variation, self._fixture_store, self._fixture_namespace): _call_with_fixtures = functools.partial(self._fixture_store.call_with_fixtures, namespace=self._fixture_namespace) - _call_with_fixtures(self.before, trigger_test_start=True) + _call_with_fixtures(self.before) try: with handling_exceptions(): result = _call_with_fixtures(method, trigger_test_start=True) diff --git a/slash/parallel/server.py b/slash/parallel/server.py index a1e31912..8687131f 100644 --- a/slash/parallel/server.py +++ b/slash/parallel/server.py @@ -178,8 +178,9 @@ def finished_test(self, client_id, result_dict): with _get_test_context(self.tests[test_index], logging=False) as (result, _): result.deserialize(result_dict) context.session.reporter.report_test_end(self.tests[test_index], result) - if not result.is_success(allow_skips=True) and config.root.run.stop_on_error: - _logger.debug("Stopping (run.stop_on_error==True)") + if result.has_fatal_exception() or (not result.is_success(allow_skips=True) and config.root.run.stop_on_error): + _logger.debug("Server stops serving tests, run.stop_on_error: {}, result.has_fatal_exception: {}", + config.root.run.stop_on_error, result.has_fatal_exception()) self.state = ServerStates.STOP_TESTS_SERVING self._mark_unrun_tests() else: diff --git a/slash/utils/python.py b/slash/utils/python.py index 67f53539..7ac54370 100644 --- a/slash/utils/python.py +++ b/slash/utils/python.py @@ -105,3 +105,15 @@ def call_all_raise_first(_funcs, *args, **kwargs): exc_info = sys.exc_info() if exc_info is not None: reraise(*exc_info) + + +def resolve_underlying_function(thing): + """Gets the underlying (real) function for functions, wrapped functions, methods, etc. + Returns the same object for things that are not functions + """ + while True: + wrapped = getattr(thing, "__func__", None) or getattr(thing, "__wrapped__", None) or getattr(thing, "__wraps__", None) + if wrapped is None: + break + thing = wrapped + return thing diff --git a/tests/test_fixture_start_end_test.py b/tests/test_fixture_start_end_test.py index 739d191e..df8a5657 100644 --- a/tests/test_fixture_start_end_test.py +++ b/tests/test_fixture_start_end_test.py @@ -68,26 +68,12 @@ def test_start(*_, **__): return param - @slash.fixture(scope='module') - def fixture_failing(this): - @this.test_start - def test_start(*_, **__): - raise AssertionError() - - return None - class SomeTest(slash.Test): @slash.parametrize("param", [10, 20, 30]) - def test_something(self, param, fixture, fixture_failing): # pylint: disable=unused-argument - slash.context.result.data["value"] = param + fixture - - suite_builder.build().run().assert_all(6).exception(ZeroDivisionError).with_data( - [ - {"value": 11}, {"value": 12}, - {"value": 21}, {"value": 22}, - {"value": 31}, {"value": 32} - ] - ) + def test_something(self, param, fixture): # pylint: disable=unused-argument + slash.add_error("Not supposed to reach here").mark_fatal() + + suite_builder.build().run().assert_all(6).exception(ZeroDivisionError) def test_fixture_start_test_raises_exception_w_before(suite_builder): @@ -105,26 +91,12 @@ def test_start(*_, **__): return param - @slash.fixture(scope='module') - def fixture_failing(this): - @this.test_start - def test_start(*_, **__): - raise AssertionError() - - return None - class SomeTest(slash.Test): - def before(self, fixture, fixture_failing): # pylint: disable=unused-argument,arguments-differ - self.data = fixture + def before(self, fixture): # pylint: disable=unused-argument,arguments-differ + self.data = 1 @slash.parametrize("param", [10, 20, 30]) - def test_something(self, param): - slash.context.result.data["value"] = param + self.data - - suite_builder.build().run().assert_all(6).exception(ZeroDivisionError).with_data( - [ - {"value": 11}, {"value": 12}, - {"value": 21}, {"value": 22}, - {"value": 31}, {"value": 32} - ] - ) + def test_something(self, param): # pylint: disable=unused-argument + slash.add_error("Not supposed to reach here").mark_fatal() + + suite_builder.build().run().assert_all(6).exception(ZeroDivisionError) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index bf418979..6242f7f8 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -201,6 +201,15 @@ def test_child_errors_in_cleanup_are_session_errors(parallel_suite): assert not session_results.results.is_success() assert session_results.parallel_manager.server.worker_error_reported +def test_child_fatal_error_terminates_session(parallel_suite): + parallel_suite[0].expect_failure() + parallel_suite[0].append_line("import slash") + parallel_suite[0].append_line("slash.add_error('bla').mark_fatal()") + session_results = parallel_suite.run(num_workers=1, verify=False).session + first_result = session_results.results[0] + assert first_result.is_error() and first_result.has_fatal_errors() + assert session_results.results.get_num_not_run() == len(parallel_suite) - 1 + def test_traceback_vars(parallel_suite): #code to be inserted: # def test_traceback_frames(): diff --git a/tests/test_python_utils.py b/tests/test_python_utils.py index daed9988..1d54b6de 100644 --- a/tests/test_python_utils.py +++ b/tests/test_python_utils.py @@ -1,7 +1,13 @@ # pylint: disable=redefined-outer-name import pytest +from slash._compat import PY2 -from slash.utils.python import call_all_raise_first +if PY2: + from slash.utils.python import wraps +else: + from functools import wraps + +from slash.utils.python import call_all_raise_first, resolve_underlying_function def test_call_all_raise_first(funcs): @@ -35,3 +41,54 @@ class CustomException(Exception): return self.exc_type return [Func() for _ in range(10)] + + +@pytest.mark.parametrize('class_method', [True, False]) +def test_resolve_underlying_function_method(class_method): + if class_method: + decorator = classmethod + else: + decorator = lambda f: f + + class Blap(object): + + @decorator + def method(self): + pass + + resolved = resolve_underlying_function(Blap.method) + assert resolved is resolve_underlying_function(Blap.method) # stable + assert not hasattr(resolved, '__func__') + assert resolved.__name__ == 'method' + + +@pytest.mark.parametrize('thing', [object(), object, None, 2, "string"]) +def test_resolve_underlying_function_method_no_op(thing): + assert resolve_underlying_function(thing) is thing + + +def _example_decorator(func): + @wraps(func) + def new_func(): + pass + + return new_func + +def test_resolve_underlying_decorator_regular_func(): + + def orig(): + pass + decorated = _example_decorator(orig) + assert resolve_underlying_function(decorated) is orig + +def test_resolve_underlying_decorator_method(): + + class Blap(object): + + def orig(self): + pass + + decorated = _example_decorator(orig) + + assert resolve_underlying_function(Blap.decorated) is resolve_underlying_function(Blap.orig) + assert resolve_underlying_function(Blap.decorated).__name__ == 'orig' diff --git a/tests/test_requirements.py b/tests/test_requirements.py index ee1d6d6a..cdc3664b 100644 --- a/tests/test_requirements.py +++ b/tests/test_requirements.py @@ -1,5 +1,4 @@ # pylint: disable=redefined-outer-name - import gossip import pytest import slash @@ -7,16 +6,22 @@ from .utils import make_runnable_tests from .utils.suite_writer.suite import Suite from .utils.code_formatter import CodeFormatter -from slash._compat import ExitStack +from slash._compat import ExitStack, PY2 + +if PY2: + from slash.utils.python import wraps +else: + from functools import wraps -_UNMET_REQ_DECORATOR = 'slash.requires(lambda: False)' -_MET_REQ_DECORATOR = 'slash.requires(lambda: True)' +_UNMET_REQ_DECORATOR = "slash.requires(lambda: False)" +_MET_REQ_DECORATOR = "slash.requires(lambda: True)" def test_ensure_requirements_called_eagerly(checkpoint1, checkpoint2): def predicate_1(): checkpoint1() return False + def predicate_2(): checkpoint2() return False @@ -29,48 +34,62 @@ def test_something(): with slash.Session() as session: with session.get_started_context(): slash.runner.run_tests(make_runnable_tests(test_something)) - [result] = [res for res in session.results.iter_all_results() if not res.is_global_result()] + [result] = [ + res for res in session.results.iter_all_results() if not res.is_global_result() + ] assert result.is_skip() assert checkpoint1.called assert checkpoint2.called + def test_requirements_raises_exception(suite, suite_test): @suite_test.file.append_body def __code__(): # pylint: disable=unused-variable def fail_predicate(): # pylint: disable=unused-variable - raise Exception('Failing') - suite_test.add_decorator('slash.requires(fail_predicate)') + raise Exception("Failing") + + suite_test.add_decorator("slash.requires(fail_predicate)") suite_test.expect_error() summary = suite.run() assert not summary.session.results.is_success(allow_skips=True) def test_requirements_mismatch_session_success(suite, suite_test): - suite_test.add_decorator('slash.requires(False)') + suite_test.add_decorator("slash.requires(False)") suite_test.expect_skip() summary = suite.run() assert summary.session.results.is_success(allow_skips=True) -@pytest.mark.parametrize('requirement_fullfilled', [True, False]) -@pytest.mark.parametrize('use_message', [True, False]) -@pytest.mark.parametrize('use_fixtures', [True, False]) -@pytest.mark.parametrize('message_in_retval', [True, False]) -def test_requirements(suite, suite_test, requirement_fullfilled, use_fixtures, use_message, message_in_retval): +@pytest.mark.parametrize("requirement_fullfilled", [True, False]) +@pytest.mark.parametrize("use_message", [True, False]) +@pytest.mark.parametrize("use_fixtures", [True, False]) +@pytest.mark.parametrize("message_in_retval", [True, False]) +def test_requirements( + suite, + suite_test, + requirement_fullfilled, + use_fixtures, + use_message, + message_in_retval, +): message = "requires something very important" if use_message and message_in_retval: - retval = '({}, {!r})'.format(requirement_fullfilled, message) + retval = "({}, {!r})".format(requirement_fullfilled, message) else: retval = requirement_fullfilled - suite_test.add_decorator('slash.requires((lambda: {}), {!r})'.format(retval, message if use_message and not message_in_retval else '')) + suite_test.add_decorator( + "slash.requires((lambda: {}), {!r})".format( + retval, message if use_message and not message_in_retval else "" + ) + ) if not requirement_fullfilled: suite_test.expect_skip() if use_fixtures: - suite_test.depend_on_fixture( - suite.slashconf.add_fixture()) + suite_test.depend_on_fixture(suite.slashconf.add_fixture()) results = suite.run() if requirement_fullfilled: assert results[suite_test].is_success() @@ -88,11 +107,10 @@ def test_requirements_functions_no_message(suite, suite_test): results = suite.run() result = results[suite_test] [skip] = result.get_skips() - assert 'lambda' in skip + assert "lambda" in skip def test_requirements_on_class(): - def req1(): pass @@ -101,7 +119,6 @@ def req2(): @slash.requires(req1) class Test(slash.Test): - @slash.requires(req2) def test_something(self): pass @@ -109,46 +126,51 @@ def test_something(self): with slash.Session(): [test] = make_runnable_tests(Test) # pylint: disable=unbalanced-tuple-unpacking - - assert set([r._req for r in test.get_requirements()]) == set([req1, req2]) # pylint: disable=protected-access + assert set([r._req for r in test.get_requirements()]) == set( # pylint: disable=protected-access + [req1, req2] + ) @pytest.fixture def filename_test_fixture(tmpdir): - returned = str(tmpdir.join('testfile.py')) + returned = str(tmpdir.join("testfile.py")) - with open(returned, 'w') as f: + with open(returned, "w") as f: with ExitStack() as stack: code = CodeFormatter(f) - code.writeln('import slash') - code.writeln('@slash.fixture') - code.writeln('@slash.requires({}, {})'.format(_UNMET_REQ_DECORATOR, '"msg1"')) - code.writeln('def fixture():') + code.writeln("import slash") + code.writeln("@slash.fixture") + code.writeln( + "@slash.requires({}, {})".format(_UNMET_REQ_DECORATOR, '"msg1"') + ) + code.writeln("def fixture():") with code.indented(): - code.writeln('return 1') + code.writeln("return 1") - code.writeln('@slash.fixture(autouse=True)') - code.writeln('@slash.requires({}, {})'.format(_MET_REQ_DECORATOR, '"msg2"')) - code.writeln('def fixture1():') + code.writeln("@slash.fixture(autouse=True)") + code.writeln("@slash.requires({}, {})".format(_MET_REQ_DECORATOR, '"msg2"')) + code.writeln("def fixture1():") with code.indented(): - code.writeln('return 1') + code.writeln("return 1") - code.writeln('class Test(slash.Test):') + code.writeln("class Test(slash.Test):") stack.enter_context(code.indented()) - code.write('def test_1(') - code.write('self, ') - code.writeln('fixture):') + code.write("def test_1(") + code.write("self, ") + code.writeln("fixture):") with code.indented(): - code.writeln('pass') + code.writeln("pass") return returned def test_requirements_on_class_with_fixture_and_autouse_fixture(filename_test_fixture): with slash.Session(): - [test] = make_runnable_tests(filename_test_fixture) # pylint: disable=unbalanced-tuple-unpacking - assert sorted([str(r) for r in test.get_requirements()]) == ['msg1', 'msg2'] + [test] = make_runnable_tests( # pylint: disable=unbalanced-tuple-unpacking + filename_test_fixture + ) + assert sorted([str(r) for r in test.get_requirements()]) == ["msg1", "msg2"] def test_unmet_requirements_trigger_avoided_test_hook(suite, suite_test): @@ -156,62 +178,67 @@ def test_unmet_requirements_trigger_avoided_test_hook(suite, suite_test): suite_test.add_decorator(_UNMET_REQ_DECORATOR) suite_test.expect_skip() - - @gossip.register('slash.test_avoided') + @gossip.register("slash.test_avoided") def test_avoided(reason): # pylint: disable=unused-variable - slash.context.result.data['avoided'] = {'reason': reason, - 'test_name': slash.context.test.__slash__.address} + slash.context.result.data["avoided"] = { + "reason": reason, + "test_name": slash.context.test.__slash__.address, + } summary = suite.run() avoided_result = summary[suite_test] - for r in summary.session.results.iter_all_results(): if r is avoided_result: - assert 'avoided' in r.data - assert 'lambda' in r.data['avoided']['reason'] - assert 'unmet requirement' in r.data['avoided']['reason'].lower() - assert r.data['avoided']['test_name'].split('_')[-1] == suite_test.id + assert "avoided" in r.data + assert "lambda" in r.data["avoided"]["reason"] + assert "unmet requirement" in r.data["avoided"]["reason"].lower() + assert r.data["avoided"]["test_name"].split("_")[-1] == suite_test.id else: - assert 'avoided' not in r.data + assert "avoided" not in r.data def test_adding_requirement_objects(): - class MyRequirement(slash.core.requirements.Requirement): pass - req = MyRequirement('bla') + req = MyRequirement("bla") @slash.requires(req) def test_something(): pass with slash.Session(): - [test] = make_runnable_tests(test_something) # pylint: disable=unbalanced-tuple-unpacking + [test] = make_runnable_tests( # pylint: disable=unbalanced-tuple-unpacking + test_something + ) # pylint: disable=unbalanced-tuple-unpacking reqs = test.get_requirements() assert len(reqs) == 1 and reqs[0] is req def test_cannot_specify_message_with_requirement_object(): - class MyRequirement(slash.core.requirements.Requirement): pass with pytest.raises(AssertionError) as caught: - slash.requires(MyRequirement(''), 'message') + slash.requires(MyRequirement(""), "message") + + assert "specify message" in str(caught.value) - assert 'specify message' in str(caught.value) -@pytest.mark.parametrize('is_fixture_requirement_unmet', [True, False]) +@pytest.mark.parametrize("is_fixture_requirement_unmet", [True, False]) def test_fixture_and_test_requirements(suite, suite_test, is_fixture_requirement_unmet): suite_test.depend_on_fixture(suite.slashconf.add_fixture()) if is_fixture_requirement_unmet: - suite_test._fixtures[0][1].add_decorator(_UNMET_REQ_DECORATOR) # pylint: disable=protected-access + suite_test._fixtures[0][1].add_decorator( # pylint: disable=protected-access + _UNMET_REQ_DECORATOR + ) suite_test.add_decorator(_MET_REQ_DECORATOR) else: - suite_test._fixtures[0][1].add_decorator(_MET_REQ_DECORATOR) # pylint: disable=protected-access + suite_test._fixtures[0][1].add_decorator( # pylint: disable=protected-access + _MET_REQ_DECORATOR + ) suite_test.add_decorator(_UNMET_REQ_DECORATOR) suite_test.expect_skip() @@ -219,26 +246,96 @@ def test_fixture_and_test_requirements(suite, suite_test, is_fixture_requirement assert results[suite_test].is_skip() result = results[suite_test] [skip] = result.get_skips() - assert 'lambda' in skip + assert "lambda" in skip + def test_fixture_of_fixture_requirement(suite, suite_test): suite_test.add_decorator(_UNMET_REQ_DECORATOR) suite_test.depend_on_fixture(suite.slashconf.add_fixture()) - suite_test._fixtures[0][1].add_decorator(_MET_REQ_DECORATOR) # pylint: disable=protected-access + suite_test._fixtures[0][1].add_decorator( # pylint: disable=protected-access + _MET_REQ_DECORATOR + ) suite_test.expect_skip() results = suite.run() assert results[suite_test].is_skip() result = results[suite_test] [skip] = result.get_skips() - assert 'lambda' in skip + assert "lambda" in skip def test_autouse_fixture_requirement(): suite = Suite() for _ in range(5): - test = suite.add_test(type='function') + test = suite.add_test(type="function") test.expect_skip() fixture = suite.get_last_file().add_fixture(autouse=True) fixture.add_decorator(_UNMET_REQ_DECORATOR) suite.run() + + +def test_class_requirements_siblings(suite_builder): + @suite_builder.first_file.add_code + def __code__(): # pylint: disable=unused-variable + import slash # pylint: disable=redefined-outer-name, reimported + + @slash.requires(lambda: True) + class BaseTest(slash.Test): + pass + + @slash.requires(lambda: False) # pylint: disable=unused-variable + class FirstTest(BaseTest): + def test(self): + pass + + class SecondTest(BaseTest): # pylint: disable=unused-variable + def test(self): + pass + + suite_builder.build().run().assert_results_breakdown(skipped=1, success=1) + + +@pytest.mark.parametrize("class_can_run", [True, False]) +@pytest.mark.parametrize("method_can_run", [True, False]) +def test_class_requirements_class_and_method(class_can_run, method_can_run): + @slash.requires(lambda: class_can_run) + class Test(slash.Test): + @slash.requires(lambda: method_can_run) + def test(self): + pass + + with slash.Session() as session: + with session.get_started_context(): + slash.runner.run_tests(make_runnable_tests(Test)) + results = [res for res in session.results.iter_test_results()] + assert len(results) == 1 + if class_can_run and method_can_run: + assert results[0].is_success() + else: + assert results[0].is_skip() + + +def test_attach_requirements_through_wraps_on_function(): + + def decorator(func): + @wraps(func) + def new_func(): + pass + return new_func + + req1, req2, req3, req4 = requirements = [object() for _ in range(4)] + + @slash.requires(req4) + @slash.parametrize('x', [1, 2, 3]) + @slash.requires(req3) + @slash.parametrize('y', [2, 3, 4]) + @slash.requires(req2) + @decorator + @slash.requires(req1) + def test_something(x, y): # pylint: disable=unused-argument + pass + + found_reqs = slash.core.requirements.get_requirements(test_something) + assert len(found_reqs) == len(requirements) + for expected, found in zip(requirements, found_reqs): + assert expected is found._req # pylint: disable=trailing-whitespace, protected-access diff --git a/tests/utils/suite_builder.py b/tests/utils/suite_builder.py index 19adea71..e7915595 100644 --- a/tests/utils/suite_builder.py +++ b/tests/utils/suite_builder.py @@ -87,6 +87,14 @@ def with_data(self, data_sets): assert not results return self + def assert_results_breakdown(self, skipped=0, success=0, error=0, failure=0): + results = self.slash_app.session.results + assert skipped == results.get_num_skipped() + assert success == results.get_num_successful() + assert error == results.get_num_errors() + assert failure == results.get_num_failures() + + class AssertAllHelper(object): def __init__(self, suite_builder_result):