Report steps that failed

metabox/metabox/core/runner.py

@@ -284,15 +284,20 @@ def run(self):
                 )
             )
             scn.run()
-            if not scn.has_passed():
+            if scn.failures:
                 self.failed = True
                 logger.error(scenario_description + " scenario has failed.")
 
-                # let's escape < from the output to avoid confusing loguru
+                # let's escape < from the all outputs to avoid confusing loguru
                 # loguru assumes that <> is used for colorizing
-                output = scn.get_output_streams().strip().replace("<", r"\<")
+                logger.error("The following steps failed")
+                failed_steps = "\n".join(str(x) for x in scn.failures)
+                failed_steps = failed_steps.replace("<", r"\<")
+                logger.error(failed_steps)
 
+                output = scn.get_output_streams().strip().replace("<", r"\<")
                 logger.error("Scenario output:\n" + output)
+
                 if self.hold_on_fail:
                     if scn.mode == "remote":
                         msg = (

metabox/metabox/core/scenario.py

@@ -70,7 +70,7 @@ def __init__(
         self.agent_machine = None
         self.agent_revision = agent_revision
         self.start_session = True
-        self._failures = []
+        self.failures = []
         self._ret_code = None
         self._stdout = ""
         self._stderr = ""
@@ -82,10 +82,6 @@ def get_output_streams(self):
             return self._pts.stdout_data_full
         return self._outstr_full
 
-    def has_passed(self):
-        """Check whether all the assertions passed."""
-        return all(self._checks)
-
     def run(self):
         # Simple scenarios don't need to specify a START step
         # If there's no START step, add one unless the scenario
@@ -109,11 +105,11 @@ def run(self):
                         break
                 step.kwargs["interactive"] = interactive
             try:
-                res = step(self)
-                if res is None:
-                    res = True
+                # step that fail explicitly return false or raise an exception
+                if step(self) not in [True, None]:
+                    self.failures.append(step)
             except (TimeoutError, ConnectionError):
-                self._checks.append(False)
+                self.failures.append(step)
                 break
         if self._pts:
             self._stdout = self._pts.stdout_data_full
@@ -128,7 +124,6 @@ def _assign_outcome(self, ret_code, stdout, stderr, outstr_full):
         self._stderr = stderr
         self._outstr_full = outstr_full
 
-    # TODO: add storing of what actually failed in the assert methods
     def assert_printed(self, pattern):
         """
         Check if during Checkbox execution a line produced that matches the