Feature/skip remaining tests being executed (#1029)

* some primitive ideas

- convert certain statuses into enum
- introduce cross-executor messaging
- fix config clone

* test and tweak for config & status enums

* refine selector & local runner

* address comments; upgrade tests

* some adjustments

* some fixes

* change on locks and flags

* address more review comments

* newsfrag, doc & wording adjustment

* minor fixes

* update newsfrag & log message

doc/en/introduction.rst

@@ -340,7 +340,17 @@ Command line
       --trace-tests-output
                             Specify output file for tests impacted by change in Testplan pattern format (see --trace-tests). Will be ignored if --trace-tests is not specified. Default to standard output.
       --xfail-tests         Read a list of known to fail testcases from a JSON file with each entry looks like: {"<Multitest>:<TestSuite>:<testcase>": {"reason": <value>, "strict": <value>} }
-
+      --runtime-data PATH   Historical runtime data which will be used for Multitest auto-part and weight-based Task smart-scheduling with entries looks like:
+
+                            {
+                                "<Multitest>": {
+                                    "execution_time": 199.99,
+                                    "setup_time": 39.99,
+                                },
+                                ......
+                            }
+      --skip-remaining {cases-on-failed,cases-on-error,suites-on-failed,suites-on-error,tests-on-failed,tests-on-error}
+                            Skip the remaining Testcases/Testsuites/Multitests being executed when a Testcase has failed or raised exception.
     Filtering:
       --patterns            Test filter, supports glob notation & multiple arguments.
 
@@ -427,7 +437,7 @@ Command line
                             --report-tags-all <tag_name_1> --report-tags-all <tag_name 2>
 
                             --report-tags-all <tag_name_1> <tag_category_1>=<tag_name_2>
-      --file-log-level      {exporter_info,test_info,driver_info,critical,error,warning,info,debug,none}
+      --file-log-level {USER_INFO,CRITICAL,ERROR,WARNING,INFO,DEBUG,NONE}
 
                             Specify log level for file logs. Set to None to disable file logging.
 

doc/newsfragments/2540_new.skip_remaining.rst

@@ -0,0 +1,3 @@
+Added a new command line argument ``--skip-remaining``, a new argument ``skip_strategy`` to MultiTest, allowing remaining Testcases/Testsuites/MultiTests being skipped from execution when a Testcase has failed or raised exeception.
+
+Argument ``uid`` of :py:meth:`Testplan.add_resource <testplan.base.Testplan.add_resource>` should now match the uid of the ``resource`` argument.

testplan/base.py

@@ -258,10 +258,10 @@ def __init__(
         )
 
         # By default, a LocalRunner is added to store and execute the tests.
-        self._runnable.add_resource(LocalRunner(), uid="local_runner")
+        self._runnable.add_resource(LocalRunner())
 
         # Stores independent environments.
-        self._runnable.add_resource(Environments(), uid="environments")
+        self._runnable.add_resource(Environments())
 
     @property
     def parser(self):

testplan/common/config/base.py

@@ -167,9 +167,7 @@ def __getattr__(self, name):
             )
 
     def __repr__(self):
-        return "{}{}".format(
-            self.__class__.__name__, self._cfg_input or self._options
-        )
+        return "{}{}".format(self.__class__.__name__, self._options)
 
     def get_local(self, name, default=None):
         """Returns a local config setting (not from container)"""
@@ -184,6 +182,11 @@ def get_local(self, name, default=None):
         else:
             return default
 
+    def set_local(self, name, value):
+        """set without any check"""
+        options = self.__getattribute__("_options")
+        options[name] = value
+
     @property
     def parent(self):
         """Returns the parent configuration."""
@@ -206,20 +209,19 @@ def denormalize(self):
         new_options = {}
         for key in self._options:
             value = getattr(self, key)
-            if inspect.isclass(value) or inspect.isroutine(value):
-                # Skipping non-serializable classes and routines.
-                logger.TESTPLAN_LOGGER.debug(
-                    "Skip denormalizing option: %s", key
-                )
-                continue
             try:
                 new_options[copy.deepcopy(key)] = copy.deepcopy(value)
             except Exception as exc:
                 logger.TESTPLAN_LOGGER.warning(
                     "Failed to denormalize option: {} - {}".format(key, exc)
                 )
 
-        new = self.__class__(**new_options)
+        # XXX: we have transformed options, which should not be validated
+        # XXX: against schema again
+        new = object.__new__(self.__class__)
+        setattr(new, "_parent", None)
+        setattr(new, "_cfg_input", new_options)
+        setattr(new, "_options", new_options)
         return new
 
     @classmethod

testplan/common/entity/base.py

@@ -12,6 +12,7 @@
 from collections import OrderedDict, deque
 from contextlib import suppress
 from typing import (
+    Any,
     Callable,
     Deque,
     Dict,
@@ -20,20 +21,19 @@
     Optional,
     Tuple,
     Union,
-    Any,
 )
 
 import psutil
 from schema import Or
 
 from testplan.common.config import Config, ConfigOption
+from testplan.common.report.base import EventRecorder
 from testplan.common.utils import logger
 from testplan.common.utils.path import default_runpath, makedirs, makeemptydirs
 from testplan.common.utils.strings import slugify, uuid4
 from testplan.common.utils.thread import execute_as_thread, interruptible_join
 from testplan.common.utils.timing import wait
 from testplan.common.utils.validation import is_subclass
-from testplan.common.report.base import EventRecorder
 
 
 class Environment:
@@ -149,6 +149,9 @@ def __repr__(self):
     def __len__(self):
         return len(self._resources)
 
+    def items(self):
+        return self._resources.items()
+
     def all_status(self, target) -> bool:
         """
         Checks whether all resources have target status.
@@ -165,7 +168,7 @@ def _record_resource_exception(self, message, resource, msg_store):
         fetch_msg = "\n".join(resource.fetch_error_log())
 
         msg = message.format(
-            resource_name=resource.cfg.name,
+            resource=resource,
             traceback_exc=traceback.format_exc(),
             fetch_msg=fetch_msg,
         )
@@ -186,7 +189,8 @@ def start(self):
                 resource.start()
             except Exception:
                 self._record_resource_exception(
-                    message="While starting resource [{resource_name}]\n{traceback_exc}\n{fetch_msg}",
+                    message="While starting resource {resource}:\n"
+                    "{traceback_exc}\n{fetch_msg}",
                     resource=resource,
                     msg_store=self.start_exceptions,
                 )
@@ -207,7 +211,8 @@ def start(self):
                 resource.wait(resource.STATUS.STARTED)
             except Exception:
                 self._record_resource_exception(
-                    message="While waiting for resource [{resource_name}] to start\n{traceback_exc}\n{fetch_msg}",
+                    message="While waiting for resource {resource} to start:\n"
+                    "{traceback_exc}\n{fetch_msg}",
                     resource=resource,
                     msg_store=self.start_exceptions,
                 )
@@ -273,7 +278,8 @@ def stop(self, is_reversed=False):
                 resource.stop()
             except Exception:
                 self._record_resource_exception(
-                    message="While stopping resource [{resource_name}]\n{traceback_exc}\n{fetch_msg}",
+                    message="While stopping resource {resource}:"
+                    "\n{traceback_exc}\n{fetch_msg}",
                     resource=resource,
                     msg_store=self.stop_exceptions,
                 )
@@ -596,8 +602,7 @@ def abort(self):
 
         self._should_abort = True
         for dep in self.abort_dependencies():
-            if dep is not None:
-                self._abort_entity(dep)
+            self._abort_entity(dep)
 
         self.logger.info("Aborting %s", self)
         self.aborting()
@@ -631,7 +636,6 @@ def _abort_entity(self, entity, wait_timeout=None):
             self.logger.error(traceback.format_exc())
             self.logger.error("Exception on aborting %s - %s", entity, exc)
         else:
-
             if (
                 wait(
                     predicate=lambda: entity.aborted is True,
@@ -640,7 +644,11 @@ def _abort_entity(self, entity, wait_timeout=None):
                 )
                 is False
             ):
-                self.logger.error("Timeout on waiting to abort %s.", entity)
+                self.logger.error(
+                    "Timeout on waiting to abort %s after %d seconds.",
+                    entity,
+                    timeout,
+                )
 
     def aborting(self):
         """
@@ -1000,6 +1008,7 @@ def _post_run_checks(self, start_threads, start_procs):
         :param start_procs: processes before run
         :type start_procs: ``list`` of ``Process``
         """
+        # XXX: do we want to suppress process/thread check for tests?
         end_threads = threading.enumerate()
         if start_threads != end_threads:
             new_threads = [

testplan/common/utils/logger.py

@@ -10,9 +10,9 @@
     - Test progress information (e.g. Pass / Fail status)
     - Exporter statuses
 """
+import logging
 import os
 import sys
-import logging
 
 from testplan.common.utils.strings import Color, uuid4
 from testplan.report import Status
@@ -74,16 +74,22 @@ def user_info(self, msg, *args, **kwargs):
         """Log 'msg % args' with severity 'USER_INFO'"""
         self._custom_log(USER_INFO, msg, *args, **kwargs)
 
-    def log_test_status(self, name, status, indent=0, level=USER_INFO):
+    def log_test_status(
+        self,
+        name: str,
+        status: Status,
+        indent: int = 0,
+        level: int = USER_INFO,
+    ):
         """Shortcut to log a pass/fail status for a test."""
-        if Status.STATUS_CATEGORY[status] == Status.PASSED:
-            pass_label = Color.green(status.title())
-        elif Status.STATUS_CATEGORY[status] in [Status.FAILED, Status.ERROR]:
-            pass_label = Color.red(status.title())
-        elif Status.STATUS_CATEGORY[status] == Status.UNSTABLE:
-            pass_label = Color.yellow(status.title())
+        if status.normalised() == Status.PASSED:
+            pass_label = Color.green(status.name.title())
+        elif status <= Status.FAILED:
+            pass_label = Color.red(status.name.title())
+        elif status.normalised() == Status.UNSTABLE:
+            pass_label = Color.yellow(status.name.title())
         else:  # unknown
-            pass_label = status
+            pass_label = status.name.title()
 
         indent_str = indent * " "
         msg = self._TEST_STATUS_FORMAT

testplan/common/utils/process.py

@@ -2,15 +2,16 @@
 
 import functools
 import platform
-from signal import Signals
 import subprocess
 import threading
-import psutil
 import time
 import warnings
 from enum import Enum, auto
+from signal import Signals
 from typing import IO, Any, Callable, List, Union
 
+import psutil
+
 from testplan.common.utils.logger import TESTPLAN_LOGGER
 from testplan.common.utils.timing import exponential_interval, get_sleeper
 
@@ -253,6 +254,7 @@ def execute_cmd(
 
     if isinstance(cmd, list):
         cmd = [str(a) for a in cmd]
+        # FIXME: not good enough, need shell escaping
         cmd_string = " ".join(cmd)  # for logging, easy to copy and execute
     else:
         cmd_string = cmd