Feature/skip remaining tests being executed

testplan/common/config/base.py

@@ -167,9 +167,7 @@ def __getattr__(self, name):
             )
 
     def __repr__(self):
-        return "{}{}".format(
-            self.__class__.__name__, self._cfg_input or self._options
-        )
+        return "{}{}".format(self.__class__.__name__, self._options)
 
     def get_local(self, name, default=None):
         """Returns a local config setting (not from container)"""
@@ -184,6 +182,11 @@ def get_local(self, name, default=None):
         else:
             return default
 
+    def set_local(self, name, value):
+        """set without any check"""
+        options = self.__getattribute__("_options")
+        options[name] = value
+
     @property
     def parent(self):
         """Returns the parent configuration."""
@@ -206,20 +209,19 @@ def denormalize(self):
         new_options = {}
         for key in self._options:
             value = getattr(self, key)
-            if inspect.isclass(value) or inspect.isroutine(value):
-                # Skipping non-serializable classes and routines.
-                logger.TESTPLAN_LOGGER.debug(
-                    "Skip denormalizing option: %s", key
-                )
-                continue
             try:
                 new_options[copy.deepcopy(key)] = copy.deepcopy(value)
             except Exception as exc:
                 logger.TESTPLAN_LOGGER.warning(
                     "Failed to denormalize option: {} - {}".format(key, exc)
                 )
 
-        new = self.__class__(**new_options)
+        # XXX: we have transformed options, which should not be validated
+        # XXX: against schema again
+        new = object.__new__(self.__class__)
+        setattr(new, "_parent", None)
+        setattr(new, "_cfg_input", new_options)
+        setattr(new, "_options", new_options)
         return new
 
     @classmethod

testplan/common/entity/base.py

@@ -12,6 +12,7 @@
 from collections import OrderedDict, deque
 from contextlib import suppress
 from typing import (
+    Any,
     Callable,
     Deque,
     Dict,
@@ -20,20 +21,19 @@
     Optional,
     Tuple,
     Union,
-    Any,
 )
 
 import psutil
 from schema import Or
 
 from testplan.common.config import Config, ConfigOption
+from testplan.common.report.base import EventRecorder
 from testplan.common.utils import logger
 from testplan.common.utils.path import default_runpath, makedirs, makeemptydirs
 from testplan.common.utils.strings import slugify, uuid4
 from testplan.common.utils.thread import execute_as_thread, interruptible_join
 from testplan.common.utils.timing import wait
 from testplan.common.utils.validation import is_subclass
-from testplan.common.report.base import EventRecorder
 
 
 class Environment:
@@ -149,6 +149,9 @@ def __repr__(self):
     def __len__(self):
         return len(self._resources)
 
+    def items(self):
+        return self._resources.items()
+
     def all_status(self, target) -> bool:
         """
         Checks whether all resources have target status.
@@ -165,7 +168,7 @@ def _record_resource_exception(self, message, resource, msg_store):
         fetch_msg = "\n".join(resource.fetch_error_log())
 
         msg = message.format(
-            resource_name=resource.cfg.name,
+            resource=str(resource),
             traceback_exc=traceback.format_exc(),
             fetch_msg=fetch_msg,
         )
@@ -186,7 +189,8 @@ def start(self):
                 resource.start()
             except Exception:
                 self._record_resource_exception(
-                    message="While starting resource [{resource_name}]\n{traceback_exc}\n{fetch_msg}",
+                    message="While starting resource {resource}:\n"
+                    "{traceback_exc}\n{fetch_msg}",
                     resource=resource,
                     msg_store=self.start_exceptions,
                 )
@@ -207,7 +211,8 @@ def start(self):
                 resource.wait(resource.STATUS.STARTED)
             except Exception:
                 self._record_resource_exception(
-                    message="While waiting for resource [{resource_name}] to start\n{traceback_exc}\n{fetch_msg}",
+                    message="While waiting for resource {resource} to start:\n"
+                    "{traceback_exc}\n{fetch_msg}",
                     resource=resource,
                     msg_store=self.start_exceptions,
                 )
@@ -273,7 +278,8 @@ def stop(self, is_reversed=False):
                 resource.stop()
             except Exception:
                 self._record_resource_exception(
-                    message="While stopping resource [{resource_name}]\n{traceback_exc}\n{fetch_msg}",
+                    message="While stopping resource {resource}:"
+                    "\n{traceback_exc}\n{fetch_msg}",
                     resource=resource,
                     msg_store=self.stop_exceptions,
                 )
@@ -596,8 +602,7 @@ def abort(self):
 
         self._should_abort = True
         for dep in self.abort_dependencies():
-            if dep is not None:
-                self._abort_entity(dep)
+            self._abort_entity(dep)
 
         self.logger.info("Aborting %s", self)
         self.aborting()
@@ -631,7 +636,6 @@ def _abort_entity(self, entity, wait_timeout=None):
             self.logger.error(traceback.format_exc())
             self.logger.error("Exception on aborting %s - %s", entity, exc)
         else:
-
             if (
                 wait(
                     predicate=lambda: entity.aborted is True,
@@ -640,7 +644,11 @@ def _abort_entity(self, entity, wait_timeout=None):
                 )
                 is False
             ):
-                self.logger.error("Timeout on waiting to abort %s.", entity)
+                self.logger.error(
+                    "Timeout on waiting to abort %s after %d seconds.",
+                    entity,
+                    timeout,
+                )
 
     def aborting(self):
         """
@@ -1000,6 +1008,7 @@ def _post_run_checks(self, start_threads, start_procs):
         :param start_procs: processes before run
         :type start_procs: ``list`` of ``Process``
         """
+        # XXX: do we want to suppress process/thread check for tests?
         end_threads = threading.enumerate()
         if start_threads != end_threads:
             new_threads = [

testplan/common/utils/logger.py

@@ -10,9 +10,9 @@
     - Test progress information (e.g. Pass / Fail status)
     - Exporter statuses
 """
+import logging
 import os
 import sys
-import logging
 
 from testplan.common.utils.strings import Color, uuid4
 from testplan.report import Status
@@ -74,16 +74,22 @@ def user_info(self, msg, *args, **kwargs):
         """Log 'msg % args' with severity 'USER_INFO'"""
         self._custom_log(USER_INFO, msg, *args, **kwargs)
 
-    def log_test_status(self, name, status, indent=0, level=USER_INFO):
+    def log_test_status(
+        self,
+        name: str,
+        status: Status,
+        indent: int = 0,
+        level: int = USER_INFO,
+    ):
         """Shortcut to log a pass/fail status for a test."""
-        if Status.STATUS_CATEGORY[status] == Status.PASSED:
-            pass_label = Color.green(status.title())
-        elif Status.STATUS_CATEGORY[status] in [Status.FAILED, Status.ERROR]:
-            pass_label = Color.red(status.title())
-        elif Status.STATUS_CATEGORY[status] == Status.UNSTABLE:
-            pass_label = Color.yellow(status.title())
+        if status.normalised() == Status.PASSED:
+            pass_label = Color.green(status.value.title())
+        elif status <= Status.FAILED:
+            pass_label = Color.red(status.value.title())
+        elif status.normalised() == Status.UNSTABLE:
+            pass_label = Color.yellow(status.value.title())
         else:  # unknown
-            pass_label = status
+            pass_label = status.value.title()
 
         indent_str = indent * " "
         msg = self._TEST_STATUS_FORMAT

testplan/common/utils/process.py

@@ -2,15 +2,16 @@
 
 import functools
 import platform
-from signal import Signals
 import subprocess
 import threading
-import psutil
 import time
 import warnings
 from enum import Enum, auto
+from signal import Signals
 from typing import IO, Any, Callable, List, Union
 
+import psutil
+
 from testplan.common.utils.logger import TESTPLAN_LOGGER
 from testplan.common.utils.timing import exponential_interval, get_sleeper
 
@@ -253,6 +254,7 @@ def execute_cmd(
 
     if isinstance(cmd, list):
         cmd = [str(a) for a in cmd]
+        # FIXME: not good enough, need shell escaping
         cmd_string = " ".join(cmd)  # for logging, easy to copy and execute
     else:
         cmd_string = cmd

testplan/common/utils/selector.py

@@ -0,0 +1,104 @@
+# decouple represetation from evaluation
+# typevar always of kind *, type hint won't work much here
+
+from dataclasses import dataclass
+from typing import Any, Callable, Generic, Set, TypeVar
+
+from typing_extensions import Protocol, Self
+
+T = TypeVar("T")
+U = TypeVar("U")
+
+
+class Functor(Protocol, Generic[T]):
+    def map(self, f: Callable[[T], U]) -> Self:
+        # map :: f t -> (t -> u) -> f u
+        ...
+
+
+@dataclass
+class And2(Generic[T]):
+    lterm: T
+    rterm: T
+
+    def map(self, f):
+        return And2(f(self.lterm), f(self.rterm))
+
+
+@dataclass
+class Or2(Generic[T]):
+    lterm: T
+    rterm: T
+
+    def map(self, f):
+        return Or2(f(self.lterm), f(self.rterm))
+
+
+@dataclass
+class Not(Generic[T]):
+    term: T
+
+    def map(self, f):
+        return Not(f(self.term))
+
+
+@dataclass
+class Eq(Generic[U]):
+    val: U
+
+    def map(self, _):
+        return self
+
+
+@dataclass
+class Const(Generic[T]):
+    val: bool
+
+    def map(self, _):
+        return self
+
+
+Expr = TypeVar("Expr", bound=Functor)
+
+
+def cata(f: Callable, rep: Expr):
+    # cata :: (f t -> t) -> f (f (f ...)) -> t
+    return f(rep.map(lambda x: cata(f, x)))
+
+
+def eval_on_set(s: Set) -> Callable:
+    def _(x):
+        if isinstance(x, Const):
+            return {i for i in s if x.val}
+        if isinstance(x, Eq):
+            return {i for i in s if i == x.val}
+        if isinstance(x, And2):
+            return x.lterm & x.rterm
+        if isinstance(x, Or2):
+            return x.lterm | x.rterm
+        if isinstance(x, Not):
+            return s - x.term
+        raise TypeError(f"unexpected {x}")
+
+    return _
+
+
+def apply_on_set(rep: Expr, s: Set) -> Set:
+    return cata(eval_on_set(s), rep)
+
+
+def apply_single(rep: Expr, i: Any) -> bool:
+    def _(x):
+        if isinstance(x, Const):
+            return x.val
+        if isinstance(x, Eq):
+            return x.val == i
+        if isinstance(x, And2):
+            return x.lterm and x.rterm
+        if isinstance(x, Or2):
+            return x.lterm or x.rterm
+        if isinstance(x, Not):
+            return not x.term
+        raise TypeError(f"unexpected {x}")
+
+    return cata(_, rep)

testplan/exporters/testing/pdf/renderers/entries/assertions.py

@@ -81,7 +81,7 @@ def get_header(self, source, depth, row_idx):
                 header,
                 "",
                 "",
-                (Status.PASSED if passed else Status.FAILED).title(),
+                (Status.PASSED if passed else Status.FAILED).value.title(),
             ],
             style=default_assertion_style(passed=passed, depth=depth),
             start=row_idx,
@@ -520,7 +520,9 @@ def get_detail(self, source, depth, row_idx):
                         description,
                         "",
                         "",
-                        (Status.PASSED if passed else Status.FAILED).title(),
+                        (
+                            Status.PASSED if passed else Status.FAILED
+                        ).value.title(),
                     ],
                     style=styles,
                     start=row_idx + counter,

testplan/exporters/testing/pdf/renderers/reports.py

@@ -45,8 +45,8 @@ def format_status(report_status: Status) -> str:
     erroneous tests will be displayed as failed.
     """
     if report_status in (Status.FAILED, Status.ERROR):
-        return Status.FAILED.title()
-    return report_status.title()
+        return Status.FAILED.value.title()
+    return report_status.value.title()
 
 
 @registry.bind(TestReport)

testplan/parser.py

@@ -18,7 +18,7 @@
     ReportTagsAction,
     styles,
 )
-from testplan.testing import filtering, listing, ordering
+from testplan.testing import common, filtering, listing, ordering
 
 
 class HelpParser(argparse.ArgumentParser):
@@ -205,6 +205,17 @@ def generate_parser(self) -> HelpParser:
 }""",
         )
 
+        general_group.add_argument(
+            "--skip-remaining",
+            metavar="OPTION",
+            choices=common.SkipStrategy.all_options(),
+            dest="skip_strategy",
+            help="When a Testcase has failed or error has occurred, skip the "
+            "remaining of Testcases in the same Testsuite/Multitest/Testplan (or "
+            "anything with equal level) being executed. Valid values: "
+            + str(common.SkipStrategy.all_options()),
+        )
+
         filter_group = parser.add_argument_group("Filtering")
 
         filter_pattern_group = filter_group.add_mutually_exclusive_group()