Add grammar-violating mutations to the Generator (#262)

With a runtime flag, we can enable from now to apply grammar violating
mutations on the trees. In this first commit, unrestricted node deletion
and node hoisting are implemented.

grammarinator/generate.py

@@ -62,7 +62,7 @@ def generator_tool_helper(args, weights, lock):
                          rule=args.rule, out_format=args.out, lock=lock,
                          limit=RuleSize(depth=args.max_depth, tokens=args.max_tokens),
                          population=DefaultPopulation(args.population, args.tree_extension, args.tree_codec) if args.population else None,
-                         generate=args.generate, mutate=args.mutate, recombine=args.recombine, keep_trees=args.keep_trees,
+                         generate=args.generate, mutate=args.mutate, recombine=args.recombine, unrestricted=args.unrestricted, keep_trees=args.keep_trees,
                          transformers=args.transformer, serializer=args.serializer,
                          cleanup=False, encoding=args.encoding, errors=args.encoding_errors, dry_run=args.dry_run)
 
@@ -108,6 +108,8 @@ def execute():
                         help='disable test generation by mutation (disabled by default if no population is given).')
     parser.add_argument('--no-recombine', dest='recombine', default=True, action='store_false',
                         help='disable test generation by recombination (disabled by default if no population is given).')
+    parser.add_argument('--no-grammar-violations', dest='unrestricted', default=True, action='store_false',
+                        help='disable applying grammar-violating mutators (enabled by default)')
     parser.add_argument('--keep-trees', default=False, action='store_true',
                         help='keep generated tests to participate in further mutations or recombinations (only if population is given).')
     add_tree_format_argument(parser)

grammarinator/runtime/rule.py

@@ -9,6 +9,8 @@
 from math import inf
 from textwrap import indent
 
+from itertools import zip_longest
+
 
 class RuleSize:
     """
@@ -218,6 +220,25 @@ def equals(self, other):
         """
         return self.__class__ == other.__class__ and self.name == other.name
 
+    def equalTokens(self, other):
+        """
+        Compare the tokens in the sub-trees of two nodes.
+
+        :param Rule other: The node to compare the current node to.
+        :return: Whether the two nodes are equal.
+        :rtype: bool
+        """
+        return all(self_token == other_token for self_token, other_token in zip_longest(self.tokens(), other.tokens()))
+
+    def tokens(self):
+        """
+        Generator method to iterate over the (non-empty) tokens (i.e., strings) of the sub-tree of the node.
+
+        :return: Iterator over token string contents.
+        :rtype: iterator[str]
+        """
+        raise NotImplementedError()
+
     def _dbg_(self):
         """
         Called by :meth:`__format__` to compute the "debug" string
@@ -318,6 +339,10 @@ def __iadd__(self, item):
     def equals(self, other):
         return super().equals(other) and len(self.children) == len(other.children) and all(child.equals(other.children[i]) for i, child in enumerate(self.children))
 
+    def tokens(self):
+        for child in self.children:
+            yield from child.tokens()
+
     def __str__(self):
         return ''.join(str(child) for child in self.children)
 
@@ -391,6 +416,10 @@ def __init__(self, *, name=None, src=None, size=None, immutable=False):
     def equals(self, other):
         return super().equals(other) and self.src == other.src
 
+    def tokens(self):
+        if self.src:
+            yield self.src
+
     def __str__(self):
         return self.src
 

grammarinator/tool/generator.py

@@ -15,7 +15,7 @@
 from os.path import abspath, dirname
 from shutil import rmtree
 
-from ..runtime import DefaultModel, RuleSize, WeightedModel
+from ..runtime import DefaultModel, RuleSize, UnparserRule, WeightedModel
 
 logger = logging.getLogger(__name__)
 
@@ -126,7 +126,7 @@ class GeneratorTool:
     """
 
     def __init__(self, generator_factory, out_format, lock=None, rule=None, limit=None,
-                 population=None, generate=True, mutate=True, recombine=True, keep_trees=False,
+                 population=None, generate=True, mutate=True, recombine=True, unrestricted=True, keep_trees=False,
                  transformers=None, serializer=None,
                  cleanup=True, encoding='utf-8', errors='strict', dry_run=False):
         """
@@ -158,6 +158,7 @@ def __init__(self, generator_factory, out_format, lock=None, rule=None, limit=No
         :param bool generate: Enable generating new test cases from scratch, i.e., purely based on grammar.
         :param bool mutate: Enable mutating existing test cases, i.e., re-generate part of an existing test case based on grammar.
         :param bool recombine: Enable recombining existing test cases, i.e., replace part of a test case with a compatible part from another test case.
+        :param bool unrestricted: Enable applying possibly grammar-violating creators.
         :param bool keep_trees: Keep generated trees to participate in further mutations or recombinations
                (otherwise, only the initial population will be mutated or recombined). It has effect only if
                population is defined.
@@ -186,6 +187,7 @@ def __init__(self, generator_factory, out_format, lock=None, rule=None, limit=No
         self._enable_generation = generate
         self._enable_mutation = mutate
         self._enable_recombination = recombine
+        self._enable_unrestricted_creators = unrestricted
         self._keep_trees = keep_trees
         self._cleanup = cleanup
         self._encoding = encoding
@@ -200,6 +202,10 @@ def __init__(self, generator_factory, out_format, lock=None, rule=None, limit=No
             self.shuffle_quantifieds,
             self.hoist_rule,
         ]
+        self._unrestricted_mutators = [
+            self.unrestricted_delete,
+            self.unrestricted_hoist_rule,
+        ]
         self._recombiners = [
             self.replace_node,
             self.insert_quantified,
@@ -276,6 +282,8 @@ def create(self):
         if self._population:
             if self._enable_mutation:
                 creators.extend(self._mutators)
+                if self._enable_unrestricted_creators:
+                    creators.extend(self._unrestricted_mutators)
             if self._enable_recombination:
                 creators.extend(self._recombiners)
         return self._create_tree(creators, individual1, individual2)
@@ -289,7 +297,8 @@ def mutate(self, individual=None):
 
         Supported mutation operators: :meth:`regenerate_rule`,
         :meth:`delete_quantified`, :meth:`replicate_quantified`,
-        :meth:`shuffle_quantifieds`, :meth:`hoist_rule`
+        :meth:`shuffle_quantifieds`, :meth:`hoist_rule`,
+        :meth:`unrestricted_delete`, :meth:`unrestricted_hoist_rule`
 
         :param ~grammarinator.runtime.Individual individual: The population item to
             be mutated.
@@ -300,7 +309,10 @@ def mutate(self, individual=None):
         # If you call this explicitly, then so be it, even if mutation is disabled.
         # If individual is None, population MUST exist.
         individual = self._ensure_individual(individual)
-        return self._create_tree(self._mutators, individual, None)
+        mutators = self._mutators
+        if self._enable_unrestricted_creators:
+            mutators.extend(self._unrestricted_mutators)
+        return self._create_tree(mutators, individual, None)
 
     def recombine(self, individual1=None, individual2=None):
         """
@@ -480,6 +492,25 @@ def delete_quantified(self, individual=None, _=None):
         # Return with the original root, whether the deletion was successful or not.
         return root
 
+    def unrestricted_delete(self, individual=None, _=None):
+        """
+        Remove a subtree rooted in any kind of rule node randomly without any
+        further restriction.
+
+        :param ~grammarinator.runtime.Individual individual: The population item to be mutated.
+        :return: The root of the modified tree.
+        :rtype: Rule
+        """
+        individual = self._ensure_individual(individual)
+        root, annot = individual.root, individual.annotations
+        options = [node for node in annot.rules if node != root]
+        if options:
+            removed_node = random.choice(options)
+            removed_node.remove()
+
+        # Return with the original root, whether the deletion was successful or not.
+        return root
+
     def replicate_quantified(self, individual=None, _=None):
         """
         Select a quantified sub-tree randomly, replicate it and insert it again if
@@ -541,3 +572,28 @@ def hoist_rule(self, individual=None, _=None):
                     return root
                 parent = parent.parent
         return root
+
+    def unrestricted_hoist_rule(self, individual=None, _=None):
+        """
+        Select two rule nodes from the input individual which are in
+        ancestor-descendant relationship (without type compatibility check) and
+        replace the ancestor with the selected descendant.
+
+        :param ~grammarinator.runtime.Individual individual: The population item to be mutated.
+        :return: The root of the modified tree.
+        :rtype: Rule
+        """
+        individual = self._ensure_individual(individual)
+        root, annot = individual.root, individual.annotations
+        for rule in random.sample(annot.rules, k=len(annot.rules)):
+            options = []
+            parent = rule.parent
+            while parent and parent != root:
+                if isinstance(parent, UnparserRule) and len(parent.children) > 1 and not rule.equalTokens(parent):
+                    options.append(parent)
+                parent = parent.parent
+
+            if options:
+                random.choice(options).replace(rule)
+                return root
+        return root