From f3c0d142135250650daee701425e5d39a4f7f6b9 Mon Sep 17 00:00:00 2001
From: Akos Kiss <akiss@inf.u-szeged.hu>
Date: Mon, 28 Oct 2024 08:17:19 +0100
Subject: [PATCH] Ensure that listener sees correct name when lexer rule refers
 to another lexer rule (#239)

Since token sub-rules (e.g., fragments) are not represented by
unique nodes in the tree, even though listeners are called when the
generation of a sub-rule starts or ends, the listeners only see the
top-level token node and the name of the top-level lexer rule.

This commit changes this by faking the top-level token node to have
the name of the sub-rule rule while generating the sub-rule.

Co-authored-by: Renata Hodovan <reni@inf.u-szeged.hu>
---
 grammarinator/runtime/generator.py | 16 ++++++++++
 tests/grammars/Fragment.g4         | 48 ++++++++++++++++++++++++++++++
 2 files changed, 64 insertions(+)
 create mode 100644 tests/grammars/Fragment.g4

diff --git a/grammarinator/runtime/generator.py b/grammarinator/runtime/generator.py
index 7fe6a0c..194f2b2 100644
--- a/grammarinator/runtime/generator.py
+++ b/grammarinator/runtime/generator.py
@@ -61,16 +61,28 @@ class UnlexerRuleContext(RuleContext):
 
     def __init__(self, gen, name, parent=None):
         if isinstance(parent, UnlexerRule):
+            # If parent node is also an UnlexerRule then this is a sub-rule and
+            # actually no child node is created, but the parent is kept as the
+            # current node
             super().__init__(gen, parent)
             self._start_depth = None
+            # So, save the name of the parent node and also that of the sub-rule
+            self._parent_name = parent.name
+            self._name = name
         else:
             node = UnlexerRule(name=name)
             if parent:
                 parent += node
             super().__init__(gen, node)
             self._start_depth = self.gen._size.depth
+            self._parent_name = None
+            self._name = None
 
     def __enter__(self):
+        # When entering a sub-rule, rename the current node to reflect the name
+        # of the sub-rule
+        if self._name is not None and self._parent_name is not None:
+            self.node.name = self._name
         super().__enter__()
         # Increment token count with the current token.
         self.gen._size.tokens += 1
@@ -84,6 +96,10 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         super().__exit__(exc_type, exc_val, exc_tb)
         if self._start_depth is not None:
             self.node.size.depth -= self._start_depth
+        # When exiting a sub-rule, change the name of the current node back to
+        # that of the parent
+        if self._name is not None and self._parent_name is not None:
+            self.node.name = self._parent_name
 
 
 class UnparserRuleContext(RuleContext):
diff --git a/tests/grammars/Fragment.g4 b/tests/grammars/Fragment.g4
new file mode 100644
index 0000000..814efdc
--- /dev/null
+++ b/tests/grammars/Fragment.g4
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2024 Renata Hodovan, Akos Kiss.
+ *
+ * Licensed under the BSD 3-Clause License
+ * <LICENSE.rst or https://opensource.org/licenses/BSD-3-Clause>.
+ * This file may not be copied, modified, or distributed except
+ * according to those terms.
+ */
+
+/*
+ * This test checks whether listeners see the correct names when lexer rules
+ * refer to other lexer rules (usually, fragment rules).
+ */
+
+// TEST-PROCESS: {grammar}.g4 -o {tmpdir}
+// TEST-GENERATE: {grammar}Generator.{grammar}Generator -r start -j 1 -n 1 --listener {grammar}Generator.CustomListener -o {tmpdir}/{grammar}%d.txt
+
+grammar Fragment;
+
+@header {
+
+from collections import Counter
+from grammarinator.runtime import Listener
+
+
+class CustomListener(Listener):
+
+    def __init__(self):
+        self.cnt_enters = Counter()
+        self.cnt_exits = Counter()
+
+    def enter_rule(self, node):
+        self.cnt_enters[node.name] += 1
+
+    def exit_rule(self, node):
+        self.cnt_exits[node.name] += 1
+        if node.name == 'start':
+            assert self.cnt_enters == Counter(start=1, A=1, B=1, C=1, D=1, E=1), self.cnt_enters
+            assert self.cnt_exits == Counter(start=1, A=1, B=1, C=1, D=1, E=1), self.cnt_exits
+}
+
+
+start : A;
+A : 'a' B ;
+B : 'b' C ;
+fragment C : 'c' D ;
+fragment D : 'd' E ;
+E : 'e' ;