Merge pull request #1347 from MegaIng/fix-1345-2

Fix 1345 attempt two

lark/parsers/lalr_interactive_parser.py

@@ -102,9 +102,14 @@ def choices(self):
     def accepts(self):
         """Returns the set of possible tokens that will advance the parser into a new valid state."""
         accepts = set()
+        conf_no_callbacks = copy(self.parser_state.parse_conf)
+        # We don't want to call callbacks here since those might have arbitrary side effects
+        # and are unnecessarily slow.
+        conf_no_callbacks.callbacks = {}
         for t in self.choices():
             if t.isupper(): # is terminal?
                 new_cursor = copy(self)
+                new_cursor.parser_state.parse_conf = conf_no_callbacks
                 try:
                     new_cursor.feed_token(self.lexer_thread._Token(t, ''))
                 except UnexpectedToken:

lark/parsers/lalr_parser.py

@@ -147,7 +147,7 @@ def feed_token(self, token, is_end=False):
                 else:
                     s = []
 
-                value = callbacks[rule](s)
+                value = callbacks[rule](s) if callbacks else s
 
                 _action, new_state = states[state_stack[-1]][rule.origin.name]
                 assert _action is Shift

tests/test_parser.py

@@ -2595,6 +2595,28 @@ def test_iter_parse(self):
             assert next(i) == 'a'
             assert next(i) == 'b'
 
+        @unittest.skipIf(PARSER!='lalr', "interactive_parser is only implemented for LALR at the moment")
+        def test_interactive_treeless_transformer(self):
+            grammar = r"""
+                start: SYM+
+
+                SYM: "a" | "b"
+            """
+
+            class SYMTransformer(lark.visitors.Transformer):
+                def SYM(self, token):
+                    return {"a": 1, "b": 2}[str(token)]
+
+            parser = _Lark(grammar, transformer=SYMTransformer())
+            res = parser.parse("aba")
+            self.assertEqual(res.children, [1, 2, 1])
+            ip = parser.parse_interactive("aba")
+            ip.exhaust_lexer()
+            # Previously `accepts` would call `SYMTransformer.SYM` with `Token('SYM', '')`, which would cause an error.
+            self.assertEqual(ip.accepts(), {"$END", "SYM"})
+            res = ip.feed_eof()
+            self.assertEqual(res.children, [1, 2, 1])
+
         @unittest.skipIf(PARSER!='lalr', "Tree-less mode is only supported in lalr")
         def test_default_in_treeless_mode(self):
             grammar = r"""