Support multi token generation in lexer (#18)

- Support multi token generation in lexer from single match
- Add test about multi token generation
- Version up plare

plare/__init__.py

@@ -1 +1 @@
-__version__ = "1.2.0"
+__version__ = "1.3.0"

plare/lexer.py

@@ -16,7 +16,10 @@ def __init__(
             str,
             list[
                 tuple[
-                    str, Callable[[str, T, int, int], Token | str] | type[Token] | str
+                    str,
+                    Callable[[str, T, int, int], Token | str | list[Token]]
+                    | type[Token]
+                    | str,
                 ]
             ],
         ],
@@ -59,6 +62,8 @@ def lex(self, var: str, src: str) -> Generator[Token]:
                         match token:
                             case Token():
                                 yield token
+                            case list():
+                                yield from token
                             case _:
                                 var = token
                 newlines = matched.count("\n")

tests/test_lexer.py

@@ -88,3 +88,35 @@ def test_lex_positive_integer_fail_on_tailing_plus():
     lexer = make_positive_integer_lexer()
     with pytest.raises(LexingError):
         list(lexer.lex("start", "+123+"))
+
+
+class SPACE(Token):
+    pass
+
+
+def test_lex_multiple_tokens_for_single_match():
+    lexer = Lexer(
+        {
+            "start": [
+                (
+                    r"[ \t\n]+",
+                    lambda matched, state, lineno, offset: [
+                        SPACE(c, lineno=lineno, offset=offset + i)
+                        for i, c in enumerate(matched)
+                    ],
+                ),
+            ]
+        }
+    )
+    tokens = list(lexer.lex("start", " \t\n"))
+    assert len(tokens) == 4
+    assert isinstance(tokens[0], SPACE)
+    assert tokens[0].lineno == 1
+    assert tokens[0].offset == 0
+    assert isinstance(tokens[1], SPACE)
+    assert tokens[1].lineno == 1
+    assert tokens[1].offset == 1
+    assert isinstance(tokens[2], SPACE)
+    assert tokens[2].lineno == 1
+    assert tokens[2].offset == 2
+    assert isinstance(tokens[3], EOF)