diff --git a/plare/__init__.py b/plare/__init__.py index c68196d..67bc602 100644 --- a/plare/__init__.py +++ b/plare/__init__.py @@ -1 +1 @@ -__version__ = "1.2.0" +__version__ = "1.3.0" diff --git a/plare/lexer.py b/plare/lexer.py index 2edf96c..ad235fc 100644 --- a/plare/lexer.py +++ b/plare/lexer.py @@ -16,7 +16,10 @@ def __init__( str, list[ tuple[ - str, Callable[[str, T, int, int], Token | str] | type[Token] | str + str, + Callable[[str, T, int, int], Token | str | list[Token]] + | type[Token] + | str, ] ], ], @@ -59,6 +62,8 @@ def lex(self, var: str, src: str) -> Generator[Token]: match token: case Token(): yield token + case list(): + yield from token case _: var = token newlines = matched.count("\n") diff --git a/tests/test_lexer.py b/tests/test_lexer.py index 7c53f41..b7e0d65 100644 --- a/tests/test_lexer.py +++ b/tests/test_lexer.py @@ -88,3 +88,35 @@ def test_lex_positive_integer_fail_on_tailing_plus(): lexer = make_positive_integer_lexer() with pytest.raises(LexingError): list(lexer.lex("start", "+123+")) + + +class SPACE(Token): + pass + + +def test_lex_multiple_tokens_for_single_match(): + lexer = Lexer( + { + "start": [ + ( + r"[ \t\n]+", + lambda matched, state, lineno, offset: [ + SPACE(c, lineno=lineno, offset=offset + i) + for i, c in enumerate(matched) + ], + ), + ] + } + ) + tokens = list(lexer.lex("start", " \t\n")) + assert len(tokens) == 4 + assert isinstance(tokens[0], SPACE) + assert tokens[0].lineno == 1 + assert tokens[0].offset == 0 + assert isinstance(tokens[1], SPACE) + assert tokens[1].lineno == 1 + assert tokens[1].offset == 1 + assert isinstance(tokens[2], SPACE) + assert tokens[2].lineno == 1 + assert tokens[2].offset == 2 + assert isinstance(tokens[3], EOF)