From 73bcf82d1e1364fe2df32a746a2482f1f6a4e679 Mon Sep 17 00:00:00 2001 From: Mark Date: Mon, 4 Jun 2018 21:49:44 +0200 Subject: [PATCH] More lexing possibilities #52 --- src/mango/lexing/code_lexer.rs | 16 ++++++++++++++-- src/mango/token/tests.rs | 2 +- src/mango/token/tokens/association.rs | 6 +++++- src/mango/util/strslice/slice.rs | 2 +- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/mango/lexing/code_lexer.rs b/src/mango/lexing/code_lexer.rs index 5574f253..a371a7dc 100644 --- a/src/mango/lexing/code_lexer.rs +++ b/src/mango/lexing/code_lexer.rs @@ -4,6 +4,7 @@ use mango::lexing::string_lexer::StringLexer; use mango::lexing::typ::Lexer; use mango::lexing::typ::MaybeToken; use mango::token::special::UnlexableToken; +use mango::token::tokens::AssociationToken; use mango::token::tokens::EndBlockToken; use mango::token::tokens::EndStatementToken; use mango::token::tokens::IdentifierToken; @@ -155,6 +156,19 @@ impl Lexer for CodeLexer { self.reader_or_delegate = ReaderOrDelegate::Delegate(sublexer); return self.lex(); } + // Association (before operator) + let association_match_res = self + .reader + .borrow_mut() + .matches(&AssociationToken::subpattern()); + if let Match(token) = association_match_res { + if token.chars().last().unwrap() == '=' { + // return Token(Tokens::Association(AssociationToken::from_str(token[..1]).unwrap())); + return Token(Tokens::Association(AssociationToken::from_unprefixed())); // TODO + } else { + return Token(Tokens::Association(AssociationToken::from_unprefixed())); + } + } // Operator let operator_match_res = self .reader @@ -163,8 +177,6 @@ impl Lexer for CodeLexer { if let Match(token) = operator_match_res { return Token(Tokens::Operator(OperatorToken::from_str(&token).unwrap())); } - // Association - // todo // Grouping symbols if let Match(_) = self.reader.borrow_mut().matches("(") { return Token(Tokens::ParenthesisOpen(ParenthesisOpenToken::new())); diff --git a/src/mango/token/tests.rs b/src/mango/token/tests.rs index 732d01a3..6b4ce6b8 100644 --- a/src/mango/token/tests.rs +++ b/src/mango/token/tests.rs @@ -15,7 +15,7 @@ fn test_tokens_eq() { Keyword(KeywordToken::from_str("let").unwrap()), Keyword(KeywordToken::from_str("mut").unwrap()), Identifier(IdentifierToken::from_name(my_var)), - Association(AssociationToken::from_unmutated()), + Association(AssociationToken::from_unprefixed()), Literal(LiteralToken::int(21)), EndStatement(EndStatementToken::new_semicolon()), Identifier(IdentifierToken::from_name(my_var)), diff --git a/src/mango/token/tokens/association.rs b/src/mango/token/tokens/association.rs index a8c44da2..1c857ee2 100644 --- a/src/mango/token/tokens/association.rs +++ b/src/mango/token/tokens/association.rs @@ -11,7 +11,7 @@ pub struct AssociationToken { } impl AssociationToken { - pub fn from_unmutated() -> Self { + pub fn from_unprefixed() -> Self { AssociationToken { symbol: Option::None, } @@ -26,6 +26,10 @@ impl AssociationToken { symbol: Option::Some(symbol), } } + + pub fn subpattern() -> String { + format!("{}=", Symbol::subpattern()) + } } impl ToText for AssociationToken { diff --git a/src/mango/util/strslice/slice.rs b/src/mango/util/strslice/slice.rs index 88eea284..3260c401 100644 --- a/src/mango/util/strslice/slice.rs +++ b/src/mango/util/strslice/slice.rs @@ -29,7 +29,7 @@ pub fn charslice>(text: S, start: isize, end: isize) -> String { let new_end = (charcount as isize + end) as usize; assert!( new_end >= from, - "charslice: 'start' may not be before 'end' (end was positive)" + "charslice: 'start' may not be before 'end' (end was negative)" ); length = new_end - from; } else {