From 089c17e1da7f88d1583516cde956f857499a62f8 Mon Sep 17 00:00:00 2001 From: Mark Date: Tue, 19 Jun 2018 07:57:10 +0200 Subject: [PATCH] Resolved technical errors, only lexing problems remain #52 --- src/mango/lexing/code_lexer.rs | 19 ++++++++++--------- src/mango/lexing/combi_lexer.rs | 17 ++++++++++++----- src/mango/lexing/util/lex_all.rs | 11 +++++++++++ src/mango/token/collect/all.rs | 2 +- src/mango/token/tokens/association.rs | 2 +- src/mango/util/codeparts/operator.rs | 2 +- 6 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/mango/lexing/code_lexer.rs b/src/mango/lexing/code_lexer.rs index 59134bda..64a90955 100644 --- a/src/mango/lexing/code_lexer.rs +++ b/src/mango/lexing/code_lexer.rs @@ -18,6 +18,8 @@ use mango::token::tokens::ParenthesisOpenToken; use mango::token::tokens::StartBlockToken; use mango::token::Tokens; use mango::util::collection::Queue; +use mango::util::strslice::char_ops::CharOps; +use mango::util::strslice::charsliceto; pub struct CodeLexer { indent: i32, @@ -145,11 +147,13 @@ impl SubLexer for CodeLexer { // Association (before operator) if let Match(token) = reader.matches(&AssociationToken::subpattern()) { debug_assert!(token.chars().last().unwrap() == '='); - if token.chars().count() > 1 { - panic!(); // TODO - return SubLexerResult::single( - (Tokens::Association(AssociationToken::from_unprefixed())), - ); + if token.char_len() > 1 { + match AssociationToken::from_str(charsliceto(token, -1)) { + Ok(association) => { + return SubLexerResult::single((Tokens::Association(association))) + } + Err(msg) => panic!(format!("Invalid association prefix: {}", msg)), + } } else { return SubLexerResult::single( (Tokens::Association(AssociationToken::from_unprefixed())), @@ -173,10 +177,7 @@ impl SubLexer for CodeLexer { // If the code gets here, it did not recognize the text as any token return match reader.matches(r"[^\s]+") { Match(word) => SubLexerResult::single(Tokens::Unlexable(UnlexableToken::new(word))), - NoMatch() => { - println!("END {:?}", reader); // todo: tmp - panic!("Do not know how to proceed with parsing") - } + NoMatch() => panic!("Do not know how to proceed with parsing"), EOF() => { if self.indent <= 0 { return SubLexerResult::End; diff --git a/src/mango/lexing/combi_lexer.rs b/src/mango/lexing/combi_lexer.rs index 9a3965ec..a31a02db 100644 --- a/src/mango/lexing/combi_lexer.rs +++ b/src/mango/lexing/combi_lexer.rs @@ -81,16 +81,22 @@ mod tests { use mango::token::tokens::ParenthesisOpenToken; use mango::token::tokens::StartBlockToken; use mango::token::Tokens; + use mango::util::encdec::to_text::ToText; use std::cell::RefCell; use std::rc::Rc; fn assert_text_to_tokens(text: &str, tokens: Vec) { + let expected = LexList::from_tokens(tokens); + let actual = lex_all(&mut CombiLexer::new(Box::new(StringReader::new( + text.to_owned(), + )))); assert_eq!( - LexList::from_tokens(tokens), - lex_all(&mut CombiLexer::new(Box::new(StringReader::new( - text.to_owned() - )))) - ) + expected, + actual, + "expected: {}\nactual: {}", + expected.to_text(), + actual.to_text(), + ); } #[test] @@ -115,6 +121,7 @@ mod tests { Tokens::Literal(LiteralToken::Int(0)), Tokens::EndStatement(EndStatementToken::new_end_line()), Tokens::Keyword(KeywordToken::from_str("for".to_owned()).unwrap()), + Tokens::Identifier(IdentifierToken::from_str("x".to_owned()).unwrap()), Tokens::Operator(OperatorToken::from_str("<").unwrap()), Tokens::Literal(LiteralToken::Int(128)), Tokens::EndStatement(EndStatementToken::new_end_line()), diff --git a/src/mango/lexing/util/lex_all.rs b/src/mango/lexing/util/lex_all.rs index 82ee0c1d..e10557a8 100644 --- a/src/mango/lexing/util/lex_all.rs +++ b/src/mango/lexing/util/lex_all.rs @@ -1,6 +1,7 @@ use mango::lexing::typ::Lexer; use mango::lexing::typ::MaybeToken; use mango::token::Tokens; +use mango::util::encdec::ToText; /// Represents all the lex tokens in a source. #[derive(PartialEq, Eq, Debug)] @@ -18,6 +19,16 @@ impl LexList { } } +impl ToText for LexList { + fn to_text(&self) -> String { + self.tokens + .iter() + .map(|token| token.to_text()) + .collect::>() + .join(" ") + } +} + pub fn lex_all(lexer: &mut Lexer) -> LexList { let mut list = Vec::with_capacity(512); while let MaybeToken::Token(token) = lexer.lex() { diff --git a/src/mango/token/collect/all.rs b/src/mango/token/collect/all.rs index 618cc741..70576f40 100644 --- a/src/mango/token/collect/all.rs +++ b/src/mango/token/collect/all.rs @@ -55,6 +55,6 @@ mod tests { #[test] fn test_tokens_size() { - assert!(size_of::() < 32, size_of::()); + assert!(size_of::() <= 40, size_of::()); } } diff --git a/src/mango/token/tokens/association.rs b/src/mango/token/tokens/association.rs index 1c857ee2..a5be9028 100644 --- a/src/mango/token/tokens/association.rs +++ b/src/mango/token/tokens/association.rs @@ -28,7 +28,7 @@ impl AssociationToken { } pub fn subpattern() -> String { - format!("{}=", Symbol::subpattern()) + format!(r"(?:{})?=", Symbol::subpattern()) } } diff --git a/src/mango/util/codeparts/operator.rs b/src/mango/util/codeparts/operator.rs index 22d39375..31dfb043 100644 --- a/src/mango/util/codeparts/operator.rs +++ b/src/mango/util/codeparts/operator.rs @@ -35,7 +35,7 @@ impl Symbol { /// Generate an eager subpattern to match tokens, that can be composed in a regular expression. pub fn subpattern() -> &'static str { - r"(\+|\-|\*|/)" + r"[\-+*/]" } }