Some progress on lexing literals #52

mangolang · Jun 17, 2018 · 28bc91a · 28bc91a
1 parent 8b4a125
commit 28bc91a
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 38 deletions.
diff --git a/src/mango/lexing/code_lexer.rs b/src/mango/lexing/code_lexer.rs
@@ -6,6 +6,7 @@ use mango::lexing::typ::MaybeToken;
 use mango::lexing::typ::SubLexer;
 use mango::lexing::typ::SubLexerResult;
 use mango::token::special::UnlexableToken;
+use mango::token::tokens::literal::LiteralToken;
 use mango::token::tokens::AssociationToken;
 use mango::token::tokens::EndBlockToken;
 use mango::token::tokens::EndStatementToken;
@@ -17,8 +18,6 @@ use mango::token::tokens::ParenthesisOpenToken;
 use mango::token::tokens::StartBlockToken;
 use mango::token::Tokens;
 use mango::util::collection::Queue;
-use std::cell::RefCell;
-use std::rc::Rc;
 
 pub struct CodeLexer {
     indent: i32,
@@ -130,14 +129,19 @@ impl SubLexer for CodeLexer {
                 IdentifierToken::from_str(word).unwrap(),
             ));
         }
-        //        // Literal
-        //        let string_match_res = reader.matches("[a-z]?\"");
-        //        if let Match(_) = string_match_res {
-        //            let sublexer: Box<Lexer> =
-        //                Box::new(StringLexer::new_double_quoted(self.reader.clone()));
-        //            self.reader_or_delegate = ReaderOrDelegate::Delegate(sublexer);
-        //            return self.lex();
-        //        }
+        // Literal
+        if let Match(_) = reader.matches("[a-z]?\"") {
+            return Delegate(Box::new(StringLexer::new_double_quoted()));
+        }
+        if let Match(nr) = reader.matches(LiteralToken::subpattern_int()) {
+            let value = LiteralToken::parse_int(nr);
+            return SubLexerResult::single(Tokens::Literal(LiteralToken::Int(value)));
+        }
+        if let Match(nr) = reader.matches(LiteralToken::subpattern_real()) {
+            let value = LiteralToken::parse_real(nr);
+            return SubLexerResult::single(Tokens::Literal(LiteralToken::Real(value)));
+        }
+
         //        // Association (before operator)
         //        let association_match_res = self
         //            .reader

diff --git a/src/mango/lexing/combi_lexer.rs b/src/mango/lexing/combi_lexer.rs
@@ -1,26 +1,12 @@
 use mango::io::typ::Reader;
-use mango::io::typ::ReaderResult::*;
 use mango::lexing::code_lexer::CodeLexer;
-use mango::lexing::string_lexer::StringLexer;
 use mango::lexing::typ::Lexer;
 use mango::lexing::typ::MaybeToken;
 use mango::lexing::typ::SubLexer;
 use mango::lexing::typ::SubLexerResult;
-use mango::token::special::UnlexableToken;
-use mango::token::tokens::AssociationToken;
-use mango::token::tokens::EndBlockToken;
-use mango::token::tokens::EndStatementToken;
-use mango::token::tokens::IdentifierToken;
-use mango::token::tokens::KeywordToken;
-use mango::token::tokens::OperatorToken;
-use mango::token::tokens::ParenthesisCloseToken;
-use mango::token::tokens::ParenthesisOpenToken;
-use mango::token::tokens::StartBlockToken;
 use mango::token::Tokens;
 use mango::util::collection::Queue;
 use mango::util::collection::Stack;
-use std::cell::RefCell;
-use std::rc::Rc;
 
 pub struct CombiLexer {
     reader: Box<Reader>,

diff --git a/src/mango/lexing/string_lexer.rs b/src/mango/lexing/string_lexer.rs
@@ -2,10 +2,10 @@ use mango::io::typ::Reader;
 use mango::io::typ::ReaderResult::*;
 use mango::lexing::typ::Lexer;
 use mango::lexing::typ::MaybeToken;
+use mango::lexing::typ::SubLexer;
+use mango::lexing::typ::SubLexerResult;
 use mango::token::tokens::LiteralToken;
 use mango::token::Tokens;
-use std::cell::RefCell;
-use std::rc::Rc;
 
 pub enum StringType {
     SingleQuotedInline,
@@ -16,33 +16,31 @@ pub enum StringType {
 /// Lexes a string literal token.
 // Starts after the opening quote and expected to consume until closing quote.
 pub struct StringLexer {
-    reader: Rc<RefCell<Reader>>,
     typ: StringType,
 }
 
 impl StringLexer {
     // TODO: support other types of strings
-    pub fn new_double_quoted(reader: Rc<RefCell<Reader>>) -> Self {
+    pub fn new_double_quoted() -> Self {
         StringLexer {
-            reader,
             typ: StringType::DoubleQuotedInline,
         }
     }
 }
 
-impl Lexer for StringLexer {
-    fn lex(&mut self) -> MaybeToken {
+impl SubLexer for StringLexer {
+    fn lex_pass(&mut self, reader: &mut Box<Reader>) -> SubLexerResult {
         // TODO: perhaps there's a library that does parsing a string with escape characters
         // TODO: doesn't handle escaping etc at all now
         // TODO: this is going to have a problem if `matches` automatically eats whitespace
-        match self.reader.borrow_mut().matches("[^\"\\n]*") {
-            Match(value) => return MaybeToken::Token(Tokens::Literal(LiteralToken::string(value))),
+        match reader.matches("[^\"\\n]*") {
+            Match(value) => {
+                return SubLexerResult::single(Tokens::Literal(LiteralToken::string(value)))
+            }
             NoMatch() => panic!("failed to parse string"), // This can't really go wrong since empty pattern matches
-            EOF() => return MaybeToken::Token(Tokens::Literal(LiteralToken::string("".to_owned()))), // Unclosed string literal, let code parser deal with it
+            EOF() => {
+                return SubLexerResult::single(Tokens::Literal(LiteralToken::string("".to_owned())))
+            } // Unclosed string literal, let code parser deal with it
         }
     }
-
-    //    fn get_reader(&self) -> Rc<RefCell<Reader>> {
-    //        self.reader.clone()
-    //    }
 }
diff --git a/src/mango/token/tokens/literal.rs b/src/mango/token/tokens/literal.rs
@@ -1,6 +1,8 @@
 use mango::token::Token;
 use mango::util::encdec::ToText;
 use mango::util::numtype::f64eq;
+use mango::util::parsetxt::int::parse_int;
+use mango::util::parsetxt::real::parse_real;
 
 // LATER: it is likely that this will be refactored when the type system is in place.
 
@@ -40,8 +42,19 @@ impl LiteralToken {
     pub fn subpattern_real() -> &'static str {
         // TODO: do I want to allow numbers to start with a period?
         // TODO: for now, only base10 for reals (would 8b11e2 be 9*8^2 or 9*10^2?)
+        // TODO: does not deal with NaN of infinity
         r"(?:\+|-*)(?:\d(?:_?\d)*\.\d(?:_?\d)*|\d(?:_?\d)*\.|\.\d(?:_?\d)*)(?:e(?:\+|-|)\d(?:_?\d)*)?"
     }
+
+    /// Parse a string matching [subpattern_int] to an i64 integer. Overflow is possible.
+    pub fn parse_int(text: String) -> i64 {
+        parse_int(text).unwrap()
+    }
+
+    /// Parse a string matching [subpattern_real] to a f64 real. Loss of precision or overflow are possible.
+    pub fn parse_real(text: String) -> f64eq {
+        f64eq::new(parse_real(text).unwrap())
+    }
 }
 
 impl ToText for LiteralToken {