Lexer infrastructure for testing #52

src/mango/io/fortest/fromstr.rs

@@ -30,6 +30,7 @@ impl Reader for StringReader {
         REXCACHE.with(|rl| {
             let mut rexlib = rl.borrow_mut();
             let rex = rexlib.make_or_get(subpattern);
+            println!("{:?}", rex);
         });
         ReaderResult::NoMatch() // TODO
     }

src/mango/io/typ.rs

@@ -12,6 +12,8 @@ pub trait Reader {
     //    fn equals(&mut self, texts: Vec<&str>) -> ReaderResult;
 
     /// Checks whether the code from the current position matches a regex pattern.
+    ///
+    /// This has to eventually return EOF, after which it should not be called again.
     fn matches(&mut self, subpattern: &str) -> ReaderResult;
 }
 

src/mango/lexing/code_lexer.rs

@@ -38,7 +38,7 @@ pub struct CodeLexer {
 }
 
 impl CodeLexer {
-    fn new(reader: Rc<RefCell<Reader>>) -> Self {
+    pub fn new(reader: Rc<RefCell<Reader>>) -> Self {
         CodeLexer {
             reader: reader,
             reader_or_delegate: ReaderOrDelegate::Reader(),
@@ -186,7 +186,13 @@ impl Lexer for CodeLexer {
                 }
 
                 // TODO: specify the unlexable word
-                return Token(Tokens::Unlexable(UnlexableToken::new("TODO".to_owned())));
+                let unknown_word = self.reader.borrow_mut().matches(" *[^\\s]+");
+                if let Match(word) = unknown_word {
+                    return Token(Tokens::Unlexable(UnlexableToken::new(word)));
+                } else {
+                    // todo: handle better someday
+                    panic!("Do not know how to proceed with parsing");
+                }
             }
         }
     }
@@ -198,3 +204,27 @@ impl Lexer for CodeLexer {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::CodeLexer;
+    use mango::io::fortest::StringReader;
+    use mango::io::typ::Reader;
+    use mango::lexing::util::lex_all::{lex_all, LexList};
+    use std::cell::RefCell;
+    use std::rc::Rc;
+
+    #[test]
+    fn test_lexing() {
+        assert_eq!(
+            LexList::from_tokens(vec![]),
+            lex_all(Rc::new(RefCell::new(StringReader::new(
+                "let x = 0\nfor x < 128\n\tx += 1\n".to_owned(),
+            ))))
+        )
+        //        assert_eq!(1, cnt, "No item in ProblemCollector");
+    }
+
+    #[test]
+    fn test_lexing_delegation() {}
+}

src/mango/lexing/mod.rs

@@ -5,3 +5,5 @@ pub mod code_lexer;
 pub mod comment_lexer;
 
 pub mod string_lexer;
+
+pub mod util;

src/mango/lexing/util/lex_all.rs

@@ -0,0 +1,34 @@
+use mango::io::typ::Reader;
+use mango::lexing::code_lexer::CodeLexer;
+use mango::lexing::typ::Lexer;
+use mango::lexing::typ::MaybeToken;
+use mango::token::Token;
+use mango::token::Tokens;
+use std::cell::RefCell;
+use std::rc::Rc;
+
+/// Represents all the lex tokens in a source.
+#[derive(PartialEq, Eq, Debug)]
+pub struct LexList {
+    tokens: Vec<Tokens>,
+}
+
+impl LexList {
+    pub fn from_tokens(tokens: Vec<Tokens>) -> Self {
+        LexList { tokens }
+    }
+
+    pub fn from_reader(reader: Rc<RefCell<Reader>>) -> Self {
+        lex_all(reader)
+    }
+}
+
+pub fn lex_all(reader: Rc<RefCell<Reader>>) -> LexList {
+    let mut list = Vec::with_capacity(512);
+    let mut lexer = CodeLexer::new(reader);
+    while let MaybeToken::Token(token) = lexer.lex() {
+        list.push(token)
+    }
+    list.shrink_to_fit();
+    LexList { tokens: list }
+}

src/mango/lexing/util/mod.rs

@@ -0,0 +1 @@
+pub mod lex_all;

src/mango/util/strslice/slice.rs

@@ -12,10 +12,6 @@ pub fn charslice<S: Into<String>>(text: S, start: isize, end: isize) -> String {
             -start as usize <= charcount,
             "charslice: if 'start' is negative, the magnitude may not exceed the length"
         );
-        println!(
-            ">> charcount as isize + start = {} + {}",
-            charcount as isize, start
-        );
         from = (charcount as isize + start) as usize;
     } else {
         from = start as usize;