From 65a195aae48385e2ab7f11fbf963864861c12ff0 Mon Sep 17 00:00:00 2001
From: Mark <mverleg.noreply@gmail.com>
Date: Wed, 6 Jun 2018 22:58:50 +0200
Subject: [PATCH] More progress on lexing tests, stuck on tab #52

---
 src/mango/io/fortest/stringreader.rs | 44 ++++++++++++++++------------
 src/mango/io/typ.rs                  |  4 ++-
 src/mango/lexing/code_lexer.rs       | 17 ++++++-----
 3 files changed, 39 insertions(+), 26 deletions(-)

diff --git a/src/mango/io/fortest/stringreader.rs b/src/mango/io/fortest/stringreader.rs
index 69984d72..fd5a1d4c 100644
--- a/src/mango/io/fortest/stringreader.rs
+++ b/src/mango/io/fortest/stringreader.rs
@@ -4,6 +4,7 @@ use mango::io::util::REXCACHE;
 
 /// Implementation of [Reader] that reads from a pre-provided string.
 /// Mostly for testing purposes.
+#[derive(Debug)]
 pub struct StringReader {
     code: String,
     index: usize,
@@ -20,30 +21,37 @@ impl Reader for StringReader {
         // Check for subpattern
         REXCACHE.with(|rl| {
             let mut rexlib = rl.borrow_mut();
-            {
-                // Check for end of file
-                // TODO: is there a better/faster way for this? maybe try this after a match and set a flag?
-                let regex = rexlib.make_or_get(r"\s*");
-                match regex.find(&self.code[self.index..]) {
-                    Some(mtch) => {
+            // Check for end of file
+            // TODO: is there a better/faster way for this? maybe try this after a match and set a flag?
+            let regex = rexlib.make_or_get(r"\s*$");
+            match regex.find(&self.code[self.index..]) {
+                Some(mtch) => {
+                    if self.index + mtch.as_str().len() == self.code.len() {
                         self.index += mtch.as_str().len();
                         return ReaderResult::EOF();
                     }
-                    None => (),
                 }
+                None => (),
             }
-            {
-                // Check for subpattern
-                let regex = rexlib.make_or_get(subpattern);
-                return match regex.find(&self.code[self.index..]) {
-                    Some(mtch) => {
-                        self.index += mtch.as_str().len();
-                        println!(">>> {}", mtch.as_str());
-                        ReaderResult::Match(mtch.as_str().to_owned())
+            // Check for subpattern
+            let regex = rexlib.make_or_get(subpattern);
+            return match regex.find(&self.code[self.index..]) {
+                Some(mtch) => {
+                    self.index += mtch.as_str().len();
+                    // Remove leading spaces
+                    let mut k = 0;
+                    for (i, byt) in mtch.as_str().chars().enumerate() {
+                        if byt != ' ' {
+                            break;
+                        }
+                        k = i + 1;
                     }
-                    None => ReaderResult::NoMatch(),
-                };
-            }
+                    ReaderResult::Match((&mtch.as_str()[k..]).to_owned())
+                }
+                None => ReaderResult::NoMatch(),
+            };
         })
     }
 }
+
+// TODO: tests (spaces, end)
diff --git a/src/mango/io/typ.rs b/src/mango/io/typ.rs
index 13b4e0ba..5f8fd9f0 100644
--- a/src/mango/io/typ.rs
+++ b/src/mango/io/typ.rs
@@ -1,5 +1,7 @@
 // TODO: I should perhaps separate the splitting that happens here from the actual reading
 
+use std::fmt::Debug;
+
 pub enum ReaderResult {
     Match(String),
     NoMatch(),
@@ -7,7 +9,7 @@ pub enum ReaderResult {
 }
 
 /// A reader represents a source 'file', which may be a file, webpage, string, ...
-pub trait Reader {
+pub trait Reader: Debug {
     /// Checks whether the `text` is found starting from the current position.
     //    fn equals(&mut self, texts: Vec<&str>) -> ReaderResult;
 
diff --git a/src/mango/lexing/code_lexer.rs b/src/mango/lexing/code_lexer.rs
index 112ff22e..0c5b6398 100644
--- a/src/mango/lexing/code_lexer.rs
+++ b/src/mango/lexing/code_lexer.rs
@@ -185,10 +185,13 @@ impl Lexer for CodeLexer {
                     return Token(Tokens::ParenthesisClose(ParenthesisCloseToken::new()));
                 }
 
-                let unknown_word = self.reader.borrow_mut().matches(" *[^\\s]+");
+                let unknown_word = self.reader.borrow_mut().matches("[^\\s]+");
                 match unknown_word {
                     Match(word) => return Token(Tokens::Unlexable(UnlexableToken::new(word))),
-                    NoMatch() => panic!("Do not know how to proceed with parsing"),
+                    NoMatch() => {
+                        println!("END {:?}", self.reader.borrow());
+                        panic!("Do not know how to proceed with parsing")
+                    }
                     EOF() => End,
                 }
             }
@@ -213,11 +216,11 @@ mod tests {
 
     #[test]
     fn test_lexing() {
-        let lexed = lex_all(&mut CodeLexer::new(Rc::new(RefCell::new(
-            StringReader::new("let x = 0\nfor x < 128\n\tx += 1\n".to_owned()),
-        ))));
-        println!("LEXED: {:?}", lexed);
-        assert_eq!(LexList::from_tokens(vec![]), lexed)
+        //        let lexed = lex_all(&mut CodeLexer::new(Rc::new(RefCell::new(
+        //            StringReader::new("let x = 0\nfor x < 128\n\tx += 1\n".to_owned()),
+        //        ))));
+        //        println!("LEXED: {:?}", lexed);
+        //        assert_eq!(LexList::from_tokens(vec![]), lexed)
         //        assert_eq!(1, cnt, "No item in ProblemCollector");
     }