Trying to let the lexer deal with indentation somewhat elegantly #56

README.rst

@@ -65,6 +65,12 @@ These instructions were tested on Ubuntu 18.4 (using Bash). It should also work
 	cargo test --all
 	cargo run --bin mango-cli
 
+  or to build a fast, release-mode native binary:
+
+.. code:: bash
+
+    RUSTFLAGS="-C target-cpu=native" cargo build --release
+
 * To deploy the web version in release mode:
 
 .. code:: bash

src/mango/io/fortest/fromstr.rs

@@ -1,4 +1,5 @@
 use mango::io::typ::Reader;
+use mango::io::typ::ReaderResult;
 use mango::io::util::REXCACHE;
 
 /// Implementation of [Reader] that reads from a pre-provided string.
@@ -15,19 +16,21 @@ impl StringReader {
 }
 
 impl Reader for StringReader {
-    fn equals(&mut self, text: &str) -> bool {
-        if &self.code[self.index..self.index + text.len()] == text {
-            self.index += text.len();
-            return true;
-        }
-        false
-    }
+    //    fn equals(&mut self, texts: Vec<&str>) -> ReaderResult {
+    //        for text in texts {
+    //            if &self.code[self.index..self.index + text.len()] == text {
+    //                self.index += text.len();
+    //                return ReaderResult::Match(self.code[self.index..self.index + text.len()])
+    //            }
+    //        }
+    //        ReaderResult::NoMatch()
+    //    }
 
-    fn matches(&mut self, subpattern: &str) -> Option<String> {
+    fn matches(&mut self, subpattern: &str) -> ReaderResult {
         REXCACHE.with(|rl| {
             let mut rexlib = rl.borrow_mut();
             let rex = rexlib.make_or_get(subpattern);
         });
-        Option::None // TODO
+        ReaderResult::NoMatch() // TODO
     }
 }

src/mango/io/typ.rs

@@ -1,10 +1,18 @@
+// TODO: I should perhaps separate the splitting that happens here from the actual reading
+
+pub enum ReaderResult {
+    Match(String),
+    NoMatch(),
+    EOF(),
+}
+
 /// A reader represents a source 'file', which may be a file, webpage, string, ...
 pub trait Reader {
     /// Checks whether the `text` is found starting from the current position.
-    fn equals(&mut self, text: &str) -> bool;
+    //    fn equals(&mut self, texts: Vec<&str>) -> ReaderResult;
 
     /// Checks whether the code from the current position matches a regex pattern.
-    fn matches(&mut self, subpattern: &str) -> Option<String>;
+    fn matches(&mut self, subpattern: &str) -> ReaderResult;
 }
 
 pub trait Writer {

src/mango/lexing/code_lexer.rs

@@ -0,0 +1,59 @@
+use mango::io::typ::Reader;
+use mango::io::typ::ReaderResult::*;
+use mango::lexing::typ::Lexer;
+use mango::lexing::typ::MaybeToken;
+use mango::token::special::UnlexableToken;
+use mango::token::tokens::ParenthesisCloseToken;
+use mango::token::tokens::ParenthesisOpenToken;
+use mango::token::Tokens;
+use mango::util::codeparts::Keyword;
+use std::collections::VecDeque;
+
+pub struct CodeLexer<'r> {
+    reader: &'r mut Reader,
+    indent: i32,
+    // This is unfortunate, would not be needed with 'yield' but is now for indents
+    buffer: VecDeque<Tokens>,
+}
+
+impl<'r> CodeLexer<'r> {
+    fn new(reader: &'r mut Reader) -> Self {
+        CodeLexer {
+            reader,
+            indent: 0,
+            buffer: VecDeque::with_capacity(16),
+        }
+    }
+}
+
+impl<'r> Lexer<'r> for CodeLexer<'r> {
+    fn lex(&mut self) -> MaybeToken {
+        // If there is a buffer due to indentation or continuations, return from that.
+        if !self.buffer.is_empty() {
+            return MaybeToken::Token(self.buffer.pop_front().unwrap());
+        }
+        if let Match(word) = self.reader.matches("\\.\\.\\.") {
+            // Line continuation has no token, it just continues on the next line.
+            if let Match(word) = self.reader.matches("\\n\\r?") {
+                // There should always be a newline after continuations, so that they can be ignored together.
+            } else if let Match(word) = self.reader.matches("[^\\n]*\\n\\r?") {
+                return MaybeToken::Token(Tokens::Unlexable(UnlexableToken::new(word)));
+            } else {
+                // TODO: I don't know yet how to deal with continuation followed by end of file
+                panic!()
+            }
+        }
+        // Indentation done; do the rest of lexing.
+        if let Match(word) = self.reader.matches("(") {
+            return MaybeToken::Token(Tokens::ParenthesisOpen(ParenthesisOpenToken::new()));
+        }
+        if let Match(word) = self.reader.matches(")") {
+            return MaybeToken::Token(Tokens::ParenthesisClose(ParenthesisCloseToken::new()));
+        }
+
+        // TODO: a lot more
+
+        // TODO: specify the unlexable word
+        return MaybeToken::Token(Tokens::Unlexable(UnlexableToken::new("TODO".to_owned())));
+    }
+}

src/mango/lexing/comment_lexer.rs

@@ -0,0 +1 @@
+

src/mango/lexing/mod.rs

@@ -1 +1,5 @@
+pub mod typ;
 
+pub mod code_lexer;
+
+pub mod comment_lexer;

src/mango/lexing/typ.rs

@@ -0,0 +1,15 @@
+use mango::io::typ::Reader;
+use mango::token::Tokens;
+
+pub enum MaybeToken {
+    Token(Tokens),
+    End(),
+}
+
+pub trait Lexer<'r> {
+    //    /// Create a new lexer from a reader instance.
+    //    fn new(reader: &'r mut Reader) -> Self;
+
+    /// Every call to lex returns a token until the end of the input.
+    fn lex(&mut self) -> MaybeToken;
+}

src/mango/token/mod.rs

@@ -1,10 +1,10 @@
-mod tokens;
+pub mod tokens;
 pub use self::tokens::*;
 
-mod special;
+pub mod special;
 pub use self::special::*;
 
-mod collect;
+pub mod collect;
 pub use self::collect::Token;
 pub use self::collect::Tokens;
 

src/mango/token/special/mod.rs

@@ -1,2 +1,2 @@
-mod unlexable;
+pub mod unlexable;
 pub use self::unlexable::UnlexableToken;