MWE of the lexer works! #52

mangolang · Jun 20, 2018 · f79c75b · f79c75b
1 parent 1d0c8d5
commit f79c75b
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 17 deletions.
diff --git a/src/mango/lexing/code_lexer.rs b/src/mango/lexing/code_lexer.rs
@@ -22,7 +22,7 @@ use mango::util::strslice::char_ops::CharOps;
 use mango::util::strslice::charsliceto;
 
 pub struct CodeLexer {
-    indent: i32,
+    indent: i32, // -1: finished
 }
 
 // TODO: keep the regexes in thread local global scope storage
@@ -141,12 +141,6 @@ impl SubLexer for CodeLexer {
             return SubLexerResult::single(Tokens::Literal(LiteralToken::Real(value)));
         }
 
-        // Operator (before association)
-        if let Match(token) = reader.matches(OperatorToken::subpattern()) {
-            return SubLexerResult::single(Tokens::Operator(
-                OperatorToken::from_str(&token).unwrap(),
-            ));
-        }
         // Association (before operator)
         if let Match(token) = reader.matches(&AssociationToken::subpattern()) {
             debug_assert!(token.chars().last().unwrap() == '=');
@@ -163,6 +157,12 @@ impl SubLexer for CodeLexer {
                 );
             }
         }
+        // Operator (after association)
+        if let Match(token) = reader.matches(OperatorToken::subpattern()) {
+            return SubLexerResult::single(Tokens::Operator(
+                OperatorToken::from_str(&token).unwrap(),
+            ));
+        }
         // Grouping symbols
         if let Match(_) = reader.matches(r"\(") {
             return SubLexerResult::single(Tokens::ParenthesisOpen(ParenthesisOpenToken::new()));
@@ -176,16 +176,15 @@ impl SubLexer for CodeLexer {
             Match(word) => SubLexerResult::single(Tokens::Unlexable(UnlexableToken::new(word))),
             NoMatch() => panic!("Do not know how to proceed with parsing"),
             EOF() => {
-                if self.indent <= 0 {
+                if self.indent < 0 {
                     return SubLexerResult::End;
                 }
-                // TODO: currently the EndStatement is only made if the file stops on an indented line
                 let mut tokens = vec![Tokens::EndStatement(EndStatementToken::new_end_line())];
                 for _ in 0..self.indent {
                     // This line is dedented, make end tokens.
                     tokens.push(Tokens::EndBlock(EndBlockToken::new(true, false)));
                 }
-                self.indent = 0;
+                self.indent = -1;
                 SubLexerResult::Result(tokens)
             }
         };

diff --git a/src/mango/lexing/combi_lexer.rs b/src/mango/lexing/combi_lexer.rs
@@ -40,10 +40,6 @@ impl Lexer for CombiLexer {
                 match lexer.lex_pass(&mut self.reader) {
                     SubLexerResult::Result(tokens) => {
                         if tokens.len() > 0 {
-                            if tokens.len() > 1 {
-                                // TODO
-                                println!(">> GOING TO ADD: {:?}", tokens);
-                            }
                             // The sublexer produced tokens, queue them.
                             self.buffer.append(tokens);
                             self.lex() // TODO: if every branch does this, move it down
@@ -107,9 +103,10 @@ mod tests {
     fn test_lexing_individual() {
         assert_text_to_tokens(
             "if",
-            vec![Tokens::Keyword(
-                KeywordToken::from_str("if".to_owned()).unwrap(),
-            )],
+            vec![
+                Tokens::Keyword(KeywordToken::from_str("if".to_owned()).unwrap()),
+                Tokens::EndStatement(EndStatementToken::new_end_line()),
+            ],
         );
         // todo: more
     }
@@ -133,6 +130,7 @@ mod tests {
                 Tokens::Identifier(IdentifierToken::from_str("x".to_owned()).unwrap()),
                 Tokens::Association(AssociationToken::from_str("+".to_owned()).unwrap()),
                 Tokens::Literal(LiteralToken::Int(1)),
+                Tokens::EndStatement(EndStatementToken::new_end_line()),
                 Tokens::EndBlock(EndBlockToken::new(true, false)),
             ],
         );