Lexer has an infinite loop somewhere #52

mangolang · Jun 17, 2018 · ba3acda · ba3acda
1 parent 946f417
commit ba3acda
Show file tree

Hide file tree

Showing 4 changed files with 47 additions and 36 deletions.
diff --git a/src/mango/io/util.rs b/src/mango/io/util.rs
@@ -19,8 +19,7 @@ impl RegexCache {
             match Regex::new(&format!("^ *{}", subpattern)) {
                 Err(err) => panic!(format!(
                     "Invalid regular expression '{}' while adding to library; this is a bug:\n{:?}",
-                    subpattern,
-                    err
+                    subpattern, err
                 )),
                 Ok(regex) => {
                     self.cache.insert(subpattern.to_owned(), regex);

diff --git a/src/mango/lexing/code_lexer.rs b/src/mango/lexing/code_lexer.rs
@@ -147,40 +147,45 @@ impl SubLexer for CodeLexer {
             debug_assert!(token.chars().last().unwrap() == '=');
             if token.chars().count() > 1 {
                 panic!(); // TODO
-                return SubLexerResult::single((Tokens::Association(AssociationToken::from_unprefixed())));
+                return SubLexerResult::single(
+                    (Tokens::Association(AssociationToken::from_unprefixed())),
+                );
             } else {
-                return SubLexerResult::single((Tokens::Association(AssociationToken::from_unprefixed())));
+                return SubLexerResult::single(
+                    (Tokens::Association(AssociationToken::from_unprefixed())),
+                );
             }
         }
-        //        // Operator
-        //        let operator_match_res = self
-        //            .reader
-        //            .borrow_mut()
-        //            .matches(OperatorToken::subpattern());
-        //        if let Match(token) = operator_match_res {
-        //            return Token(Tokens::Operator(OperatorToken::from_str(&token).unwrap()));
-        //        }
-        //        // Grouping symbols
-        //        if let Match(_) = reader.matches(r"\(") {
-        //            return Token(Tokens::ParenthesisOpen(ParenthesisOpenToken::new()));
-        //        }
-        //        if let Match(_) = reader.matches(r"\)") {
-        //            return Token(Tokens::ParenthesisClose(ParenthesisCloseToken::new()));
-        //        }
-        //
-        //        let unknown_word = reader.matches("[^\\s]+");
-        //        match unknown_word {
-        //            Match(word) => return Token(Tokens::Unlexable(UnlexableToken::new(word))),
-        //            NoMatch() => {
-        //                println!("END {:?}", self.reader.borrow()); // TODO
-        //                panic!("Do not know how to proceed with parsing")
-        //            }
-        //            EOF() => {
-        //                // TODO: also dedent and end statement here
-        //                End
-        //            }
-        //        }
+        // Operator
+        if let Match(token) = reader.matches(OperatorToken::subpattern()) {
+            return SubLexerResult::single(Tokens::Operator(
+                OperatorToken::from_str(&token).unwrap(),
+            ));
+        }
+        // Grouping symbols
+        if let Match(_) = reader.matches(r"\(") {
+            return SubLexerResult::single(Tokens::ParenthesisOpen(ParenthesisOpenToken::new()));
+        }
+        if let Match(_) = reader.matches(r"\)") {
+            return SubLexerResult::single(Tokens::ParenthesisClose(ParenthesisCloseToken::new()));
+        }
 
-        panic!() // TODO TMP
+        // If the code gets here, it did not recognize the text as any token
+        return match reader.matches(r"[^\s]+") {
+            Match(word) => SubLexerResult::single(Tokens::Unlexable(UnlexableToken::new(word))),
+            NoMatch() => {
+                println!("END {:?}", reader); // todo: tmp
+                panic!("Do not know how to proceed with parsing")
+            }
+            EOF() => {
+                // TODO: also dedent and end statement here
+                let mut tokens = vec![Tokens::EndStatement(EndStatementToken::new_end_line())];
+                for _ in 0..self.indent {
+                    // This line is dedented, make end tokens.
+                    tokens.push(Tokens::EndBlock(EndBlockToken::new(true, false)));
+                }
+                SubLexerResult::Result(tokens)
+            }
+        };
     }
 }
diff --git a/src/mango/util/parsetxt/real.rs b/src/mango/util/parsetxt/real.rs
@@ -29,7 +29,11 @@ pub fn parse_real<S: Into<String>>(text: S) -> Result<f64, RealParseFailReason>
     {
         None => return Err(RealParseFailReason::Invalid),
         Some(captures) => {
-            let multiplier = captures.name("multiplier").unwrap().as_str().without_char(&'_')
+            let multiplier = captures
+                .name("multiplier")
+                .unwrap()
+                .as_str()
+                .without_char(&'_')
                 .parse::<f64>()
                 .unwrap();
             match captures.name("exponent") {
@@ -39,7 +43,9 @@ pub fn parse_real<S: Into<String>>(text: S) -> Result<f64, RealParseFailReason>
                 }
                 Some(exponent_match) => {
                     // This real is in exponential notation
-                    let exponent = exponent_match.as_str().without_char(&'_')
+                    let exponent = exponent_match
+                        .as_str()
+                        .without_char(&'_')
                         .parse::<f64>()
                         .unwrap();
                     // TODO: is there a numerically smarter way to do this?

diff --git a/src/mango/util/strslice/char_ops.rs b/src/mango/util/strslice/char_ops.rs
@@ -1,4 +1,3 @@
-
 pub trait CharOps {
     /// Remove all matching characters from the string.
     // Signature may be changed to support a set of characters, if the need arises.
@@ -19,10 +18,12 @@ impl<'a> CharOps for &'a str {
 
 impl CharOps for String {
     fn without_char(&self, strip: &char) -> String {
+        println!("String.without_char");
         (&self).without_char(strip)
     }
 
     fn char_len(&self) -> usize {
+        println!("String.char_len");
         (&self).char_len()
     }
 }