Fix various problems incl stack/queue bug #52

src/mango/io/util.rs

@@ -16,7 +16,7 @@ impl RegexCache {
 
     pub fn make_or_get(&mut self, subpattern: &str) -> &Regex {
         if !self.cache.contains_key(subpattern) {
-            match Regex::new(&format!("^ *{}", subpattern)) {
+            match Regex::new(&format!(r"^ *{}", subpattern)) {
                 Err(err) => panic!(format!(
                     "Invalid regular expression '{}' while adding to library; this is a bug:\n{:?}",
                     subpattern, err

src/mango/lexing/code_lexer.rs

@@ -23,17 +23,13 @@ use mango::util::strslice::charsliceto;
 
 pub struct CodeLexer {
     indent: i32,
-    buffer: Queue<Tokens>,
 }
 
 // TODO: keep the regexes in thread local global scope storage
 
 impl CodeLexer {
     pub fn new() -> Self {
-        CodeLexer {
-            indent: 0,
-            buffer: Queue::new(),
-        }
+        CodeLexer { indent: 0 }
     }
 
     fn lex_indents(&mut self, reader: &mut Box<Reader>) -> Vec<Tokens> {
@@ -43,7 +39,7 @@ impl CodeLexer {
         }
         let mut tokens: Vec<Tokens> = Vec::with_capacity(8);
         if line_indent < self.indent {
-            if let Match(_) = reader.matches(r"end\s") {
+            if let Match(_) = reader.matches(r"end") {
                 // If this is followed by an 'end' keyword, then that 'end' is redundant.
                 tokens.push(Tokens::EndBlock(EndBlockToken::new(true, true)));
             } else {
@@ -53,11 +49,12 @@ impl CodeLexer {
                 // This line is dedented, make end tokens.
                 tokens.push(Tokens::EndBlock(EndBlockToken::new(true, false)));
             }
-        }
-        for _ in self.indent..line_indent {
-            // This line is indented, make start tokens.
-            // TODO: increasing indent by more than one should be a warning
-            self.buffer.push(Tokens::StartBlock(StartBlockToken::new()));
+        } else {
+            for _ in self.indent..line_indent {
+                // This line is indented, make start tokens.
+                // TODO: increasing indent by more than one should be a warning
+                tokens.push(Tokens::StartBlock(StartBlockToken::new()));
+            }
         }
         self.indent = line_indent;
         tokens
@@ -144,6 +141,12 @@ impl SubLexer for CodeLexer {
             return SubLexerResult::single(Tokens::Literal(LiteralToken::Real(value)));
         }
 
+        // Operator (before association)
+        if let Match(token) = reader.matches(OperatorToken::subpattern()) {
+            return SubLexerResult::single(Tokens::Operator(
+                OperatorToken::from_str(&token).unwrap(),
+            ));
+        }
         // Association (before operator)
         if let Match(token) = reader.matches(&AssociationToken::subpattern()) {
             debug_assert!(token.chars().last().unwrap() == '=');
@@ -160,12 +163,6 @@ impl SubLexer for CodeLexer {
                 );
             }
         }
-        // Operator
-        if let Match(token) = reader.matches(OperatorToken::subpattern()) {
-            return SubLexerResult::single(Tokens::Operator(
-                OperatorToken::from_str(&token).unwrap(),
-            ));
-        }
         // Grouping symbols
         if let Match(_) = reader.matches(r"\(") {
             return SubLexerResult::single(Tokens::ParenthesisOpen(ParenthesisOpenToken::new()));

src/mango/lexing/combi_lexer.rs

@@ -40,6 +40,10 @@ impl Lexer for CombiLexer {
                 match lexer.lex_pass(&mut self.reader) {
                     SubLexerResult::Result(tokens) => {
                         if tokens.len() > 0 {
+                            if tokens.len() > 1 {
+                                // TODO
+                                println!(">> GOING TO ADD: {:?}", tokens);
+                            }
                             // The sublexer produced tokens, queue them.
                             self.buffer.append(tokens);
                             self.lex() // TODO: if every branch does this, move it down
@@ -93,7 +97,7 @@ mod tests {
         assert_eq!(
             expected,
             actual,
-            "expected: {}\nactual:   {}",
+            "\nexpected:\n{}\nactual:\n{}",
             expected.to_text(),
             actual.to_text(),
         );

src/mango/util/codeparts/operator.rs

@@ -11,21 +11,32 @@ pub enum Symbol {
     Dash,
     Asterisk,
     Slash,
+    LT,
+    GT,
+    Eq,
+    LE,
+    GE,
+    Exclamation,
+    Question,
 }
 
 impl Symbol {
     pub fn new<S: Into<String>>(symbol_txt: S) -> Result<Self, Msg> {
+        use self::Symbol::*;
         let ssymbol_txt = symbol_txt.into();
         match &*ssymbol_txt {
-            "+" => Ok(Symbol::Plus),
-            "-" => Ok(Symbol::Dash),
-            "*" => Ok(Symbol::Asterisk),
-            "/" => Ok(Symbol::Slash),
-            "<" => Ok(Symbol::Slash),
-            ">" => Ok(Symbol::Slash),
-            "==" => Ok(Symbol::Slash),
-            ">=" => Ok(Symbol::Slash),
-            "<=" => Ok(Symbol::Slash),
+            "+" => Ok(Plus),
+            "-" => Ok(Dash),
+            "*" => Ok(Asterisk),
+            "/" => Ok(Slash),
+            // TODO: how do I know < is an operator, rather than e.g. a generic?
+            "<" => Ok(LT),
+            ">" => Ok(GT),
+            "==" => Ok(Eq),
+            "<=" => Ok(LE),
+            ">=" => Ok(GE),
+            "!" => Ok(Exclamation),
+            "?" => Ok(Question),
             _ => Err(Msg::from_valid(&format!(
                 "Unknown symbol: '{}'",
                 ssymbol_txt
@@ -35,20 +46,28 @@ impl Symbol {
 
     /// Generate an eager subpattern to match tokens, that can be composed in a regular expression.
     pub fn subpattern() -> &'static str {
-        r"[\-+*/]"
+        r"(?:\+|-|\*|/|<=|>=|==|>|<)"
     }
 }
 
 impl Display for Symbol {
     fn fmt(&self, f: &mut Formatter) -> fResult {
+        use self::Symbol::*;
         write!(
             f,
             "{}",
             match *self {
-                Symbol::Plus => "+",
-                Symbol::Dash => "-",
-                Symbol::Asterisk => "*",
-                Symbol::Slash => "/",
+                Plus => "+",
+                Dash => "-",
+                Asterisk => "*",
+                Slash => "/",
+                LT => "<",
+                GT => ">",
+                Eq => "==",
+                LE => "<=",
+                GE => ">=",
+                Exclamation => "!",
+                Question => "?",
             }
         )
     }

src/mango/util/collection/queue.rs

@@ -1,26 +1,45 @@
+use std::collections::VecDeque;
+
 /// A one-ended queue. See also [Stack].
 /// This is just a wrapper around vec so nobody pushes or pops the wrong end.
 pub struct Queue<T> {
-    items: Vec<T>,
+    items: VecDeque<T>,
 }
 
 impl<T> Queue<T> {
     pub fn new() -> Self {
         Queue {
-            items: Vec::with_capacity(16),
+            items: VecDeque::with_capacity(16),
         }
     }
 
     pub fn push(&mut self, value: T) {
-        self.items.push(value)
+        self.items.push_back(value)
     }
 
     pub fn pop(&mut self) -> Option<T> {
-        self.items.pop()
+        self.items.pop_front()
     }
 
     /// Moves all the elements from a vector into the queue.
-    pub fn append(&mut self, mut other: Vec<T>) {
-        self.items.append(&mut other);
+    pub fn append(&mut self, other: Vec<T>) {
+        for item in other.into_iter() {
+            self.items.push_back(item);
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::Queue;
+
+    #[test]
+    fn test_queue() {
+        let mut queue: Queue<i32> = Queue::new();
+        queue.push(1);
+        queue.push(2);
+        assert_eq!(1, queue.pop().unwrap());
+        assert_eq!(2, queue.pop().unwrap());
+        assert!(queue.pop().is_none());
     }
 }

src/mango/util/collection/stack.rs

@@ -25,3 +25,18 @@ impl<T> Stack<T> {
         self.items.back_mut()
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::Stack;
+
+    #[test]
+    fn test_stack() {
+        let mut stack: Stack<i32> = Stack::new();
+        stack.push(1);
+        stack.push(2);
+        assert_eq!(2, stack.pop().unwrap());
+        assert_eq!(1, stack.pop().unwrap());
+        assert!(stack.pop().is_none());
+    }
+}