From 1d0c8d5838749a7f14a4aeb633b5ef89df85c7a3 Mon Sep 17 00:00:00 2001 From: Mark Date: Tue, 19 Jun 2018 22:49:03 +0200 Subject: [PATCH] Fix various problems incl stack/queue bug #52 --- src/mango/io/util.rs | 2 +- src/mango/lexing/code_lexer.rs | 31 +++++++++--------- src/mango/lexing/combi_lexer.rs | 6 +++- src/mango/util/codeparts/operator.rs | 47 +++++++++++++++++++--------- src/mango/util/collection/queue.rs | 31 ++++++++++++++---- src/mango/util/collection/stack.rs | 15 +++++++++ 6 files changed, 93 insertions(+), 39 deletions(-) diff --git a/src/mango/io/util.rs b/src/mango/io/util.rs index a8cbb96d..258d0e8c 100644 --- a/src/mango/io/util.rs +++ b/src/mango/io/util.rs @@ -16,7 +16,7 @@ impl RegexCache { pub fn make_or_get(&mut self, subpattern: &str) -> &Regex { if !self.cache.contains_key(subpattern) { - match Regex::new(&format!("^ *{}", subpattern)) { + match Regex::new(&format!(r"^ *{}", subpattern)) { Err(err) => panic!(format!( "Invalid regular expression '{}' while adding to library; this is a bug:\n{:?}", subpattern, err diff --git a/src/mango/lexing/code_lexer.rs b/src/mango/lexing/code_lexer.rs index 64a90955..0dfbda60 100644 --- a/src/mango/lexing/code_lexer.rs +++ b/src/mango/lexing/code_lexer.rs @@ -23,17 +23,13 @@ use mango::util::strslice::charsliceto; pub struct CodeLexer { indent: i32, - buffer: Queue, } // TODO: keep the regexes in thread local global scope storage impl CodeLexer { pub fn new() -> Self { - CodeLexer { - indent: 0, - buffer: Queue::new(), - } + CodeLexer { indent: 0 } } fn lex_indents(&mut self, reader: &mut Box) -> Vec { @@ -43,7 +39,7 @@ impl CodeLexer { } let mut tokens: Vec = Vec::with_capacity(8); if line_indent < self.indent { - if let Match(_) = reader.matches(r"end\s") { + if let Match(_) = reader.matches(r"end") { // If this is followed by an 'end' keyword, then that 'end' is redundant. tokens.push(Tokens::EndBlock(EndBlockToken::new(true, true))); } else { @@ -53,11 +49,12 @@ impl CodeLexer { // This line is dedented, make end tokens. tokens.push(Tokens::EndBlock(EndBlockToken::new(true, false))); } - } - for _ in self.indent..line_indent { - // This line is indented, make start tokens. - // TODO: increasing indent by more than one should be a warning - self.buffer.push(Tokens::StartBlock(StartBlockToken::new())); + } else { + for _ in self.indent..line_indent { + // This line is indented, make start tokens. + // TODO: increasing indent by more than one should be a warning + tokens.push(Tokens::StartBlock(StartBlockToken::new())); + } } self.indent = line_indent; tokens @@ -144,6 +141,12 @@ impl SubLexer for CodeLexer { return SubLexerResult::single(Tokens::Literal(LiteralToken::Real(value))); } + // Operator (before association) + if let Match(token) = reader.matches(OperatorToken::subpattern()) { + return SubLexerResult::single(Tokens::Operator( + OperatorToken::from_str(&token).unwrap(), + )); + } // Association (before operator) if let Match(token) = reader.matches(&AssociationToken::subpattern()) { debug_assert!(token.chars().last().unwrap() == '='); @@ -160,12 +163,6 @@ impl SubLexer for CodeLexer { ); } } - // Operator - if let Match(token) = reader.matches(OperatorToken::subpattern()) { - return SubLexerResult::single(Tokens::Operator( - OperatorToken::from_str(&token).unwrap(), - )); - } // Grouping symbols if let Match(_) = reader.matches(r"\(") { return SubLexerResult::single(Tokens::ParenthesisOpen(ParenthesisOpenToken::new())); diff --git a/src/mango/lexing/combi_lexer.rs b/src/mango/lexing/combi_lexer.rs index a31a02db..34b6cbe9 100644 --- a/src/mango/lexing/combi_lexer.rs +++ b/src/mango/lexing/combi_lexer.rs @@ -40,6 +40,10 @@ impl Lexer for CombiLexer { match lexer.lex_pass(&mut self.reader) { SubLexerResult::Result(tokens) => { if tokens.len() > 0 { + if tokens.len() > 1 { + // TODO + println!(">> GOING TO ADD: {:?}", tokens); + } // The sublexer produced tokens, queue them. self.buffer.append(tokens); self.lex() // TODO: if every branch does this, move it down @@ -93,7 +97,7 @@ mod tests { assert_eq!( expected, actual, - "expected: {}\nactual: {}", + "\nexpected:\n{}\nactual:\n{}", expected.to_text(), actual.to_text(), ); diff --git a/src/mango/util/codeparts/operator.rs b/src/mango/util/codeparts/operator.rs index 31dfb043..b001bfda 100644 --- a/src/mango/util/codeparts/operator.rs +++ b/src/mango/util/codeparts/operator.rs @@ -11,21 +11,32 @@ pub enum Symbol { Dash, Asterisk, Slash, + LT, + GT, + Eq, + LE, + GE, + Exclamation, + Question, } impl Symbol { pub fn new>(symbol_txt: S) -> Result { + use self::Symbol::*; let ssymbol_txt = symbol_txt.into(); match &*ssymbol_txt { - "+" => Ok(Symbol::Plus), - "-" => Ok(Symbol::Dash), - "*" => Ok(Symbol::Asterisk), - "/" => Ok(Symbol::Slash), - "<" => Ok(Symbol::Slash), - ">" => Ok(Symbol::Slash), - "==" => Ok(Symbol::Slash), - ">=" => Ok(Symbol::Slash), - "<=" => Ok(Symbol::Slash), + "+" => Ok(Plus), + "-" => Ok(Dash), + "*" => Ok(Asterisk), + "/" => Ok(Slash), + // TODO: how do I know < is an operator, rather than e.g. a generic? + "<" => Ok(LT), + ">" => Ok(GT), + "==" => Ok(Eq), + "<=" => Ok(LE), + ">=" => Ok(GE), + "!" => Ok(Exclamation), + "?" => Ok(Question), _ => Err(Msg::from_valid(&format!( "Unknown symbol: '{}'", ssymbol_txt @@ -35,20 +46,28 @@ impl Symbol { /// Generate an eager subpattern to match tokens, that can be composed in a regular expression. pub fn subpattern() -> &'static str { - r"[\-+*/]" + r"(?:\+|-|\*|/|<=|>=|==|>|<)" } } impl Display for Symbol { fn fmt(&self, f: &mut Formatter) -> fResult { + use self::Symbol::*; write!( f, "{}", match *self { - Symbol::Plus => "+", - Symbol::Dash => "-", - Symbol::Asterisk => "*", - Symbol::Slash => "/", + Plus => "+", + Dash => "-", + Asterisk => "*", + Slash => "/", + LT => "<", + GT => ">", + Eq => "==", + LE => "<=", + GE => ">=", + Exclamation => "!", + Question => "?", } ) } diff --git a/src/mango/util/collection/queue.rs b/src/mango/util/collection/queue.rs index bd239bef..3f18fad4 100644 --- a/src/mango/util/collection/queue.rs +++ b/src/mango/util/collection/queue.rs @@ -1,26 +1,45 @@ +use std::collections::VecDeque; + /// A one-ended queue. See also [Stack]. /// This is just a wrapper around vec so nobody pushes or pops the wrong end. pub struct Queue { - items: Vec, + items: VecDeque, } impl Queue { pub fn new() -> Self { Queue { - items: Vec::with_capacity(16), + items: VecDeque::with_capacity(16), } } pub fn push(&mut self, value: T) { - self.items.push(value) + self.items.push_back(value) } pub fn pop(&mut self) -> Option { - self.items.pop() + self.items.pop_front() } /// Moves all the elements from a vector into the queue. - pub fn append(&mut self, mut other: Vec) { - self.items.append(&mut other); + pub fn append(&mut self, other: Vec) { + for item in other.into_iter() { + self.items.push_back(item); + } + } +} + +#[cfg(test)] +mod tests { + use super::Queue; + + #[test] + fn test_queue() { + let mut queue: Queue = Queue::new(); + queue.push(1); + queue.push(2); + assert_eq!(1, queue.pop().unwrap()); + assert_eq!(2, queue.pop().unwrap()); + assert!(queue.pop().is_none()); } } diff --git a/src/mango/util/collection/stack.rs b/src/mango/util/collection/stack.rs index 055c0a18..942e43cd 100644 --- a/src/mango/util/collection/stack.rs +++ b/src/mango/util/collection/stack.rs @@ -25,3 +25,18 @@ impl Stack { self.items.back_mut() } } + +#[cfg(test)] +mod tests { + use super::Stack; + + #[test] + fn test_stack() { + let mut stack: Stack = Stack::new(); + stack.push(1); + stack.push(2); + assert_eq!(2, stack.pop().unwrap()); + assert_eq!(1, stack.pop().unwrap()); + assert!(stack.pop().is_none()); + } +}