Resolved technical errors, only lexing problems remain #52

src/mango/lexing/code_lexer.rs

@@ -18,6 +18,8 @@ use mango::token::tokens::ParenthesisOpenToken;
 use mango::token::tokens::StartBlockToken;
 use mango::token::Tokens;
 use mango::util::collection::Queue;
+use mango::util::strslice::char_ops::CharOps;
+use mango::util::strslice::charsliceto;
 
 pub struct CodeLexer {
     indent: i32,
@@ -145,11 +147,13 @@ impl SubLexer for CodeLexer {
         // Association (before operator)
         if let Match(token) = reader.matches(&AssociationToken::subpattern()) {
             debug_assert!(token.chars().last().unwrap() == '=');
-            if token.chars().count() > 1 {
-                panic!(); // TODO
-                return SubLexerResult::single(
-                    (Tokens::Association(AssociationToken::from_unprefixed())),
-                );
+            if token.char_len() > 1 {
+                match AssociationToken::from_str(charsliceto(token, -1)) {
+                    Ok(association) => {
+                        return SubLexerResult::single((Tokens::Association(association)))
+                    }
+                    Err(msg) => panic!(format!("Invalid association prefix: {}", msg)),
+                }
             } else {
                 return SubLexerResult::single(
                     (Tokens::Association(AssociationToken::from_unprefixed())),
@@ -173,10 +177,7 @@ impl SubLexer for CodeLexer {
         // If the code gets here, it did not recognize the text as any token
         return match reader.matches(r"[^\s]+") {
             Match(word) => SubLexerResult::single(Tokens::Unlexable(UnlexableToken::new(word))),
-            NoMatch() => {
-                println!("END {:?}", reader); // todo: tmp
-                panic!("Do not know how to proceed with parsing")
-            }
+            NoMatch() => panic!("Do not know how to proceed with parsing"),
             EOF() => {
                 if self.indent <= 0 {
                     return SubLexerResult::End;

src/mango/lexing/combi_lexer.rs

@@ -81,16 +81,22 @@ mod tests {
     use mango::token::tokens::ParenthesisOpenToken;
     use mango::token::tokens::StartBlockToken;
     use mango::token::Tokens;
+    use mango::util::encdec::to_text::ToText;
     use std::cell::RefCell;
     use std::rc::Rc;
 
     fn assert_text_to_tokens(text: &str, tokens: Vec<Tokens>) {
+        let expected = LexList::from_tokens(tokens);
+        let actual = lex_all(&mut CombiLexer::new(Box::new(StringReader::new(
+            text.to_owned(),
+        ))));
         assert_eq!(
-            LexList::from_tokens(tokens),
-            lex_all(&mut CombiLexer::new(Box::new(StringReader::new(
-                text.to_owned()
-            ))))
-        )
+            expected,
+            actual,
+            "expected: {}\nactual:   {}",
+            expected.to_text(),
+            actual.to_text(),
+        );
     }
 
     #[test]
@@ -115,6 +121,7 @@ mod tests {
                 Tokens::Literal(LiteralToken::Int(0)),
                 Tokens::EndStatement(EndStatementToken::new_end_line()),
                 Tokens::Keyword(KeywordToken::from_str("for".to_owned()).unwrap()),
+                Tokens::Identifier(IdentifierToken::from_str("x".to_owned()).unwrap()),
                 Tokens::Operator(OperatorToken::from_str("<").unwrap()),
                 Tokens::Literal(LiteralToken::Int(128)),
                 Tokens::EndStatement(EndStatementToken::new_end_line()),

src/mango/lexing/util/lex_all.rs

@@ -1,6 +1,7 @@
 use mango::lexing::typ::Lexer;
 use mango::lexing::typ::MaybeToken;
 use mango::token::Tokens;
+use mango::util::encdec::ToText;
 
 /// Represents all the lex tokens in a source.
 #[derive(PartialEq, Eq, Debug)]
@@ -18,6 +19,16 @@ impl LexList {
     }
 }
 
+impl ToText for LexList {
+    fn to_text(&self) -> String {
+        self.tokens
+            .iter()
+            .map(|token| token.to_text())
+            .collect::<Vec<_>>()
+            .join(" ")
+    }
+}
+
 pub fn lex_all(lexer: &mut Lexer) -> LexList {
     let mut list = Vec::with_capacity(512);
     while let MaybeToken::Token(token) = lexer.lex() {

src/mango/token/collect/all.rs

@@ -55,6 +55,6 @@ mod tests {
 
     #[test]
     fn test_tokens_size() {
-        assert!(size_of::<Tokens>() < 32, size_of::<Tokens>());
+        assert!(size_of::<Tokens>() <= 40, size_of::<Tokens>());
     }
 }

src/mango/token/tokens/association.rs

@@ -28,7 +28,7 @@ impl AssociationToken {
     }
 
     pub fn subpattern() -> String {
-        format!("{}=", Symbol::subpattern())
+        format!(r"(?:{})?=", Symbol::subpattern())
     }
 }
 

src/mango/util/codeparts/operator.rs

@@ -35,7 +35,7 @@ impl Symbol {
 
     /// Generate an eager subpattern to match tokens, that can be composed in a regular expression.
     pub fn subpattern() -> &'static str {
-        r"(\+|\-|\*|/)"
+        r"[\-+*/]"
     }
 }