From 94e2cdf25cbad5356ba4f6ffe8470276af3f48ba Mon Sep 17 00:00:00 2001 From: KPMGE Date: Sat, 23 Mar 2024 16:09:35 -0300 Subject: [PATCH 1/8] :sparkles: implement string tokenization --- src/lexer.rs | 25 ++++++++++++++++++++++++- src/token.rs | 5 +++-- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index 46fa24d..5ccdec1 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; + use std::collections::HashMap; use crate::token::Token; @@ -52,6 +52,7 @@ impl Lexer { ',' => Token::Comma, '/' => Token::Slash, ';' => Token::Semicolon, + '"' => Token::String(self.read_string()), '=' => match self.peek_char(self.read_position) { Some('=') => { self.read_char(); @@ -88,6 +89,28 @@ impl Lexer { token } + fn read_string(&mut self) -> String { + if self.current_char.unwrap() != '"' { + panic!("Unexpected start of string, expected: '\"', got: {:?}", self.current_char); + } + + let mut str = String::new(); + + self.read_char(); + + while let Some(c) = self.current_char { + if c == '"' { + break; + } + str.push(c); + self.read_char(); + } + + self.read_char(); + + str + } + fn read_number(&mut self) -> String { let start_pos = self.current_position; diff --git a/src/token.rs b/src/token.rs index e6beaa5..3b90e80 100644 --- a/src/token.rs +++ b/src/token.rs @@ -13,8 +13,6 @@ pub enum Token { Slash, GreaterThan, LessThan, - Int(String), - Identifier(String), LeftParentesis, Let, True, @@ -27,4 +25,7 @@ pub enum Token { RightParentesis, LeftBrace, RightBrace, + Int(String), + Identifier(String), + String(String), } From 17062acbabdb401d8d428820b67cfb3d9278fea5 Mon Sep 17 00:00:00 2001 From: KPMGE Date: Sat, 23 Mar 2024 16:11:12 -0300 Subject: [PATCH 2/8] :white_check_mark: ensure lexer works correctly with strings --- tests/lexer.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/lexer.rs b/tests/lexer.rs index 7759db6..cc3b1c8 100644 --- a/tests/lexer.rs +++ b/tests/lexer.rs @@ -76,3 +76,15 @@ fn given_code_with_integers_it_should_parse_correctly() { assert!(token == expected_token); } + +#[test] +fn given_code_with_a_strinig_it_should_parse_correctly() { + let code = "\"kevin\""; + + let mut lexer = Lexer::new(code.to_string()); + + let token = lexer.next_token(); + let expected_token = Token::String("kevin".to_string()); + + assert!(token == expected_token); +} From 1c5b0e5665c46a93a4cdd8997d1d03f92ed3c252 Mon Sep 17 00:00:00 2001 From: KPMGE Date: Sat, 23 Mar 2024 16:16:18 -0300 Subject: [PATCH 3/8] :sparkles: implement parsing for strings --- src/ast.rs | 1 + src/parser.rs | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/ast.rs b/src/ast.rs index 0fe7fa1..1b1de43 100644 --- a/src/ast.rs +++ b/src/ast.rs @@ -5,6 +5,7 @@ pub enum Expression { Int(i32), Identifier(String), Boolean(bool), + String(String), Prefix { operator: Token, // Token::Bang, Token::Minus right: Box, diff --git a/src/parser.rs b/src/parser.rs index c5964d9..e5edbb7 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -150,6 +150,13 @@ impl Parser { false } + fn parse_string(&mut self) -> Option { + if let Token::String(s) = &self.current_token { + return Some(Expression::String(s.clone())); + } + None + } + fn parse_identifier(&mut self) -> Option { if let Token::Identifier(name) = &self.current_token { return Some(Expression::Identifier(name.clone())); @@ -399,6 +406,7 @@ impl Token { fn prefix_parse_fn(&self) -> Option Option> { match self { + Token::String(_) => Some(Parser::parse_string), Token::Identifier(_) => Some(Parser::parse_identifier), Token::Int(_) => Some(Parser::parse_int), Token::Bang => Some(Parser::parse_prefix_expression), From 71a8a36a904589b4d9a07bb017e307de0a7f59f1 Mon Sep 17 00:00:00 2001 From: KPMGE Date: Sat, 23 Mar 2024 16:18:17 -0300 Subject: [PATCH 4/8] :white_check_mark: ensure parser parses strins correctly --- tests/parser.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/parser.rs b/tests/parser.rs index 2f5df9b..5fba7dd 100644 --- a/tests/parser.rs +++ b/tests/parser.rs @@ -434,6 +434,32 @@ fn given_a_call_expression_it_should_parse_correctly() { } } +#[test] +fn given_a_string_expression_it_should_parse_correctly() { + let code = "\"kevin\""; + let expected_expression = Expression::String("kevin".to_string()); + + let lexer = Lexer::new(code.to_string()); + let mut parser = Parser::new(lexer); + let parsed_program = parser.parse_program(); + + assert_eq!(parser.errors.len(), 0); + + match parsed_program { + AstNode::Program { statements } => { + assert_eq!(statements.len(), 1); + + let statement = statements.first().unwrap(); + + match statement { + AstNode::Expression(expression) => assert_eq!(*expression, expected_expression), + _ => panic!("Unexpected expression!"), + } + } + _ => panic!("Unexpected AstNode!"), + } +} + fn assert_boolean_expression(code: &str, expected_expression: &Expression) { let lexer = Lexer::new(code.to_string()); let mut parser = Parser::new(lexer); From 7710efc019e24d7ede2c895662f3c4ddded6ae6b Mon Sep 17 00:00:00 2001 From: KPMGE Date: Sat, 23 Mar 2024 16:18:42 -0300 Subject: [PATCH 5/8] :sparkles: add support for string evaluation --- src/evaluator.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/evaluator.rs b/src/evaluator.rs index 8c75a5c..e2051a0 100644 --- a/src/evaluator.rs +++ b/src/evaluator.rs @@ -11,6 +11,7 @@ pub struct Evaluator { pub enum Object { Integer(i32), Boolean(bool), + String(String), Return(Box), Function { parameters: Vec, // Token::Identifier @@ -39,6 +40,7 @@ impl Evaluator { match expression { Expression::Int(value) => Object::Integer(value), Expression::Boolean(value) => Object::Boolean(value), + Expression::String(value) => Object::String(value), Expression::Prefix { operator, right } => { let right = self.eval(AstNode::Expression(*right)); self.eval_prefix_expression(operator, right) @@ -234,6 +236,7 @@ impl Object { match self { Object::Integer(value) => format!("{value}"), Object::Boolean(value) => format!("{value}"), + Object::String(value) => format!("{value}"), Object::Return(value) => value.inspect(), Object::Function { .. } => "function".to_string(), Object::Null => "null".to_string(), From 7b8f3202d4145bf71d0f6f84065088f8d9154397 Mon Sep 17 00:00:00 2001 From: KPMGE Date: Sat, 23 Mar 2024 16:21:13 -0300 Subject: [PATCH 6/8] :white_check_mark: ensure evaluator evaluates strins correctly --- tests/evaluator.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/evaluator.rs b/tests/evaluator.rs index 005affd..5d059aa 100644 --- a/tests/evaluator.rs +++ b/tests/evaluator.rs @@ -131,3 +131,22 @@ fn given_return_statements_it_should_evaluate_correctly() { assert_eq!(evaluated_obj, *expected_objects.get(idx).unwrap()); }) } + +#[test] +fn given_a_string_expression_it_should_evaluate_correctly() { + let code = "\"kevin\""; + let expected_obj = Object::String("kevin".to_string()); + + let lexer = Lexer::new(code.to_string()); + let mut parser = Parser::new(lexer); + let parsed_program = parser.parse_program(); + let node = match parsed_program { + AstNode::Program { statements } => statements.first().unwrap().clone(), + _ => panic!("Unexpected AstNode!"), + }; + + let mut evaluator = Evaluator::new(); + let evaluated_obj = evaluator.eval(node); + + assert_eq!(evaluated_obj, expected_obj); +} From 3e84b13c2c3df176666de919412f8f7e0709ee5f Mon Sep 17 00:00:00 2001 From: KPMGE Date: Sat, 23 Mar 2024 16:22:21 -0300 Subject: [PATCH 7/8] :recycle: format code --- src/lexer.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/lexer.rs b/src/lexer.rs index 5ccdec1..884d585 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,4 +1,5 @@ - use std::collections::HashMap; + +use std::collections::HashMap; use crate::token::Token; @@ -91,7 +92,10 @@ impl Lexer { fn read_string(&mut self) -> String { if self.current_char.unwrap() != '"' { - panic!("Unexpected start of string, expected: '\"', got: {:?}", self.current_char); + panic!( + "Unexpected start of string, expected: '\"', got: {:?}", + self.current_char + ); } let mut str = String::new(); From 149af0c30a7a9d9d43bdade52e433e03be538446 Mon Sep 17 00:00:00 2001 From: KPMGE Date: Sat, 23 Mar 2024 16:22:29 -0300 Subject: [PATCH 8/8] :recycle: fix clippy --- src/evaluator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/evaluator.rs b/src/evaluator.rs index e2051a0..acaa8f0 100644 --- a/src/evaluator.rs +++ b/src/evaluator.rs @@ -236,7 +236,7 @@ impl Object { match self { Object::Integer(value) => format!("{value}"), Object::Boolean(value) => format!("{value}"), - Object::String(value) => format!("{value}"), + Object::String(value) => value.to_string(), Object::Return(value) => value.inspect(), Object::Function { .. } => "function".to_string(), Object::Null => "null".to_string(),