diff --git a/src/compiler/ast.rs b/src/compiler/ast.rs index c126b4e5..8d6523f4 100644 --- a/src/compiler/ast.rs +++ b/src/compiler/ast.rs @@ -779,7 +779,7 @@ impl<'a> ModuleUnit<'a> { } fn def_func(&mut self) -> AstError<()> { - let funcname = self.get_token_checked(Token::ID(0))?; + let funcname = self.get_token_checked_with_val(Token::ID(0))?; let name_id = self.insert_sym_with_error(funcname)?; // lex args self.get_token_checked(Token::LeftSmallBrace)?; @@ -793,7 +793,7 @@ impl<'a> ModuleUnit<'a> { if t != Token::Comma { self.token_lexer.borrow_mut().next_back(t); } - let name_id = self.get_token_checked(Token::ID(0))?; + let name_id = self.get_token_checked_with_val(Token::ID(0))?; argname.push(name_id); self.get_token_checked(Token::Colon)?; ty_list.push(self.lex_ty(false)?); @@ -851,7 +851,7 @@ impl<'a> ModuleUnit<'a> { match t { Token::Var => { // 声明属性 - let attr_name_tok = self.get_token_checked(Token::ID(0))?; + let attr_name_tok = self.get_token_checked_with_val(Token::ID(0))?; let attr_id = self.insert_sym_with_error(attr_name_tok)?; self.get_token_checked(Token::Colon)?; let ty = self.lex_ty(false)?; @@ -884,7 +884,7 @@ impl<'a> ModuleUnit<'a> { self.self_scope.clone(), )))); self.self_scope.borrow_mut().in_class = true; - let name = self.get_token_checked(Token::ID(0))?; + let name = self.get_token_checked_with_val(Token::ID(0))?; let name_id = self.insert_sym_with_error(name)?; let mut class_obj = CustomType::new( name_id, @@ -903,7 +903,7 @@ impl<'a> ModuleUnit<'a> { } fn lex_import(&mut self, istry: bool) -> AstError<()> { - let tok = self.get_token_checked(Token::StringValue(0))?; + let tok = self.get_token_checked_with_val(Token::StringValue(0))?; // import的路径 let mut path_with_dot = self.token_lexer.borrow_mut().get_constpool().id_str[tok].clone(); // 具体文件的路径 diff --git a/src/compiler/ast/ast_base.rs b/src/compiler/ast/ast_base.rs index 451db3a4..676dde2f 100644 --- a/src/compiler/ast/ast_base.rs +++ b/src/compiler/ast/ast_base.rs @@ -79,7 +79,7 @@ impl<'a> ModuleUnit<'a> { /// 解析一个类型名为id pub fn lex_ty(&mut self, istry: bool) -> AstError { - let t = self.get_token_checked(Token::ID(0))?; + let t = self.get_token_checked_with_val(Token::ID(0))?; let ty = match self.self_scope.borrow().get_type_id_by_token(t) { None => self.try_err( istry, @@ -109,11 +109,15 @@ impl<'a> ModuleUnit<'a> { )) } - /// 获取一个token并检查该token是否正确 - pub fn get_token_checked(&mut self, ty: Token) -> AstError { + /// 获取一个token并检查该token是否正确,仅仅检查带有值的token类型 + pub fn get_token_checked_with_val(&mut self, ty: Token) -> AstError { let t = self.token_lexer.borrow_mut().next_token()?; match (ty, t) { (Token::ID(_), Token::ID(data)) => Ok(data), + (Token::IntValue(_), Token::IntValue(data)) => Ok(data), + (Token::FloatValue(_), Token::FloatValue(data)) => Ok(data), + (Token::StringValue(_), Token::StringValue(data)) => Ok(data), + (Token::CharValue(_), Token::CharValue(data)) => Ok(data as usize), _ => { self.token_lexer.borrow_mut().next_back(t); self.gen_unexpected_token_error(t)? @@ -121,7 +125,8 @@ impl<'a> ModuleUnit<'a> { } } - pub fn check_next_token(&mut self, ty: Token) -> AstError<()> { + /// 获取一个token并检查该token是否正确,仅仅检查不带有值的token类型 + pub fn get_token_checked(&mut self, ty: Token) -> AstError<()> { let t = self.token_lexer.borrow_mut().next_token()?; if t == ty { Ok(()) diff --git a/src/lib.rs b/src/lib.rs index 9af2e98b..354758c0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -48,6 +48,9 @@ make_commands!(Commands, Build {}, Run {}, Dis { }, | Tshell {}, Version {}, New { #[arg()] project_name: String + }, Token { + #[arg()] + files: Vec }); shadow_rs::shadow!(build); @@ -117,6 +120,11 @@ pub fn run() -> Result<(), Box> { Commands::New { project_name } => { tools::new_project(&project_name)?; } + Commands::Token { files } => { + for i in files { + tools::token(&i); + } + } }; Ok(()) } diff --git a/src/tools.rs b/src/tools.rs index bff8f01f..0e0b905a 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -3,10 +3,12 @@ pub mod dis; pub mod new; pub mod run; pub mod style; +pub mod token; pub mod tshell; pub use build::build; pub use dis::dis; pub use new::new_project; pub use run::run; +pub use token::token; pub use tshell::tshell; diff --git a/src/tools/token.rs b/src/tools/token.rs new file mode 100644 index 00000000..1288c67d --- /dev/null +++ b/src/tools/token.rs @@ -0,0 +1,11 @@ +use crate::compiler::{token::Token, ValuePool}; +use logos::Logos; +use std::fs::read_to_string; + +pub fn token(file_path: &str) { + let s = read_to_string(file_path).unwrap(); + let lexer = Token::lexer_with_extras(&s, ValuePool::default()); + for token in lexer { + println!("{}", token.unwrap()); + } +}