Skip to content

Commit

Permalink
fix token lexer unittest
Browse files Browse the repository at this point in the history
  • Loading branch information
@limuy2022
limuy2022 committed Feb 5, 2024
1 parent 0b21bef commit d0e64d2
Show file tree
Hide file tree
Showing 5 changed files with 73 additions and 56 deletions.
2 changes: 0 additions & 2 deletions rust/derive/tests/import_module_test.rs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
use derive::{def_module, trc_class, trc_function, trc_method};

#[test]
/// just a place for developer,not real test
fn test_output() {}
8 changes: 0 additions & 8 deletions rust/src/compiler.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,10 +136,6 @@ pub struct ValuePool {

const INT_VAL_POOL_ZERO: usize = 0;
const INT_VAL_POOL_ONE: usize = 1;
const INT_ID_POS: usize = 0;
const FLOAT_ID_POS: usize = 1;
const BOOL_ID_POS: usize = 2;
const STR_ID_POS: usize = 3;

macro_rules! gen_add_funcs {
($($func_name:ident => ($const_pool:ident, $id_pool:ident, $type:ty)),*) => {
Expand Down Expand Up @@ -171,10 +167,6 @@ impl ValuePool {
};
ret.add_int(0);
ret.add_int(1);
ret.add_id("int".to_string());
ret.add_id("float".to_string());
ret.add_id("bool".to_string());
ret.add_id("str".to_string());
ret
}

Expand Down
61 changes: 54 additions & 7 deletions rust/src/compiler/ast.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use crate::base::codegen::{Inst, Opcode, NO_ARG};
use crate::base::stdlib::{RustFunction, STDLIB_ROOT};
use crate::base::{codegen::StaticData, error::*};
use gettextrs::gettext;
use std::borrow::Borrow;
use std::cell::RefCell;
use std::rc::Rc;

Expand Down Expand Up @@ -85,16 +86,20 @@ macro_rules! ExprGen {

impl<'a> AstBuilder<'a> {
pub fn new(token_lexer: TokenLex<'a>) -> Self {
let prelude = STDLIB_ROOT.sub_modules.get("prelude").unwrap().clone();
for i in prelude.functions {
token_lexer.compiler_data.const_pool.add_id(i.0.clone());
}
for i in prelude.classes {
token_lexer.compiler_data.const_pool.add_id(i.0.clone());
}
let root_scope = Rc::new(RefCell::new(SymScope::new(None)));
// 为root scope添加prelude
let mut ret = AstBuilder {
token_lexer,
staticdata: StaticData::new(),
self_scope: root_scope,
};
for i in &STDLIB_ROOT.sub_modules.get("prelude").unwrap().functions {
ret.token_lexer.compiler_data.const_pool.add_id(i.0.clone());
}
ret.self_scope
.as_ref()
.borrow_mut()
Expand Down Expand Up @@ -197,7 +202,7 @@ impl<'a> AstBuilder<'a> {
let argv_list = self.opt_args()?;
// match )
self.check_next_token(TokenType::RightSmallBrace)?;
let tmp = self.self_scope.borrow();
let tmp = self.self_scope.as_ref().borrow();
let func_obj = tmp.get_function(idx).unwrap();
match func_obj.get_io().check_argvs(argv_list) {
Err(e) => {
Expand Down Expand Up @@ -247,19 +252,61 @@ impl<'a> AstBuilder<'a> {
self.staticdata
.inst
.push(Inst::new(Opcode::LoadInt, t.data.unwrap()));
return Ok(TypeAllowNull::Yes(INT_TYPE.clone()));
return Ok(TypeAllowNull::Yes(
self.self_scope
.as_ref()
.borrow()
.get_type(
*self
.token_lexer
.compiler_data
.const_pool
.name_pool
.get("int")
.unwrap(),
)
.clone(),
));
}
TokenType::FloatValue => {
self.staticdata
.inst
.push(Inst::new(Opcode::LoadFloat, t.data.unwrap()));
return Ok(TypeAllowNull::Yes(FLOAT_TYPE.clone()));
return Ok(TypeAllowNull::Yes(
self.self_scope
.as_ref()
.borrow()
.get_type(
*self
.token_lexer
.compiler_data
.const_pool
.name_pool
.get("float")
.unwrap(),
)
.clone(),
));
}
TokenType::StringValue => {
self.staticdata
.inst
.push(Inst::new(Opcode::LoadString, t.data.unwrap()));
return Ok(TypeAllowNull::Yes(STR_TYPE.clone()));
return Ok(TypeAllowNull::Yes(
self.self_scope
.as_ref()
.borrow()
.get_type(
*self
.token_lexer
.compiler_data
.const_pool
.name_pool
.get("str")
.unwrap(),
)
.clone(),
));
}
_ => {
self.token_lexer.next_back(t.clone());
Expand Down
9 changes: 1 addition & 8 deletions rust/src/compiler/scope.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use super::{ValuePool, BOOL_ID_POS, FLOAT_ID_POS, INT_ID_POS, STR_ID_POS};
use super::ValuePool;
use crate::base::stdlib::{ClassInterface, FunctionInterface, IOType, Stdlib, STDLIB_ROOT};
use lazy_static::lazy_static;
use std::{cell::RefCell, collections::HashMap, fmt::Display, rc::Rc};
Expand Down Expand Up @@ -143,13 +143,6 @@ impl ClassInterface for CommonType {
}
}

lazy_static! {
pub static ref INT_TYPE: Type = Box::new(CommonType::new(INT_ID_POS, "int"));
pub static ref FLOAT_TYPE: Type = Box::new(CommonType::new(FLOAT_ID_POS, "float"));
pub static ref STR_TYPE: Type = Box::new(CommonType::new(STR_ID_POS, "str"));
pub static ref BOOL_TYPE: Type = Box::new(CommonType::new(BOOL_ID_POS, "bool"));
}

pub struct SymScope {
prev_scope: Option<Rc<RefCell<SymScope>>>,
sym_map: HashMap<usize, usize>,
Expand Down
49 changes: 18 additions & 31 deletions rust/src/compiler/token.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -811,10 +811,7 @@ mod tests {
use std::{collections::HashSet, fmt::Debug, hash::Hash};

use super::*;
use crate::compiler::{
Float, InputSource, Option, Pool, INT_ID_POS, INT_VAL_POOL_ONE, INT_VAL_POOL_ZERO,
STR_ID_POS,
};
use crate::compiler::{Float, InputSource, Option, Pool, INT_VAL_POOL_ONE, INT_VAL_POOL_ZERO};

macro_rules! gen_test_token_env {
($test_string:expr, $env_name:ident) => {
Expand Down Expand Up @@ -997,16 +994,16 @@ func main() {
Token::new(TokenType::Import, None),
Token::new(TokenType::StringValue, Some(0)),
Token::new(TokenType::Func, None),
Token::new(TokenType::ID, Some(4)),
Token::new(TokenType::ID, Some(0)),
Token::new(TokenType::LeftSmallBrace, None),
Token::new(TokenType::ID, Some(INT_ID_POS)),
Token::new(TokenType::ID, Some(5)),
Token::new(TokenType::ID, Some(1)),
Token::new(TokenType::ID, Some(2)),
Token::new(TokenType::RightSmallBrace, None),
Token::new(TokenType::Arrow, None),
Token::new(TokenType::ID, Some(3)),
Token::new(TokenType::LeftBigBrace, None),
Token::new(TokenType::If, None),
Token::new(TokenType::ID, Some(5)),
Token::new(TokenType::ID, Some(2)),
Token::new(TokenType::Mod, None),
Token::new(TokenType::IntValue, Some(2)),
Token::new(TokenType::Equal, None),
Expand All @@ -1022,25 +1019,25 @@ func main() {
Token::new(TokenType::RightBigBrace, None),
Token::new(TokenType::RightBigBrace, None),
Token::new(TokenType::Func, None),
Token::new(TokenType::ID, Some(6)),
Token::new(TokenType::ID, Some(4)),
Token::new(TokenType::LeftSmallBrace, None),
Token::new(TokenType::RightSmallBrace, None),
Token::new(TokenType::LeftBigBrace, None),
Token::new(TokenType::ID, Some(7)),
Token::new(TokenType::ID, Some(5)),
Token::new(TokenType::LeftSmallBrace, None),
Token::new(TokenType::StringValue, Some(3)),
Token::new(TokenType::RightSmallBrace, None),
Token::new(TokenType::ID, Some(8)),
Token::new(TokenType::ID, Some(6)),
Token::new(TokenType::Store, None),
Token::new(TokenType::ID, Some(4)),
Token::new(TokenType::ID, Some(0)),
Token::new(TokenType::LeftSmallBrace, None),
Token::new(TokenType::ID, Some(9)),
Token::new(TokenType::ID, Some(7)),
Token::new(TokenType::LeftSmallBrace, None),
Token::new(TokenType::RightSmallBrace, None),
Token::new(TokenType::RightSmallBrace, None),
Token::new(TokenType::ID, Some(7)),
Token::new(TokenType::ID, Some(5)),
Token::new(TokenType::LeftSmallBrace, None),
Token::new(TokenType::ID, Some(8)),
Token::new(TokenType::ID, Some(6)),
Token::new(TokenType::RightSmallBrace, None),
Token::new(TokenType::RightBigBrace, None),
],
Expand All @@ -1053,11 +1050,11 @@ func main() {
check(
&mut t,
vec![
Token::new(TokenType::ID, Some(0)),
Token::new(TokenType::ID, Some(1)),
Token::new(TokenType::ID, Some(2)),
Token::new(TokenType::ID, Some(3)),
Token::new(TokenType::ID, Some(4)),
Token::new(TokenType::ID, Some(5)),
Token::new(TokenType::ID, Some(6)),
Token::new(TokenType::ID, Some(7)),
Token::new(TokenType::ID, Some(8)),
],
);
check_pool(
Expand All @@ -1067,10 +1064,6 @@ func main() {
String::from("_fuck"),
String::from("天帝abc"),
String::from("abc天帝"),
"int".to_string(),
"float".to_string(),
"bool".to_string(),
"str".to_string(),
],
&t.compiler_data.const_pool.name_pool,
);
Expand All @@ -1096,18 +1089,12 @@ func main() {
&mut t,
vec![
Token::new(TokenType::IntValue, Some(2)),
Token::new(TokenType::ID, Some(4)),
Token::new(TokenType::ID, Some(0)),
],
);
check_pool(vec![0xabc, 0, 1], &t.compiler_data.const_pool.const_ints);
check_pool(
vec![
"hds".to_string(),
"int".to_string(),
"float".to_string(),
"bool".to_string(),
"str".to_string(),
],
vec!["hds".to_string()],
&t.compiler_data.const_pool.name_pool,
);
}
Expand Down

0 comments on commit d0e64d2

Please sign in to comment.