Skip to content

Commit

Permalink
update tokenlex
Browse files Browse the repository at this point in the history 
update types operators
  • Loading branch information
@limuy2022
limuy2022 committed Jan 31, 2024
1 parent 95d9dae commit cd9136c
Show file tree
Hide file tree
Showing 7 changed files with 254 additions and 36 deletions.
46 changes: 46 additions & 0 deletions rust/docs/usage.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,26 @@ if 1 == 1 {
}
```

```for``` support two kinds

First:

```cpp
for i := 0; i < 10; i++ {
println(i)
}
```

Second:

```python
for i in xxx {
println(i)
}
```

```xxx``` should be a iterable var

## the comments of Trc

Trc support two kinds of comments
Expand All @@ -147,6 +167,14 @@ First,we support the UTF-8 with your var name.So you can define your var like th
你好:=90
```

the compiler will regard this var as a int var.

Sometimes maybe you want to define the type.Do it like this:

```go
int a:=90
```

## Data structures for Trc

Std lib provide many kinds of data structures for Trc.Here is the list:
Expand All @@ -160,3 +188,21 @@ Std lib provide many kinds of data structures for Trc.Here is the list:
|forward list|
|stack|
|deque|

## Function

Define a function like this:

```go
func add(int a, int b) -> int {
return a + b;
}
```

Or define a template function:

```go
func add<T>(T a, T b) -> T {
return a + b;
}
```
41 changes: 41 additions & 0 deletions rust/src/compiler/ast.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ pub struct AstBuilder<'a> {
self_scope: Rc<RefCell<SymScope>>,
}

#[derive(PartialEq, Debug)]
enum LexState {
Success,
Failure,
Expand Down Expand Up @@ -138,7 +139,43 @@ impl<'a> AstBuilder<'a> {
Ok(LexState::Success)
}

fn val(&mut self, istry: bool) -> AstError {
Ok(LexState::Success)
}

fn item(&mut self, istry: bool) -> AstError {
match self.val(istry) {
Err(e) => {
return Err(e);
}
Ok(v) => {
if v == LexState::Success {
return Ok(LexState::Success);
}
}
}
let t = self.token_lexer.next_token()?;
match t.tp {
TokenType::IntValue => {
self.staticdata
.inst
.push(Inst::new(Opcode::LoadInt, t.data.unwrap()));
}
TokenType::FloatValue => {
self.staticdata
.inst
.push(Inst::new(Opcode::LoadFloat, t.data.unwrap()));
}
TokenType::StringValue => {
self.staticdata
.inst
.push(Inst::new(Opcode::LoadString, t.data.unwrap()));
}
_ => {
panic!("unreachable!")
}
}

Ok(LexState::Success)
}

Expand All @@ -147,6 +184,9 @@ impl<'a> AstBuilder<'a> {
if next_token.tp == TokenType::LeftSmallBrace {
self.expr(istry)?;
self.check_next_token(TokenType::RightSmallBrace)?;
} else {
self.token_lexer.next_back(next_token);
self.item(istry)?;
}
Ok(LexState::Success)
}
Expand Down Expand Up @@ -239,6 +279,7 @@ impl<'a> AstBuilder<'a> {
}
_ => {}
}
self.token_lexer.next_back(t.clone());
match self.expr(true) {
Ok(_) => {}
Err(_) => {
Expand Down
154 changes: 129 additions & 25 deletions rust/src/compiler/token.rs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,8 @@
use super::{Compiler, Content, Float, INT_VAL_POOL_ZERO};
use super::{Compiler, Content, Float};
use crate::{
base::{
error::{
self, ErrorContent, ErrorInfo, RunResult, RuntimeError, FLOAT_OVER_FLOW,
NUMBER_OVER_FLOW, PREFIX_FOR_FLOAT, SYNTAX_ERROR,
},
utils::get_bit_num,
base::error::{
self, ErrorContent, ErrorInfo, RunResult, RuntimeError, FLOAT_OVER_FLOW, NUMBER_OVER_FLOW,
PREFIX_FOR_FLOAT, SYNTAX_ERROR,
},
cfg::FLOAT_OVER_FLOW_LIMIT,
hash_map,
Expand All @@ -16,6 +13,8 @@ use std::{collections::HashMap, fmt::Display, process::exit};

#[derive(PartialEq, Debug, Clone)]
pub enum TokenType {
// ->
Arrow,
// .
Dot,
// ,
Expand Down Expand Up @@ -72,8 +71,6 @@ pub enum TokenType {
SelfMod,
// **=
SelfPower,
// ~=
SelfBitNot,
// <<=
SelfBitLeftShift,
// >>=
Expand All @@ -88,6 +85,10 @@ pub enum TokenType {
StringValue,
FloatValue,
LongIntValue,
// ||=
SelfOr,
// &&=
SelfAnd,
// =
Assign,
// :=
Expand Down Expand Up @@ -121,8 +122,9 @@ pub enum TokenType {
Else,
Class,
Match,
// func
Func,
Import,
Return,
EndOfLine,
EndOfFile,
}
Expand Down Expand Up @@ -159,7 +161,6 @@ impl Display for TokenType {
TokenType::SelfExactDiv => res = "//=".to_string(),
TokenType::SelfMod => res = "%=".to_string(),
TokenType::SelfPower => res = "**=".to_string(),
TokenType::SelfBitNot => res = "~=".to_string(),
TokenType::SelfBitLeftShift => res = "<<=".to_string(),
TokenType::SelfBitRightShift => res = ">>=".to_string(),
TokenType::SelfBitAnd => res = "&=".to_string(),
Expand Down Expand Up @@ -192,12 +193,17 @@ impl Display for TokenType {
TokenType::Func => res = "func".to_string(),
TokenType::EndOfLine => res = "EOL".to_string(),
TokenType::EndOfFile => res = "EOF".to_string(),
TokenType::Import => res = "import".to_string(),
TokenType::Arrow => res = "->".to_string(),
TokenType::Return => res = "return".to_string(),
TokenType::SelfAnd => res = "&&=".to_string(),
TokenType::SelfOr => res = "||=".to_string(),
}
write!(f, "{}", res)
}
}

#[derive(PartialEq, Debug)]
#[derive(PartialEq, Debug, Clone)]
pub struct Token {
pub tp: TokenType,
pub data: Option<usize>,
Expand Down Expand Up @@ -251,13 +257,15 @@ macro_rules! check_braces_match {

lazy_static! {
static ref KEYWORDS: HashMap<String, TokenType> = hash_map![
String::from("while") => TokenType::While,
String::from("for") => TokenType::For,
String::from("if") => TokenType::If,
String::from("else") => TokenType::Else,
String::from("class") => TokenType::Class,
String::from("func") => TokenType::Func,
String::from("match") => TokenType::Match
"while".to_string() => TokenType::While,
"for".to_string() => TokenType::For,
"if".to_string() => TokenType::If,
"else".to_string() => TokenType::Else,
"class".to_string() => TokenType::Class,
"func".to_string() => TokenType::Func,
"match".to_string() => TokenType::Match,
"return".to_string() => TokenType::Return,
"import".to_string() => TokenType::Import
];
static ref RADIX_TO_PREFIX: HashMap<usize, &'static str> = hash_map![
2 => "0b",
Expand Down Expand Up @@ -392,7 +400,17 @@ impl TokenLex<'_> {
Token::new(TokenType::RightSmallBrace, None)
}
'+' => self.self_symbol(TokenType::Add, TokenType::SelfAdd),
'-' => self.self_symbol(TokenType::Sub, TokenType::SelfSub),
'-' => {
let c = self.compiler_data.input.read();
if c == '>' {
Token::new(TokenType::Arrow, None)
} else if c == '=' {
Token::new(TokenType::SelfSub, None)
} else {
self.compiler_data.input.unread(c);
Token::new(TokenType::Sub, None)
}
}
'*' => self.double_symbol(
TokenType::Mul,
TokenType::SelfMul,
Expand Down Expand Up @@ -425,10 +443,23 @@ impl TokenLex<'_> {
'<',
),
'~' => Token::new(TokenType::BitNot, None),
'^' => Token::new(TokenType::Xor, None),
'|' => self.binary_symbol(TokenType::Or, TokenType::BitOr, '|'),
':' => Token::new(TokenType::Colon, None),
'^' => self.self_symbol(TokenType::Xor, TokenType::SelfXor),
':' => self.binary_symbol(TokenType::Colon, TokenType::Store, '='),
';' => Token::new(TokenType::Semicolon, None),
'|' => self.double_symbol(
TokenType::BitOr,
TokenType::SelfBitOr,
TokenType::Or,
TokenType::SelfOr,
'|',
),
'&' => self.double_symbol(
TokenType::BitAnd,
TokenType::SelfBitAnd,
TokenType::And,
TokenType::SelfAnd,
'&',
),
_ => {
panic!("Not a symbol.Compiler error")
}
Expand Down Expand Up @@ -781,7 +812,7 @@ mod tests {
use std::{collections::HashSet, fmt::Debug, hash::Hash};

use super::*;
use crate::compiler::{Float, InputSource, Option, Pool, INT_VAL_POOL_ONE};
use crate::compiler::{Float, InputSource, Option, Pool, INT_VAL_POOL_ONE, INT_VAL_POOL_ZERO};

macro_rules! gen_test_token_env {
($test_string:expr, $env_name:ident) => {
Expand Down Expand Up @@ -937,7 +968,80 @@ mod tests {
}

#[test]
fn test_comprehensive_lex() {}
fn test_comprehensive_lex() {
gen_test_token_env!(
r#"
import "p"
func a(int val) -> str {
if val % 2 == 0 {
return "even"
} else {
return "odd"
}
}
func main() {
print("hello world")
p := a(intinput())
print(p)
}
"#,
t
);
check(
&mut t,
vec![
Token::new(TokenType::Import, None),
Token::new(TokenType::StringValue, Some(0)),
Token::new(TokenType::Func, None),
Token::new(TokenType::ID, Some(0)),
Token::new(TokenType::LeftSmallBrace, None),
Token::new(TokenType::ID, Some(1)),
Token::new(TokenType::ID, Some(2)),
Token::new(TokenType::RightSmallBrace, None),
Token::new(TokenType::Arrow, None),
Token::new(TokenType::ID, Some(3)),
Token::new(TokenType::LeftBigBrace, None),
Token::new(TokenType::If, None),
Token::new(TokenType::ID, Some(2)),
Token::new(TokenType::Mod, None),
Token::new(TokenType::IntValue, Some(2)),
Token::new(TokenType::Equal, None),
Token::new(TokenType::IntValue, Some(INT_VAL_POOL_ZERO)),
Token::new(TokenType::LeftBigBrace, None),
Token::new(TokenType::Return, None),
Token::new(TokenType::StringValue, Some(1)),
Token::new(TokenType::RightBigBrace, None),
Token::new(TokenType::Else, None),
Token::new(TokenType::LeftBigBrace, None),
Token::new(TokenType::Return, None),
Token::new(TokenType::StringValue, Some(2)),
Token::new(TokenType::RightBigBrace, None),
Token::new(TokenType::RightBigBrace, None),
Token::new(TokenType::Func, None),
Token::new(TokenType::ID, Some(4)),
Token::new(TokenType::LeftSmallBrace, None),
Token::new(TokenType::RightSmallBrace, None),
Token::new(TokenType::LeftBigBrace, None),
Token::new(TokenType::ID, Some(5)),
Token::new(TokenType::LeftSmallBrace, None),
Token::new(TokenType::StringValue, Some(3)),
Token::new(TokenType::RightSmallBrace, None),
Token::new(TokenType::ID, Some(6)),
Token::new(TokenType::Store, None),
Token::new(TokenType::ID, Some(0)),
Token::new(TokenType::LeftSmallBrace, None),
Token::new(TokenType::ID, Some(7)),
Token::new(TokenType::LeftSmallBrace, None),
Token::new(TokenType::RightSmallBrace, None),
Token::new(TokenType::RightSmallBrace, None),
Token::new(TokenType::ID, Some(5)),
Token::new(TokenType::LeftSmallBrace, None),
Token::new(TokenType::ID, Some(6)),
Token::new(TokenType::RightSmallBrace, None),
Token::new(TokenType::RightBigBrace, None),
],
);
}

#[test]
fn test_id_lex() {
Expand Down
5 changes: 3 additions & 2 deletions rust/src/tvm.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,8 +178,9 @@ impl<'a> Vm<'a> {
codegen::Opcode::StoreLocal => {}
codegen::Opcode::LoadString => {
self.dynadata.obj_stack.push(Box::new(TrcStr::new(
&self.static_data.constpool.stringpool
[self.static_data.inst[self.pc].operand],
self.static_data.constpool.stringpool
[self.static_data.inst[self.pc].operand]
.clone(),
)));
}
codegen::Opcode::LoadFloat => {
Expand Down
2 changes: 1 addition & 1 deletion rust/src/tvm/types.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ macro_rules! impl_oper {
fn $trait_oper_fn_name(&self, other:Box<dyn TrcObj>) -> TypeError {
match other.downcast_ref::<$self_type>() {
Some(v) => {
Ok(Box::new($newtype::new($oper(self.value, v.value)$whether_throw_error)))
Ok(Box::new($newtype::new($oper(self.value.clone(), v.value.clone())$whether_throw_error)))
},
None => {
Err(ErrorInfo::new(gettext!(OPERATOR_IS_NOT_SUPPORT, $error_oper_name, other.get_type_name()), gettext(OPERATOR_ERROR)))
Expand Down
Loading

0 comments on commit cd9136c

Please sign in to comment.