Skip to content

Commit bc64526

Browse files
authored
Merge pull request #57 from mangolang/lexer
MWE lexer #52
2 parents 4baed22 + 6dd62f7 commit bc64526

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

64 files changed

+1734
-286
lines changed

.travis.yml

+1
Original file line numberDiff line numberDiff line change
@@ -6,4 +6,5 @@ before_script:
66
sudo: false
77
cache: cargo
88
script:
9+
- cargo +nightly fmt --all -- --check
910
- cargo test --all

README.rst

+6
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@ These instructions were tested on Ubuntu 18.4 (using Bash). It should also work
6565
cargo test --all
6666
cargo run --bin mango-cli
6767
68+
or to build a fast, release-mode native binary:
69+
70+
.. code:: bash
71+
72+
RUSTFLAGS="-C target-cpu=native" cargo build --release
73+
6874
* To deploy the web version in release mode, run the script `dev/build_web.sh` (or view it for the steps needed). It uses Python's SimpleHTTPServer, if you don't have that, you can still find the deployable code in `target/deploy`.
6975

7076
* You're now ready to make changes! If you want to help, you're very welcome! Have a glance at CONTRIBUTING.rst_ if you have a minute.

dev/hooks/pre-commit

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,4 @@ set -o pipefail
88
util_dir="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils"
99

1010
# Check that the formatting is correct
11-
PYTHONPATH="$util_dir":$PYTHONPATH python3 "$util_dir/run_on_staged.py" 'cargo +nightly fmt --verbose --all -- --write-mode=diff' 'cargo test --all'
11+
PYTHONPATH="$util_dir":$PYTHONPATH python3 "$util_dir/run_on_staged.py" 'cargo +nightly fmt --all -- --check' 'cargo test --all'

dev/hooks/utils/run_on_staged.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ def do_cmds(cmds):
2121
run(cmd, allow_stderr=True, log=True)
2222
except Exception as err:
2323
stderr.write(str(err))
24-
stderr.write('FAILED, cancelling commit\n')
24+
stderr.write('\nFAILED, cancelling commit\n')
2525
return 1
2626
return 0
2727

dev/playground/src/enumhash.rs

-6
Original file line numberDiff line numberDiff line change
@@ -63,15 +63,9 @@ fn get_test_hash(x: &MyEnum) -> u64 {
6363
}
6464

6565
fn main() {
66-
<<<<<<< Updated upstream
6766
let a1: MyEnum = MyEnum::A(Alpha { val: "Hello World".to_owned() });
6867
let a2: MyEnum = MyEnum::A(Alpha { val: "Bye World".to_owned() });
6968
let a3: MyEnum = MyEnum::A(Alpha { val: "Bye World".to_owned() });
70-
=======
71-
let a1: MyEnum = MyEnum::A(Alpha { val: "Hello World".to_string() });
72-
let a2: MyEnum = MyEnum::A(Alpha { val: "Bye World".to_string() });
73-
let a3: MyEnum = MyEnum::A(Alpha { val: "Bye World".to_string() });
74-
>>>>>>> Stashed changes
7569
let b: MyEnum = MyEnum::B(Beta { nr: 8, f: 2 });
7670
let mut m = HashMap::new();
7771
println!("{:?} {:?}", a1.to_text(), b.to_text());

dev/playground/src/hashin.rs dev/playground/src/hashing.rs

-49
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ impl<H: 'static + Hasher> AnyHasher for H {
2929
}
3030
}
3131

32-
// TODO: but now I want this not for everything
3332
impl<T: 'static + Hash> MyTrait for T {
3433
fn as_any(&self) -> &Any {
3534
self as &Any
@@ -41,9 +40,6 @@ impl<T: 'static + Hash> MyTrait for T {
4140
}
4241
}
4342

44-
//impl MyTrait for A {}
45-
//impl MyTrait for B {}
46-
4743
impl Hash for MyTrait {
4844
fn hash<H: Hasher>(&self, hasher: &mut H) {
4945
self.my_hash(hasher)
@@ -57,48 +53,3 @@ fn main() {
5753
let x: &MyTrait = &A(1);
5854
x.hash(&mut hasher);
5955
}
60-
61-
62-
//trait PreS: Debug {}
63-
//
64-
//trait HasherAsAny {
65-
// fn as_any(&self) -> &Any;
66-
//}
67-
//
68-
//trait PostS {
69-
// fn as_any(&self) -> &Any;
70-
//
71-
// fn _hash<H: Hasher>(&self, hasher: H);
72-
//}
73-
//
74-
//impl<T: 'static + Hasher> HasherAsAny for T {
75-
// fn as_any(&self) -> &Any {
76-
// self as &Any
77-
// }
78-
//}
79-
//
80-
//impl<T: 'static + PreS> PostS for T {
81-
// fn as_any(&self) -> &Any {
82-
// self as &Any
83-
// }
84-
//
85-
// fn _hash<H: Hasher>(&self, hasher: H) {
86-
// self.as_any().downcast_ref::<T>().hash(hasher)
87-
// }
88-
//}
89-
//
90-
//impl PreS for A {}
91-
//
92-
//impl PreS for B {}
93-
//
94-
//impl Hash for PostS {
95-
// fn hash(&self, hasher: &mut HasherAsAny) {
96-
// self._hash(hasher.as_any().downcast_ref::<T>())
97-
// }
98-
//}
99-
//
100-
//fn main() {
101-
// let x: &PostS = &A(1);
102-
// let m = HashMap::new();
103-
// m.insert(x, 0);
104-
//}

rustfmt.toml

+1-4
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,2 @@
1-
reorder_extern_crates = true
2-
reorder_extern_crates_in_group = true
31
reorder_imports = true
4-
reorder_imports_in_group = true
5-
reorder_imported_names = true
2+
max_width = 140

src/lib.rs

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#![feature(nll)]
2+
//#![feature(generators, generator_trait)]
13
#![feature(proc_macro, wasm_custom_section, wasm_import_module)]
24
extern crate core;
35
extern crate wasm_bindgen;
@@ -10,6 +12,7 @@ extern crate derive_new;
1012

1113
pub mod mango {
1214
// Utilities
15+
pub mod io;
1316
pub mod jit;
1417
pub mod ui;
1518
pub mod util;

src/mango/ast_full/node/assignment.rs

+1-5
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,7 @@ impl AssignmentAST {
2121

2222
impl ToText for AssignmentAST {
2323
fn to_text(&self) -> String {
24-
return format!(
25-
"{0:} = ({1:})",
26-
self.assignee.to_text(),
27-
self.value.to_text()
28-
);
24+
return format!("{0:} = ({1:})", self.assignee.to_text(), self.value.to_text());
2925
}
3026
}
3127

src/mango/ast_full/node/unary_operation.rs

+1-5
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,7 @@ impl UnaryOperationAST {
2222

2323
impl ToText for UnaryOperationAST {
2424
fn to_text(&self) -> String {
25-
return format!(
26-
"({0:} {1:})",
27-
self.operator.to_text(),
28-
self.subject.to_text()
29-
);
25+
return format!("({0:} {1:})", self.operator.to_text(), self.subject.to_text());
3026
}
3127
}
3228

src/mango/ast_full/terminal/literal.rs

+1-3
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,7 @@ pub struct StringLiteralAST {
3131

3232
impl FloatLiteralAST {
3333
pub fn new(value: f64) -> Self {
34-
FloatLiteralAST {
35-
value: f64eq::new(value),
36-
}
34+
FloatLiteralAST { value: f64eq::new(value) }
3735
}
3836
}
3937

src/mango/io/fortest/mod.rs

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
pub mod stringreader;
2+
pub use self::stringreader::*;

src/mango/io/fortest/stringreader.rs

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
use mango::io::typ::Reader;
2+
use mango::io::typ::ReaderResult;
3+
use mango::io::util::REXCACHE;
4+
5+
/// Implementation of [Reader] that reads from a pre-provided string.
6+
/// Mostly for testing purposes.
7+
#[derive(Debug)]
8+
pub struct StringReader {
9+
code: String,
10+
index: usize,
11+
}
12+
13+
impl StringReader {
14+
pub fn new(code: String) -> Self {
15+
StringReader { code, index: 0 }
16+
}
17+
}
18+
19+
impl Reader for StringReader {
20+
fn matches(&mut self, subpattern: &str) -> ReaderResult {
21+
// Check for subpattern
22+
REXCACHE.with(|rl| {
23+
let mut rexlib = rl.borrow_mut();
24+
// Check for end of file
25+
// TODO: is there a better/faster way for this? maybe try this after a match and set a flag?
26+
let regex = rexlib.make_or_get(r"\s*$");
27+
match regex.find(&self.code[self.index..]) {
28+
Some(mtch) => {
29+
if self.index + mtch.as_str().len() == self.code.len() {
30+
self.index += mtch.as_str().len();
31+
return ReaderResult::EOF();
32+
}
33+
}
34+
None => (),
35+
}
36+
// Check for subpattern
37+
let regex = rexlib.make_or_get(subpattern);
38+
return match regex.find(&self.code[self.index..]) {
39+
Some(mtch) => {
40+
self.index += mtch.as_str().len();
41+
// Remove leading spaces
42+
let mut k = 0;
43+
for (i, byt) in mtch.as_str().chars().enumerate() {
44+
if byt != ' ' {
45+
break;
46+
}
47+
k = i + 1;
48+
}
49+
ReaderResult::Match((&mtch.as_str()[k..]).to_owned())
50+
}
51+
None => ReaderResult::NoMatch(),
52+
};
53+
})
54+
}
55+
56+
fn get_progress(&self) -> usize {
57+
self.index
58+
}
59+
}
60+
61+
// TODO: tests (spaces, end)

src/mango/io/mod.rs

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pub mod typ;
2+
3+
pub mod fortest;
4+
5+
pub mod util;

src/mango/io/typ.rs

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// TODO: I should perhaps separate the splitting that happens here from the actual reading
2+
3+
use std::fmt::Debug;
4+
5+
pub enum ReaderResult {
6+
Match(String),
7+
NoMatch(),
8+
EOF(),
9+
}
10+
11+
/// A reader represents a source 'file', which may be a file, webpage, string, ...
12+
pub trait Reader: Debug {
13+
/// Checks whether the `text` is found starting from the current position.
14+
// fn equals(&mut self, texts: Vec<&str>) -> ReaderResult;
15+
16+
/// Checks whether the code from the current position matches a regex pattern.
17+
///
18+
/// This has to eventually return EOF, and keep returning EOF forever after that.
19+
fn matches(&mut self, subpattern: &str) -> ReaderResult;
20+
21+
/// Return a number that can be used to check whether the state has changed.
22+
/// This need not correspond to a specific position, but should be unique for the progress.
23+
fn get_progress(&self) -> usize;
24+
}
25+
26+
pub trait Writer {
27+
// TODO
28+
}

src/mango/io/util.rs

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
use regex::Regex;
2+
use std::cell::RefCell;
3+
use std::collections::HashMap;
4+
5+
pub struct RegexCache {
6+
cache: HashMap<String, Regex>,
7+
}
8+
9+
impl RegexCache {
10+
// Not public to prevent having more than one instance.
11+
fn new() -> Self {
12+
RegexCache { cache: HashMap::new() }
13+
}
14+
15+
pub fn make_or_get(&mut self, subpattern: &str) -> &Regex {
16+
if !self.cache.contains_key(subpattern) {
17+
match Regex::new(&format!(r"^ *{}", subpattern)) {
18+
Err(err) => panic!(format!(
19+
"Invalid regular expression '{}' while adding to library; this is a bug:\n{:?}",
20+
subpattern, err
21+
)),
22+
Ok(regex) => {
23+
self.cache.insert(subpattern.to_owned(), regex);
24+
}
25+
}
26+
}
27+
self.cache.get(subpattern).unwrap()
28+
}
29+
}
30+
31+
thread_local! {
32+
pub static REXCACHE: RefCell<RegexCache> = RefCell::new(RegexCache::new())
33+
}

0 commit comments

Comments
 (0)