Merge pull request #57 from mangolang/lexer

MWE lexer #52

.travis.yml

@@ -6,4 +6,5 @@ before_script:
 sudo: false
 cache: cargo
 script:
+  - cargo +nightly fmt --all -- --check
   - cargo test --all

README.rst

@@ -65,6 +65,12 @@ These instructions were tested on Ubuntu 18.4 (using Bash). It should also work
 	cargo test --all
 	cargo run --bin mango-cli
 
+  or to build a fast, release-mode native binary:
+
+.. code:: bash
+
+    RUSTFLAGS="-C target-cpu=native" cargo build --release
+
 * To deploy the web version in release mode, run the script `dev/build_web.sh` (or view it for the steps needed). It uses Python's SimpleHTTPServer, if you don't have that, you can still find the deployable code in `target/deploy`.
 
 * You're now ready to make changes! If you want to help, you're very welcome! Have a glance at CONTRIBUTING.rst_ if you have a minute.

dev/hooks/pre-commit

@@ -8,4 +8,4 @@ set -o pipefail
 util_dir="$(dirname "$(realpath "${BASH_SOURCE[0]}")")/utils"
 
 # Check that the formatting is correct
-PYTHONPATH="$util_dir":$PYTHONPATH python3 "$util_dir/run_on_staged.py" 'cargo +nightly fmt --verbose --all -- --write-mode=diff' 'cargo test --all'
+PYTHONPATH="$util_dir":$PYTHONPATH python3 "$util_dir/run_on_staged.py" 'cargo +nightly fmt --all -- --check' 'cargo test --all'

dev/hooks/utils/run_on_staged.py

@@ -21,7 +21,7 @@ def do_cmds(cmds):
 				run(cmd, allow_stderr=True, log=True)
 			except Exception as err:
 				stderr.write(str(err))
-				stderr.write('FAILED, cancelling commit\n')
+				stderr.write('\nFAILED, cancelling commit\n')
 				return 1
 		return 0
 

dev/playground/src/enumhash.rs

@@ -63,15 +63,9 @@ fn get_test_hash(x: &MyEnum) -> u64 {
 }
 
 fn main() {
-<<<<<<< Updated upstream
     let a1: MyEnum = MyEnum::A(Alpha { val: "Hello World".to_owned() });
     let a2: MyEnum = MyEnum::A(Alpha { val: "Bye World".to_owned() });
     let a3: MyEnum = MyEnum::A(Alpha { val: "Bye World".to_owned() });
-=======
-    let a1: MyEnum = MyEnum::A(Alpha { val: "Hello World".to_string() });
-    let a2: MyEnum = MyEnum::A(Alpha { val: "Bye World".to_string() });
-    let a3: MyEnum = MyEnum::A(Alpha { val: "Bye World".to_string() });
->>>>>>> Stashed changes
     let b: MyEnum = MyEnum::B(Beta { nr: 8, f: 2 });
     let mut m = HashMap::new();
     println!("{:?} {:?}", a1.to_text(), b.to_text());

dev/playground/src/hashin.rs → dev/playground/src/hashing.rs

@@ -29,7 +29,6 @@ impl<H: 'static + Hasher> AnyHasher for H {
 	}
 }
 
-// TODO: but now I want this not for everything
 impl<T: 'static + Hash> MyTrait for T {
 	fn as_any(&self) -> &Any {
 		self as &Any
@@ -41,9 +40,6 @@ impl<T: 'static + Hash> MyTrait for T {
 	}
 }
 
-//impl MyTrait for A {}
-//impl MyTrait for B {}
-
 impl Hash for MyTrait {
 	fn hash<H: Hasher>(&self, hasher: &mut H) {
 		self.my_hash(hasher)
@@ -57,48 +53,3 @@ fn main() {
 	let x: &MyTrait = &A(1);
 	x.hash(&mut hasher);
 }
-
-
-//trait PreS: Debug {}
-//
-//trait HasherAsAny {
-//	fn as_any(&self) -> &Any;
-//}
-//
-//trait PostS {
-//	fn as_any(&self) -> &Any;
-//
-//	fn _hash<H: Hasher>(&self, hasher: H);
-//}
-//
-//impl<T: 'static + Hasher> HasherAsAny for T {
-//	fn as_any(&self) -> &Any {
-//		self as &Any
-//	}
-//}
-//
-//impl<T: 'static + PreS> PostS for T {
-//	fn as_any(&self) -> &Any {
-//		self as &Any
-//	}
-//
-//	fn _hash<H: Hasher>(&self, hasher: H) {
-//		self.as_any().downcast_ref::<T>().hash(hasher)
-//	}
-//}
-//
-//impl PreS for A {}
-//
-//impl PreS for B {}
-//
-//impl Hash for PostS {
-//	fn hash(&self, hasher: &mut HasherAsAny) {
-//		self._hash(hasher.as_any().downcast_ref::<T>())
-//	}
-//}
-//
-//fn main() {
-//	let x: &PostS = &A(1);
-//	let m = HashMap::new();
-//	m.insert(x, 0);
-//}

rustfmt.toml

@@ -1,5 +1,2 @@
-reorder_extern_crates = true
-reorder_extern_crates_in_group = true
 reorder_imports = true
-reorder_imports_in_group = true
-reorder_imported_names = true
+max_width = 140

src/lib.rs

@@ -1,3 +1,5 @@
+#![feature(nll)]
+//#![feature(generators, generator_trait)]
 #![feature(proc_macro, wasm_custom_section, wasm_import_module)]
 extern crate core;
 extern crate wasm_bindgen;
@@ -10,6 +12,7 @@ extern crate derive_new;
 
 pub mod mango {
     // Utilities
+    pub mod io;
     pub mod jit;
     pub mod ui;
     pub mod util;

src/mango/ast_full/node/assignment.rs

@@ -21,11 +21,7 @@ impl AssignmentAST {
 
 impl ToText for AssignmentAST {
     fn to_text(&self) -> String {
-        return format!(
-            "{0:} = ({1:})",
-            self.assignee.to_text(),
-            self.value.to_text()
-        );
+        return format!("{0:} = ({1:})", self.assignee.to_text(), self.value.to_text());
     }
 }
 

src/mango/ast_full/node/unary_operation.rs

@@ -22,11 +22,7 @@ impl UnaryOperationAST {
 
 impl ToText for UnaryOperationAST {
     fn to_text(&self) -> String {
-        return format!(
-            "({0:} {1:})",
-            self.operator.to_text(),
-            self.subject.to_text()
-        );
+        return format!("({0:} {1:})", self.operator.to_text(), self.subject.to_text());
     }
 }
 

src/mango/ast_full/terminal/literal.rs

@@ -31,9 +31,7 @@ pub struct StringLiteralAST {
 
 impl FloatLiteralAST {
     pub fn new(value: f64) -> Self {
-        FloatLiteralAST {
-            value: f64eq::new(value),
-        }
+        FloatLiteralAST { value: f64eq::new(value) }
     }
 }
 

src/mango/io/fortest/mod.rs

@@ -0,0 +1,2 @@
+pub mod stringreader;
+pub use self::stringreader::*;

src/mango/io/fortest/stringreader.rs

@@ -0,0 +1,61 @@
+use mango::io::typ::Reader;
+use mango::io::typ::ReaderResult;
+use mango::io::util::REXCACHE;
+
+/// Implementation of [Reader] that reads from a pre-provided string.
+/// Mostly for testing purposes.
+#[derive(Debug)]
+pub struct StringReader {
+    code: String,
+    index: usize,
+}
+
+impl StringReader {
+    pub fn new(code: String) -> Self {
+        StringReader { code, index: 0 }
+    }
+}
+
+impl Reader for StringReader {
+    fn matches(&mut self, subpattern: &str) -> ReaderResult {
+        // Check for subpattern
+        REXCACHE.with(|rl| {
+            let mut rexlib = rl.borrow_mut();
+            // Check for end of file
+            // TODO: is there a better/faster way for this? maybe try this after a match and set a flag?
+            let regex = rexlib.make_or_get(r"\s*$");
+            match regex.find(&self.code[self.index..]) {
+                Some(mtch) => {
+                    if self.index + mtch.as_str().len() == self.code.len() {
+                        self.index += mtch.as_str().len();
+                        return ReaderResult::EOF();
+                    }
+                }
+                None => (),
+            }
+            // Check for subpattern
+            let regex = rexlib.make_or_get(subpattern);
+            return match regex.find(&self.code[self.index..]) {
+                Some(mtch) => {
+                    self.index += mtch.as_str().len();
+                    // Remove leading spaces
+                    let mut k = 0;
+                    for (i, byt) in mtch.as_str().chars().enumerate() {
+                        if byt != ' ' {
+                            break;
+                        }
+                        k = i + 1;
+                    }
+                    ReaderResult::Match((&mtch.as_str()[k..]).to_owned())
+                }
+                None => ReaderResult::NoMatch(),
+            };
+        })
+    }
+
+    fn get_progress(&self) -> usize {
+        self.index
+    }
+}
+
+// TODO: tests (spaces, end)

src/mango/io/mod.rs

@@ -0,0 +1,5 @@
+pub mod typ;
+
+pub mod fortest;
+
+pub mod util;

src/mango/io/typ.rs

@@ -0,0 +1,28 @@
+// TODO: I should perhaps separate the splitting that happens here from the actual reading
+
+use std::fmt::Debug;
+
+pub enum ReaderResult {
+    Match(String),
+    NoMatch(),
+    EOF(),
+}
+
+/// A reader represents a source 'file', which may be a file, webpage, string, ...
+pub trait Reader: Debug {
+    /// Checks whether the `text` is found starting from the current position.
+    //    fn equals(&mut self, texts: Vec<&str>) -> ReaderResult;
+
+    /// Checks whether the code from the current position matches a regex pattern.
+    ///
+    /// This has to eventually return EOF, and keep returning EOF forever after that.
+    fn matches(&mut self, subpattern: &str) -> ReaderResult;
+
+    /// Return a number that can be used to check whether the state has changed.
+    /// This need not correspond to a specific position, but should be unique for the progress.
+    fn get_progress(&self) -> usize;
+}
+
+pub trait Writer {
+    // TODO
+}

src/mango/io/util.rs

@@ -0,0 +1,33 @@
+use regex::Regex;
+use std::cell::RefCell;
+use std::collections::HashMap;
+
+pub struct RegexCache {
+    cache: HashMap<String, Regex>,
+}
+
+impl RegexCache {
+    // Not public to prevent having more than one instance.
+    fn new() -> Self {
+        RegexCache { cache: HashMap::new() }
+    }
+
+    pub fn make_or_get(&mut self, subpattern: &str) -> &Regex {
+        if !self.cache.contains_key(subpattern) {
+            match Regex::new(&format!(r"^ *{}", subpattern)) {
+                Err(err) => panic!(format!(
+                    "Invalid regular expression '{}' while adding to library; this is a bug:\n{:?}",
+                    subpattern, err
+                )),
+                Ok(regex) => {
+                    self.cache.insert(subpattern.to_owned(), regex);
+                }
+            }
+        }
+        self.cache.get(subpattern).unwrap()
+    }
+}
+
+thread_local! {
+    pub static REXCACHE: RefCell<RegexCache> = RefCell::new(RegexCache::new())
+}