From 8f0dfe2bf379cd077bc098fdd64f6ff76fbde2ad Mon Sep 17 00:00:00 2001 From: arty Date: Sat, 4 Mar 2023 04:58:59 -0800 Subject: [PATCH 01/81] WIP getting a little farther --- src/compiler/preprocessor.rs | 268 +++++++++++++++++---------------- src/tests/compiler/compiler.rs | 13 ++ 2 files changed, 153 insertions(+), 128 deletions(-) diff --git a/src/compiler/preprocessor.rs b/src/compiler/preprocessor.rs index 9d364cfe4..ef1604e09 100644 --- a/src/compiler/preprocessor.rs +++ b/src/compiler/preprocessor.rs @@ -1,50 +1,61 @@ use std::borrow::Borrow; use std::rc::Rc; +use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; + use crate::compiler::compiler::KNOWN_DIALECTS; use crate::compiler::comptypes::{CompileErr, CompilerOpts, IncludeDesc}; +use crate::compiler::evaluate::Evaluator; use crate::compiler::sexp::{decode_string, enlist, parse_sexp, SExp}; use crate::compiler::srcloc::Srcloc; use crate::util::ErrInto; -pub fn process_include( +struct Preprocessor { opts: Rc, - include: IncludeDesc, -) -> Result>, CompileErr> { - let filename_and_content = opts.read_new_file(opts.filename(), decode_string(&include.name))?; - let content = filename_and_content.1; - let start_of_file = Srcloc::start(&decode_string(&include.name)); - - // Because we're also subsequently returning CompileErr later in the pipe, - // this needs an explicit err map. - parse_sexp(start_of_file.clone(), content.bytes()) - .err_into() - .and_then(|x| match x[0].proper_list() { - None => Err(CompileErr( - start_of_file, - "Includes should contain a list of forms".to_string(), - )), - Some(v) => Ok(v.iter().map(|x| Rc::new(x.clone())).collect()), - }) + evaluator: Evaluator } -/* Expand include inline in forms */ -fn process_pp_form( - opts: Rc, - includes: &mut Vec, - body: Rc, -) -> Result>, CompileErr> { - // Support using the preprocessor to collect dependencies recursively. - let recurse_dependencies = |opts: Rc, - includes: &mut Vec, - desc: IncludeDesc| - -> Result<(), CompileErr> { +impl Preprocessor { + pub fn new(opts: Rc) -> Self { + let runner = Rc::new(DefaultProgramRunner::new()); + Preprocessor { + opts: opts.clone(), + evaluator: Evaluator::new(opts, runner, Vec::new()) + } + } + + pub fn process_include( + &mut self, + include: IncludeDesc, + ) -> Result>, CompileErr> { + let filename_and_content = self.opts.read_new_file(self.opts.filename(), decode_string(&include.name))?; + let content = filename_and_content.1; + let start_of_file = Srcloc::start(&decode_string(&include.name)); + + // Because we're also subsequently returning CompileErr later in the pipe, + // this needs an explicit err map. + parse_sexp(start_of_file.clone(), content.bytes()) + .err_into() + .and_then(|x| match x[0].proper_list() { + None => Err(CompileErr( + start_of_file, + "Includes should contain a list of forms".to_string(), + )), + Some(v) => Ok(v.iter().map(|x| Rc::new(x.clone())).collect()), + }) + } + + fn recurse_dependencies( + &mut self, + includes: &mut Vec, + desc: IncludeDesc + ) -> Result<(), CompileErr> { let name_string = decode_string(&desc.name); if KNOWN_DIALECTS.contains_key(&name_string) { return Ok(()); } - let (full_name, content) = opts.read_new_file(opts.filename(), name_string)?; + let (full_name, content) = self.opts.read_new_file(self.opts.filename(), name_string)?; includes.push(IncludeDesc { name: full_name.as_bytes().to_vec(), ..desc @@ -58,123 +69,127 @@ fn process_pp_form( let program_form = parsed[0].clone(); if let Some(l) = program_form.proper_list() { for elt in l.iter() { - process_pp_form(opts.clone(), includes, Rc::new(elt.clone()))?; + self.process_pp_form(includes, Rc::new(elt.clone()))?; } } Ok(()) - }; - - let included: Option = body - .proper_list() - .map(|x| x.iter().map(|elt| elt.atomize()).collect()) - .map(|x: Vec| { - match &x[..] { - [SExp::Atom(kw, inc), SExp::Atom(nl, fname)] => { - if "include".as_bytes().to_vec() == *inc { - return Ok(Some(IncludeDesc { - kw: kw.clone(), - nl: nl.clone(), - name: fname.clone(), - })); + } + + /* Expand include inline in forms */ + fn process_pp_form( + &mut self, + includes: &mut Vec, + body: Rc, + ) -> Result>, CompileErr> { + // Support using the preprocessor to collect dependencies recursively. + let included: Option = body + .proper_list() + .map(|x| x.iter().map(|elt| elt.atomize()).collect()) + .map(|x: Vec| { + match &x[..] { + [SExp::Atom(kw, inc), SExp::Atom(nl, fname)] => { + if "include".as_bytes().to_vec() == *inc { + return Ok(Some(IncludeDesc { + kw: kw.clone(), + nl: nl.clone(), + name: fname.clone(), + })); + } } - } - [SExp::Atom(kw, inc), SExp::QuotedString(nl, _, fname)] => { - if "include".as_bytes().to_vec() == *inc { - return Ok(Some(IncludeDesc { - kw: kw.clone(), - nl: nl.clone(), - name: fname.clone(), - })); + [SExp::Atom(kw, inc), SExp::QuotedString(nl, _, fname)] => { + if "include".as_bytes().to_vec() == *inc { + return Ok(Some(IncludeDesc { + kw: kw.clone(), + nl: nl.clone(), + name: fname.clone(), + })); + } } - } - [] => {} - - // Ensure that legal empty or atom expressions don't try include - _ => { - // Include is only allowed as a proper form. It's a keyword in - // this language. - if let SExp::Atom(_, inc) = &x[0] { - if "include".as_bytes().to_vec() == *inc { - return Err(CompileErr( - body.loc(), - format!("bad tail in include {body}"), - )); + [] => {} + + // Ensure that legal empty or atom expressions don't try include + _ => { + // Include is only allowed as a proper form. It's a keyword in + // this language. + if let SExp::Atom(_, inc) = &x[0] { + if "include".as_bytes().to_vec() == *inc { + return Err(CompileErr( + body.loc(), + format!("bad tail in include {body}"), + )); + } } } } - } - Ok(None) - }) - .unwrap_or_else(|| Ok(None))?; + Ok(None) + }) + .unwrap_or_else(|| Ok(None))?; - if let Some(i) = included { - recurse_dependencies(opts.clone(), includes, i.clone())?; - process_include(opts, i) - } else { - Ok(vec![body]) + if let Some(i) = included { + self.recurse_dependencies(includes, i.clone())?; + self.process_include(i) + } else { + Ok(vec![body]) + } } -} -fn preprocess_( - opts: Rc, - includes: &mut Vec, - body: Rc, -) -> Result>, CompileErr> { - match body.borrow() { - SExp::Cons(_, head, rest) => match rest.borrow() { - SExp::Nil(_nl) => process_pp_form(opts, includes, head.clone()), + fn inject_std_macros(&mut self, body: Rc) -> SExp { + match body.proper_list() { + Some(v) => { + let include_form = Rc::new(SExp::Cons( + body.loc(), + Rc::new(SExp::atom_from_string(body.loc(), "include")), + Rc::new(SExp::Cons( + body.loc(), + Rc::new(SExp::quoted_from_string(body.loc(), "*macros*")), + Rc::new(SExp::Nil(body.loc())), + )), + )); + let mut v_clone: Vec> = v.iter().map(|x| Rc::new(x.clone())).collect(); + let include_copy: &SExp = include_form.borrow(); + v_clone.insert(0, Rc::new(include_copy.clone())); + enlist(body.loc(), v_clone) + } _ => { - let lst = process_pp_form(opts.clone(), includes, head.clone())?; - let mut rs = preprocess_(opts, includes, rest.clone())?; - let mut result = lst; - result.append(&mut rs); - Ok(result) + let body_clone: &SExp = body.borrow(); + body_clone.clone() } - }, - _ => Ok(vec![body]), + } } -} -fn inject_std_macros(body: Rc) -> SExp { - match body.proper_list() { - Some(v) => { - let include_form = Rc::new(SExp::Cons( - body.loc(), - Rc::new(SExp::atom_from_string(body.loc(), "include")), - Rc::new(SExp::Cons( - body.loc(), - Rc::new(SExp::quoted_from_string(body.loc(), "*macros*")), - Rc::new(SExp::Nil(body.loc())), - )), - )); - let mut v_clone: Vec> = v.iter().map(|x| Rc::new(x.clone())).collect(); - let include_copy: &SExp = include_form.borrow(); - v_clone.insert(0, Rc::new(include_copy.clone())); - enlist(body.loc(), v_clone) - } - _ => { - let body_clone: &SExp = body.borrow(); - body_clone.clone() + pub fn run( + &mut self, + includes: &mut Vec, + cmod: Rc, + ) -> Result>, CompileErr> { + let mut result = Vec::new(); + let mut tocompile = if self.opts.stdenv() { + let injected = self.inject_std_macros(cmod); + Rc::new(injected) + } else { + cmod + }; + + while let SExp::Cons(_,f,r) = tocompile.borrow() { + let mut lst = self.process_pp_form(includes, f.clone())?; + result.append(&mut lst); + tocompile = r.clone(); } + + Ok(result) } } pub fn preprocess( opts: Rc, includes: &mut Vec, - cmod: Rc, + cmod: Rc ) -> Result>, CompileErr> { - let tocompile = if opts.stdenv() { - let injected = inject_std_macros(cmod); - Rc::new(injected) - } else { - cmod - }; - - preprocess_(opts, includes, tocompile) + let mut p = Preprocessor::new(opts); + p.run(includes, cmod) } // Visit all files used during compilation. @@ -184,6 +199,7 @@ pub fn gather_dependencies( file_content: &str, ) -> Result, CompileErr> { let mut includes = Vec::new(); + let mut p = Preprocessor::new(opts); let loc = Srcloc::start(real_input_path); let parsed = parse_sexp(loc.clone(), file_content.bytes())?; @@ -192,13 +208,9 @@ pub fn gather_dependencies( return Ok(vec![]); } - if let Some(l) = parsed[0].proper_list() { - for elt in l.iter() { - process_pp_form(opts.clone(), &mut includes, Rc::new(elt.clone()))?; - } - } else { - return Err(CompileErr(loc, "malformed list body".to_string())); - }; + for elt in parsed.iter() { + p.run(&mut includes, elt.clone())?; + } Ok(includes) } diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index a06879f96..85741c97f 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -1192,3 +1192,16 @@ fn test_inline_out_of_bounds_diagnostic() { assert!(false); } } + +#[test] +fn test_defmac_basic_0() { + let prog = indoc! {" + (defmac double-arg (A) (list (concat A \"1\") (concat A \"2\"))) + (mod (double-arg X) + (+ X1 X2) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3 4)".to_string()).unwrap(); + assert_eq!(res.to_string(), "7"); +} From e25bf6310157f2e6fe6fd8ce931c2e0a124281df Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 8 Mar 2023 23:31:28 -0800 Subject: [PATCH 02/81] Updates, WIP --- src/compiler/evaluate.rs | 36 ++++++- src/compiler/frontend.rs | 2 +- src/compiler/preprocessor.rs | 169 +++++++++++++++++++++++++++++++-- src/compiler/sexp.rs | 116 ++++++++++++++++++++++ src/tests/compiler/compiler.rs | 7 +- 5 files changed, 316 insertions(+), 14 deletions(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 541c8c8b3..8f4a1cc18 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -70,10 +70,23 @@ pub enum ArgInputs { Pair(Rc, Rc), } +pub trait EvalExtension { + fn try_eval( + &self, + evaluator: &Evaluator, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result>, CompileErr>; +} + pub struct Evaluator { opts: Rc, runner: Rc, prims: Rc, Rc>>, + extensions: Vec>, helpers: Vec, mash_conditions: bool, ignore_exn: bool, @@ -151,7 +164,7 @@ fn get_bodyform_from_arginput(l: &Srcloc, arginput: &ArgInputs) -> Rc // // It's possible this will result in irreducible (unknown at compile time) // argument expressions. -fn create_argument_captures( +pub fn create_argument_captures( argument_captures: &mut HashMap, Rc>, formed_arguments: &ArgInputs, function_arg_spec: Rc, @@ -248,7 +261,7 @@ fn arg_inputs_primitive(arginputs: Rc) -> bool { } } -fn build_argument_captures( +pub fn build_argument_captures( l: &Srcloc, arguments_to_convert: &[Rc], args: Rc, @@ -574,6 +587,7 @@ impl<'info> Evaluator { helpers, mash_conditions: false, ignore_exn: false, + extensions: Vec::new() } } @@ -583,6 +597,7 @@ impl<'info> Evaluator { runner: self.runner.clone(), prims: self.prims.clone(), helpers: self.helpers.clone(), + extensions: self.extensions.clone(), mash_conditions: true, ignore_exn: true, } @@ -688,6 +703,19 @@ impl<'info> Evaluator { let compiled_borrowed: &SExp = compiled.borrow(); Ok(Rc::new(BodyForm::Quoted(compiled_borrowed.clone()))) } else { + for ext in self.extensions.iter() { + if let Some(res) = ext.try_eval( + self, + env, + &l, + call_name, + arguments_to_convert, + body.clone() + )? { + return Ok(res); + } + } + let pres = self .lookup_prim(l.clone(), call_name) .map(|prim| { @@ -1253,6 +1281,10 @@ impl<'info> Evaluator { self.helpers.push(h.clone()); } + pub fn add_extension(&mut self, e: Rc) { + self.extensions.push(e); + } + // The evaluator treats the forms coming up from constants as live. fn get_constant(&self, name: &[u8]) -> Option> { for h in self.helpers.iter() { diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index b25b967c0..2dbdf2a12 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -565,7 +565,7 @@ pub fn compile_helperform( matched.args, ) .map(Some) - } else if matched.op_name == b"defmacro" { + } else if matched.op_name == b"defmacro" || matched.op_name == b"defmac" { compile_defmacro( opts, l, diff --git a/src/compiler/preprocessor.rs b/src/compiler/preprocessor.rs index ef1604e09..32b96e98e 100644 --- a/src/compiler/preprocessor.rs +++ b/src/compiler/preprocessor.rs @@ -1,26 +1,69 @@ use std::borrow::Borrow; +use std::collections::HashMap; use std::rc::Rc; +use clvmr::allocator::Allocator; + use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; use crate::compiler::compiler::KNOWN_DIALECTS; -use crate::compiler::comptypes::{CompileErr, CompilerOpts, IncludeDesc}; -use crate::compiler::evaluate::Evaluator; -use crate::compiler::sexp::{decode_string, enlist, parse_sexp, SExp}; +use crate::compiler::comptypes::{BodyForm, CompileErr, CompilerOpts, HelperForm, IncludeDesc}; +use crate::compiler::evaluate::{ArgInputs, create_argument_captures, dequote, EvalExtension, Evaluator}; +use crate::compiler::frontend::compile_helperform; +use crate::compiler::sexp::{Atom, decode_string, enlist, First, NodeSel, parse_sexp, SelectNode, SExp, ThisNode}; use crate::compiler::srcloc::Srcloc; use crate::util::ErrInto; struct Preprocessor { opts: Rc, - evaluator: Evaluator + evaluator: Evaluator, + helpers: Vec, +} + +struct PreprocessorExtension { } +impl EvalExtension for PreprocessorExtension { + fn try_eval( + &self, + evaluator: &Evaluator, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result>, CompileErr> { + if name == b"string->symbol" { + if args.len() != 1 { + return Err(CompileErr(loc.clone(), "string->symbol needs 1 argument".to_string())); + } + + eprintln!("args[0] {}", args[0].to_sexp()); + + if let BodyForm::Quoted(SExp::QuotedString(al,_,an)) = args[0].borrow() { + return Ok(Some(Rc::new(BodyForm::Quoted(SExp::Atom(al.clone(),an.clone()))))); + } else if let BodyForm::Quoted(x) = args[0].borrow() { + return Err(CompileErr(loc.clone(), "string->symbol takes a string".to_string())); + } else { + return Ok(Some(body.clone())); + } + } + + Ok(None) + } +} + +impl PreprocessorExtension { + fn new() -> Self { PreprocessorExtension { } } } impl Preprocessor { pub fn new(opts: Rc) -> Self { let runner = Rc::new(DefaultProgramRunner::new()); + let mut eval = Evaluator::new(opts.clone(), runner, Vec::new()); + eval.add_extension(Rc::new(PreprocessorExtension::new())); Preprocessor { opts: opts.clone(), - evaluator: Evaluator::new(opts, runner, Vec::new()) + evaluator: eval, + helpers: Vec::new() } } @@ -76,12 +119,114 @@ impl Preprocessor { Ok(()) } + // Check for and apply preprocessor level macros. + // This is maximally permissive. + fn expand_macros( + &mut self, + body: Rc + ) -> Result, CompileErr> { + eprintln!("expand_macros {}", body); + if let SExp::Cons(l,f,r) = body.borrow() { + // First expand inner macros. + let first_expanded = self.expand_macros(f.clone())?; + let rest_expanded = self.expand_macros(r.clone())?; + let new_self = SExp::Cons(l.clone(), first_expanded, rest_expanded); + if let Ok(NodeSel::Cons((nl, name), args)) = NodeSel::Cons( + Atom::Here(()), ThisNode::Here + ).select_nodes(body.clone()) { + // See if it's a form that calls one of our macros. + for m in self.helpers.iter() { + eprintln!("want {} helper {}", decode_string(&name), m.to_sexp()); + if let HelperForm::Defun(_,mdata) = &m { // We record upfront macros + if mdata.name != name { + continue; + } + + // as inline defuns because they're closest to that + // semantically. + let mut allocator = Allocator::new(); + // The name matched, try calling it. + + // Form argument env. + let mut macro_arg_env = HashMap::new(); + let args_borrowed: &SExp = args.borrow(); + create_argument_captures( + &mut macro_arg_env, + &ArgInputs::Whole(Rc::new(BodyForm::Quoted(args_borrowed.clone()))), + mdata.args.clone() + )?; + + let res = self.evaluator.shrink_bodyform( + &mut allocator, + mdata.args.clone(), + ¯o_arg_env, + mdata.body.clone(), + false, + None + )?; + + eprintln!("shrink res {}", res.to_sexp()); + + if let Ok(unquoted) = dequote(body.loc(), res) { + return Ok(unquoted); + } + } + } + } + } + + Ok(body) + } + + // If it's a defmac (preprocessor level macro), add it to the evaulator. + fn decode_macro( + &mut self, + definition: Rc + ) -> Result, CompileErr> { + eprintln!("decode_macro {definition}"); + if let Ok(NodeSel::Cons( + defmac_loc, + NodeSel::Cons( + (nl, name), + NodeSel::Cons(args,body) + ) + )) = NodeSel::Cons( + Atom::Here("defmac"), + NodeSel::Cons( + Atom::Here(()), + NodeSel::Cons(ThisNode::Here, ThisNode::Here) + ) + ).select_nodes(definition.clone()) { + let target_defun = Rc::new(SExp::Cons( + defmac_loc.clone(), + Rc::new(SExp::atom_from_string(defmac_loc.clone(), "defun")), + Rc::new(SExp::Cons( + nl.clone(), + Rc::new(SExp::Atom(nl.clone(), name.clone())), + Rc::new(SExp::Cons( + args.loc(), args.clone(), body.clone() + )) + )) + )); + eprintln!("target_defun {target_defun}"); + if let Some(helper) = compile_helperform(self.opts.clone(), target_defun)? { + self.evaluator.add_helper(&helper); + self.helpers.push(helper); + } else { + return Err(CompileErr(definition.loc(), "defmac found but couldn't be converted to function".to_string())); + } + } + + Ok(None) + } + /* Expand include inline in forms */ fn process_pp_form( &mut self, includes: &mut Vec, - body: Rc, + unexpanded_body: Rc, ) -> Result>, CompileErr> { + let body = self.expand_macros(unexpanded_body)?; // Support using the preprocessor to collect dependencies recursively. let included: Option = body .proper_list() @@ -119,6 +264,11 @@ impl Preprocessor { body.loc(), format!("bad tail in include {body}"), )); + } else { + // Try to pick up helperforms. + if let Some(()) = self.decode_macro(body.clone())? { + return Ok(None); + } } } } @@ -128,7 +278,9 @@ impl Preprocessor { }) .unwrap_or_else(|| Ok(None))?; - if let Some(i) = included { + if let Some(()) = self.decode_macro(body.clone())? { + Ok(vec![]) + } else if let Some(i) = included { self.recurse_dependencies(includes, i.clone())?; self.process_include(i) } else { @@ -199,7 +351,8 @@ pub fn gather_dependencies( file_content: &str, ) -> Result, CompileErr> { let mut includes = Vec::new(); - let mut p = Preprocessor::new(opts); + let no_stdenv_opts = opts.set_stdenv(false); + let mut p = Preprocessor::new(no_stdenv_opts); let loc = Srcloc::start(real_input_path); let parsed = parse_sexp(loc.clone(), file_content.bytes())?; diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index d50ff13d2..f593754a3 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -804,3 +804,119 @@ where { parse_sexp_inner(start, SExpParseState::Empty, input) } + +// This is a trait that generates a haskell-like ad-hoc type from the user's +// construction of NodeSel and ThisNode. +// the result is transformed into a NodeSel tree of NodePtr if it can be. +// The type of the result is an ad-hoc shape derived from the shape of the +// original request. +// +// This mirrors code in src/classic/clvm/sexp.rs +// +// It's a nicer way of modelling matches that will overtake bespoke code for a lot +// of things. +#[derive(Debug, Clone)] +pub enum NodeSel { + Cons(T, U), +} + +#[derive(Debug, Clone)] +pub enum First { + Here(T), +} + +#[derive(Debug, Clone)] +pub enum Rest { + Here(T), +} + +#[derive(Debug, Clone)] +pub enum ThisNode { + Here, +} + +pub enum Atom { + Here(T), +} + +pub trait SelectNode { + fn select_nodes(&self, s: Rc) -> Result; +} + +impl SelectNode, E> for ThisNode { + fn select_nodes(&self, s: Rc) -> Result, E> { + Ok(s) + } +} + +impl SelectNode<(Srcloc, Vec), (Srcloc, String)> for Atom<()> { + fn select_nodes(&self, s: Rc) -> Result<(Srcloc, Vec), (Srcloc, String)> + { + if let SExp::Atom(loc,name) = s.borrow() { + return Ok((loc.clone(),name.clone())); + } + + Err((s.loc(), "Not an atom".to_string())) + } +} + +impl SelectNode for Atom<&str> { + fn select_nodes(&self, s: Rc) -> Result { + let Atom::Here(name) = self; + if let Ok((l,n)) = Atom::Here(()).select_nodes(s.clone()) { + if n == name.as_bytes() { + return Ok(l.clone()); + } + } + + Err((s.loc(), format!("Not an atom named {name}"))) + } +} + +impl SelectNode<(), E> for () { + fn select_nodes(&self, _n: Rc) -> Result<(), E> { + Ok(()) + } +} + +impl SelectNode, E> for First +where + R: SelectNode + Clone, + E: From<(Srcloc, String)>, +{ + fn select_nodes(&self, s: Rc) -> Result, E> { + let First::Here(f) = &self; + let NodeSel::Cons(first, ()) = NodeSel::Cons(f.clone(), ()).select_nodes(s)?; + Ok(First::Here(first)) + } +} + +impl SelectNode, E> for Rest +where + R: SelectNode + Clone, + E: From<(Srcloc, String)>, +{ + fn select_nodes(&self, s: Rc) -> Result, E> { + let Rest::Here(f) = &self; + let NodeSel::Cons((), rest) = NodeSel::Cons((), f.clone()).select_nodes(s)?; + Ok(Rest::Here(rest)) + } +} + +impl SelectNode, E> for NodeSel +where + R: SelectNode, + S: SelectNode, + E: From<(Srcloc, String)>, +{ + fn select_nodes(&self, s: Rc) -> Result, E> { + let NodeSel::Cons(my_left, my_right) = &self; + if let SExp::Cons(_, l, r) = s.borrow() { + let first = my_left.select_nodes(l.clone())?; + let rest = my_right.select_nodes(r.clone())?; + Ok(NodeSel::Cons(first, rest)) + } else { + Err(E::from((s.loc(), "not a cons".to_string()))) + } + } +} diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index 85741c97f..b728e2443 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -1196,9 +1196,10 @@ fn test_inline_out_of_bounds_diagnostic() { #[test] fn test_defmac_basic_0() { let prog = indoc! {" - (defmac double-arg (A) (list (concat A \"1\") (concat A \"2\"))) - (mod (double-arg X) - (+ X1 X2) + (mod (X) + (defmac double-arg (A) (list (concat A (string->symbol \"1\")) (concat A (string->symbol \"2\")))) + (defun strange (double-arg X) (+ X1 X2)) + (strange X (* 2 X)) ) "} .to_string(); From d88d1740fb2cf90b320b26c232085b1c19db8d4a Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 10 Mar 2023 12:03:18 -0800 Subject: [PATCH 03/81] Working smoke test, many issues sorted out. We should probably delay reifying the macro arguments for defmac invocations until we know the extension wants to handle this call, will do that. --- src/compiler/comptypes.rs | 1 + src/compiler/evaluate.rs | 14 +++- src/compiler/preprocessor.rs | 113 +++++++++++++++++++++++++-------- src/tests/compiler/compiler.rs | 6 +- 4 files changed, 102 insertions(+), 32 deletions(-) diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index ad3015f5c..77c5b5df6 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -579,3 +579,4 @@ pub fn join_vecs_to_string(sep: Vec, vecs: &[Vec]) -> String { decode_string(&s) } + diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 8f4a1cc18..6bdacfdfb 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -703,13 +703,25 @@ impl<'info> Evaluator { let compiled_borrowed: &SExp = compiled.borrow(); Ok(Rc::new(BodyForm::Quoted(compiled_borrowed.clone()))) } else { + let mut converted_args = Vec::new(); + for a in arguments_to_convert.iter() { + let shrunk = self.shrink_bodyform_visited( + allocator, + visited, + prog_args.clone(), + env, + a.clone(), + only_inline, + )?; + converted_args.push(shrunk); + } for ext in self.extensions.iter() { if let Some(res) = ext.try_eval( self, env, &l, call_name, - arguments_to_convert, + &converted_args, body.clone() )? { return Ok(res); diff --git a/src/compiler/preprocessor.rs b/src/compiler/preprocessor.rs index 32b96e98e..b29af42ec 100644 --- a/src/compiler/preprocessor.rs +++ b/src/compiler/preprocessor.rs @@ -20,6 +20,29 @@ struct Preprocessor { helpers: Vec, } +// If the bodyform represents a constant, only match a quoted string. +fn match_quoted_string(body: Rc) -> Result)>, CompileErr> { + if let BodyForm::Quoted(SExp::QuotedString(al,_,an)) = body.borrow() { + Ok(Some((al.clone(),an.clone()))) + } else if let BodyForm::Value(SExp::QuotedString(al,_,an)) = body.borrow() { + Ok(Some((al.clone(),an.clone()))) + } else if let BodyForm::Quoted(_) = body.borrow() { + Err(CompileErr(body.loc(), "string required".to_string())) + } else { + Ok(None) + } +} + +fn match_atom(body: Rc) -> Result)>, CompileErr> { + if let BodyForm::Quoted(SExp::Atom(al,an)) = body.borrow() { + Ok(Some((al.clone(),an.clone()))) + } else if let BodyForm::Quoted(_) = body.borrow() { + Err(CompileErr(body.loc(), "atom required".to_string())) + } else { + Ok(None) + } +} + struct PreprocessorExtension { } impl EvalExtension for PreprocessorExtension { fn try_eval( @@ -36,17 +59,38 @@ impl EvalExtension for PreprocessorExtension { return Err(CompileErr(loc.clone(), "string->symbol needs 1 argument".to_string())); } - eprintln!("args[0] {}", args[0].to_sexp()); + if let Some((loc, value)) = match_quoted_string(args[0].clone())? { - if let BodyForm::Quoted(SExp::QuotedString(al,_,an)) = args[0].borrow() { - return Ok(Some(Rc::new(BodyForm::Quoted(SExp::Atom(al.clone(),an.clone()))))); - } else if let BodyForm::Quoted(x) = args[0].borrow() { - return Err(CompileErr(loc.clone(), "string->symbol takes a string".to_string())); + return Ok(Some(Rc::new(BodyForm::Quoted(SExp::Atom(loc,value))))); + } else { + eprintln!("pp helper returned {}", decode_string(name)); + return Ok(Some(body.clone())); + } + } else if name == b"symbol->string" { + if let Some((loc, value)) = match_atom(args[0].clone())? { + return Ok(Some(Rc::new(BodyForm::Quoted(SExp::QuotedString(loc,b'\"',value))))); } else { + eprintln!("pp helper returned {}", decode_string(name)); return Ok(Some(body.clone())); } + } else if name == b"string-append" { + let mut out_vec = Vec::new(); + let mut out_loc = None; + for a in args.iter() { + if let Some((loc, mut value)) = match_quoted_string(a.clone())? { + if out_loc.is_none() { + out_loc = Some(loc); + } + out_vec.append(&mut value); + } else { + eprintln!("pp helper returned {}", decode_string(name)); + return Ok(Some(body.clone())); + } + } + return Ok(Some(Rc::new(BodyForm::Quoted(SExp::QuotedString(out_loc.unwrap_or_else(|| body.loc()),b'\"',out_vec))))); } + eprintln!("pp helper didn't handle {}", decode_string(name)); Ok(None) } } @@ -130,10 +174,10 @@ impl Preprocessor { // First expand inner macros. let first_expanded = self.expand_macros(f.clone())?; let rest_expanded = self.expand_macros(r.clone())?; - let new_self = SExp::Cons(l.clone(), first_expanded, rest_expanded); + let new_self = Rc::new(SExp::Cons(l.clone(), first_expanded, rest_expanded)); if let Ok(NodeSel::Cons((nl, name), args)) = NodeSel::Cons( Atom::Here(()), ThisNode::Here - ).select_nodes(body.clone()) { + ).select_nodes(new_self.clone()) { // See if it's a form that calls one of our macros. for m in self.helpers.iter() { eprintln!("want {} helper {}", decode_string(&name), m.to_sexp()); @@ -142,6 +186,11 @@ impl Preprocessor { continue; } + eprintln!("expanding macro {}", m.to_sexp()); + for h in self.helpers.iter() { + eprintln!("- {}", decode_string(h.name())); + } + // as inline defuns because they're closest to that // semantically. let mut allocator = Allocator::new(); @@ -165,14 +214,14 @@ impl Preprocessor { None )?; - eprintln!("shrink res {}", res.to_sexp()); - if let Ok(unquoted) = dequote(body.loc(), res) { return Ok(unquoted); } } } } + + return Ok(new_self); } Ok(body) @@ -185,35 +234,43 @@ impl Preprocessor { ) -> Result, CompileErr> { eprintln!("decode_macro {definition}"); if let Ok(NodeSel::Cons( - defmac_loc, + (defmac_loc, kw), NodeSel::Cons( (nl, name), NodeSel::Cons(args,body) ) )) = NodeSel::Cons( - Atom::Here("defmac"), + Atom::Here(()), NodeSel::Cons( Atom::Here(()), NodeSel::Cons(ThisNode::Here, ThisNode::Here) ) ).select_nodes(definition.clone()) { - let target_defun = Rc::new(SExp::Cons( - defmac_loc.clone(), - Rc::new(SExp::atom_from_string(defmac_loc.clone(), "defun")), - Rc::new(SExp::Cons( - nl.clone(), - Rc::new(SExp::Atom(nl.clone(), name.clone())), - Rc::new(SExp::Cons( - args.loc(), args.clone(), body.clone() - )) - )) - )); - eprintln!("target_defun {target_defun}"); - if let Some(helper) = compile_helperform(self.opts.clone(), target_defun)? { - self.evaluator.add_helper(&helper); - self.helpers.push(helper); - } else { - return Err(CompileErr(definition.loc(), "defmac found but couldn't be converted to function".to_string())); + let is_defmac = kw == b"defmac"; + if is_defmac || kw == b"defmacro" || kw == b"defun" || kw == b"defun-inline" || kw == b"defconst" || kw == b"defconstant" { + if is_defmac { + let target_defun = Rc::new(SExp::Cons( + defmac_loc.clone(), + Rc::new(SExp::atom_from_string(defmac_loc.clone(), "defun")), + Rc::new(SExp::Cons( + nl.clone(), + Rc::new(SExp::Atom(nl.clone(), name.clone())), + Rc::new(SExp::Cons( + args.loc(), args.clone(), body.clone() + )) + )) + )); + eprintln!("target_defun {target_defun}"); + if let Some(helper) = compile_helperform(self.opts.clone(), target_defun)? { + eprintln!("add helper {}", helper.to_sexp()); + self.evaluator.add_helper(&helper); + self.helpers.push(helper); + } else { + return Err(CompileErr(definition.loc(), "defmac found but couldn't be converted to function".to_string())); + } + } else if let Some(helper) = compile_helperform(self.opts.clone(), definition)? { + self.evaluator.add_helper(&helper); + } } } diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index b728e2443..9091ff376 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -1197,12 +1197,12 @@ fn test_inline_out_of_bounds_diagnostic() { fn test_defmac_basic_0() { let prog = indoc! {" (mod (X) - (defmac double-arg (A) (list (concat A (string->symbol \"1\")) (concat A (string->symbol \"2\")))) + (defmac double-arg (A) (list (string->symbol (string-append (symbol->string A) \"1\")) (string->symbol (string-append (symbol->string A) \"2\")))) (defun strange (double-arg X) (+ X1 X2)) (strange X (* 2 X)) ) "} .to_string(); - let res = run_string(&prog, &"(3 4)".to_string()).unwrap(); - assert_eq!(res.to_string(), "7"); + let res = run_string(&prog, &"(3)".to_string()).unwrap(); + assert_eq!(res.to_string(), "9"); } From 7f99755ac71668b3e5ad96c053e78e4800b498a4 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 10 Mar 2023 12:15:05 -0800 Subject: [PATCH 04/81] String operations require strings, not other kinds of atoms --- src/tests/compiler/compiler.rs | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index 9091ff376..833d1fe00 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -1202,7 +1202,37 @@ fn test_defmac_basic_0() { (strange X (* 2 X)) ) "} - .to_string(); + .to_string(); let res = run_string(&prog, &"(3)".to_string()).unwrap(); assert_eq!(res.to_string(), "9"); } + +#[test] +fn test_defmac_basic_shared_constant() { + let prog = indoc! {" + (mod (X) + (defconstant twostring \"2\") + (defmac double-arg (A) (list (string->symbol (string-append (symbol->string A) \"1\")) (string->symbol (string-append (symbol->string A) twostring)))) + (defun strange (double-arg X) (+ X1 X2)) + (c twostring (strange X (* 2 X))) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(\"2\" . 9)"); +} + +#[test] +fn test_defmac_basic_shared_constant_not_string_with_string_operator() { + let prog = indoc! {" + (mod (X) + (defconstant twostring 2) + (defmac double-arg (A) (list (string->symbol (string-append (symbol->string A) \"1\")) (string->symbol (string-append (symbol->string A) twostring)))) + (defun strange (double-arg X) (+ X1 X2)) + (c twostring (strange X (* 2 X))) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()); + assert!(res.is_err()); +} From 9698c94ffa3f5abdc76aaff7f06ad6e78bb3f149 Mon Sep 17 00:00:00 2001 From: arty Date: Sun, 12 Mar 2023 14:33:54 -0700 Subject: [PATCH 05/81] Move argument downstream of the extension, making it controllable and moving it out of the main path which is currently well tuned --- src/compiler/evaluate.rs | 16 +++--------- src/compiler/preprocessor.rs | 49 ++++++++++++++++++++++++++++++++++-- 2 files changed, 50 insertions(+), 15 deletions(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 6bdacfdfb..cbbb6faa3 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -74,6 +74,7 @@ pub trait EvalExtension { fn try_eval( &self, evaluator: &Evaluator, + prog_args: Rc, env: &HashMap, Rc>, loc: &Srcloc, name: &[u8], @@ -703,25 +704,14 @@ impl<'info> Evaluator { let compiled_borrowed: &SExp = compiled.borrow(); Ok(Rc::new(BodyForm::Quoted(compiled_borrowed.clone()))) } else { - let mut converted_args = Vec::new(); - for a in arguments_to_convert.iter() { - let shrunk = self.shrink_bodyform_visited( - allocator, - visited, - prog_args.clone(), - env, - a.clone(), - only_inline, - )?; - converted_args.push(shrunk); - } for ext in self.extensions.iter() { if let Some(res) = ext.try_eval( self, + prog_args.clone(), env, &l, call_name, - &converted_args, + &arguments_to_convert, body.clone() )? { return Ok(res); diff --git a/src/compiler/preprocessor.rs b/src/compiler/preprocessor.rs index b29af42ec..165087e6e 100644 --- a/src/compiler/preprocessor.rs +++ b/src/compiler/preprocessor.rs @@ -43,22 +43,53 @@ fn match_atom(body: Rc) -> Result)>, CompileEr } } +fn reify_args( + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + args: &[Rc] +) -> Result>, CompileErr> { + let mut allocator = Allocator::new(); + let mut converted_args = Vec::new(); + for a in args.iter() { + let shrunk = evaluator.shrink_bodyform( + &mut allocator, + prog_args.clone(), + env, + a.clone(), + false, + None + )?; + converted_args.push(shrunk); + } + Ok(converted_args) +} + struct PreprocessorExtension { } impl EvalExtension for PreprocessorExtension { fn try_eval( &self, evaluator: &Evaluator, + prog_args: Rc, env: &HashMap, Rc>, loc: &Srcloc, name: &[u8], - args: &[Rc], + raw_args: &[Rc], body: Rc, ) -> Result>, CompileErr> { if name == b"string->symbol" { - if args.len() != 1 { + if raw_args.len() != 1 { return Err(CompileErr(loc.clone(), "string->symbol needs 1 argument".to_string())); } + let args = reify_args( + evaluator, + prog_args, + env, + loc, + raw_args + )?; if let Some((loc, value)) = match_quoted_string(args[0].clone())? { return Ok(Some(Rc::new(BodyForm::Quoted(SExp::Atom(loc,value))))); @@ -67,6 +98,13 @@ impl EvalExtension for PreprocessorExtension { return Ok(Some(body.clone())); } } else if name == b"symbol->string" { + let args = reify_args( + evaluator, + prog_args, + env, + loc, + raw_args + )?; if let Some((loc, value)) = match_atom(args[0].clone())? { return Ok(Some(Rc::new(BodyForm::Quoted(SExp::QuotedString(loc,b'\"',value))))); } else { @@ -74,6 +112,13 @@ impl EvalExtension for PreprocessorExtension { return Ok(Some(body.clone())); } } else if name == b"string-append" { + let args = reify_args( + evaluator, + prog_args, + env, + loc, + raw_args + )?; let mut out_vec = Vec::new(); let mut out_loc = None; for a in args.iter() { From 1209065af3753916dc1dd028c11931a37cab5226 Mon Sep 17 00:00:00 2001 From: arty Date: Sun, 12 Mar 2023 18:31:33 -0700 Subject: [PATCH 06/81] Break out functions into individual containers, reify arguments in one place --- src/compiler/preprocessor.rs | 198 ++++++++++++++++++++++++++--------- 1 file changed, 147 insertions(+), 51 deletions(-) diff --git a/src/compiler/preprocessor.rs b/src/compiler/preprocessor.rs index 165087e6e..a228ce6c7 100644 --- a/src/compiler/preprocessor.rs +++ b/src/compiler/preprocessor.rs @@ -66,8 +66,8 @@ fn reify_args( Ok(converted_args) } -struct PreprocessorExtension { } -impl EvalExtension for PreprocessorExtension { +/// A function to evaluate in preprocessor macros. +trait ExtensionFunction { fn try_eval( &self, evaluator: &Evaluator, @@ -75,75 +75,171 @@ impl EvalExtension for PreprocessorExtension { env: &HashMap, Rc>, loc: &Srcloc, name: &[u8], - raw_args: &[Rc], + args: &[Rc], body: Rc, - ) -> Result>, CompileErr> { - if name == b"string->symbol" { - if raw_args.len() != 1 { - return Err(CompileErr(loc.clone(), "string->symbol needs 1 argument".to_string())); - } + ) -> Result, CompileErr>; +} - let args = reify_args( - evaluator, - prog_args, - env, - loc, - raw_args - )?; - if let Some((loc, value)) = match_quoted_string(args[0].clone())? { +struct SymbolToString { } + +impl SymbolToString { + fn new() -> Rc { Rc::new(SymbolToString { }) } +} + +impl ExtensionFunction for SymbolToString { + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + if let Some((loc, value)) = match_atom(args[0].clone())? { + return Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString(loc,b'\"',value)))); + } else { + return Ok(body.clone()); + } + } +} + +struct StringToSymbol { } + +impl StringToSymbol { + fn new() -> Rc { Rc::new(StringToSymbol { }) } +} + +impl ExtensionFunction for StringToSymbol { + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + if let Some((loc, value)) = match_quoted_string(args[0].clone())? { + return Ok(Rc::new(BodyForm::Quoted(SExp::Atom(loc,value)))); + } else { + eprintln!("pp helper returned {}", decode_string(name)); + return Ok(body.clone()); + } + } +} + +struct StringAppend { } + +impl StringAppend { + fn new() -> Rc { Rc::new(StringAppend { }) } +} - return Ok(Some(Rc::new(BodyForm::Quoted(SExp::Atom(loc,value))))); +impl ExtensionFunction for StringAppend { + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + let mut out_vec = Vec::new(); + let mut out_loc = None; + for a in args.iter() { + if let Some((loc, mut value)) = match_quoted_string(a.clone())? { + if out_loc.is_none() { + out_loc = Some(loc); + } + out_vec.append(&mut value); } else { eprintln!("pp helper returned {}", decode_string(name)); - return Ok(Some(body.clone())); + return Ok(body.clone()); } - } else if name == b"symbol->string" { + } + return Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString(out_loc.unwrap_or_else(|| body.loc()),b'\"',out_vec)))); + } +} + +/// An evaluator extension for the preprocessor. +/// +/// Implements scheme like conversion functions for handling chialisp programs and +/// bits of them. +/// +/// These functions are provided: +/// +/// Basic conversion +/// +/// string->symbol +/// symbol->string +/// string->number +/// number->string +/// +/// Generate fresh symbols +/// +/// gensym +/// +/// Working with values +/// +/// string-append s0 s1 ... +/// substring s start end +/// first +/// rest +/// cons +struct PreprocessorExtension { + extfuns: HashMap, Rc> +} + +impl PreprocessorExtension { + fn new() -> Self { + let extfuns = [ + (b"string->symbol".to_vec(), StringToSymbol::new()), + (b"symbol->string".to_vec(), SymbolToString::new()), + (b"string-append".to_vec(), StringAppend::new()), + ]; + PreprocessorExtension { extfuns: HashMap::from(extfuns) } + } +} + +impl EvalExtension for PreprocessorExtension { + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + raw_args: &[Rc], + body: Rc, + ) -> Result>, CompileErr> { + if let Some(extfun) = self.extfuns.get(name) { + eprintln!("try function {}", decode_string(name)); let args = reify_args( evaluator, - prog_args, + prog_args.clone(), env, loc, raw_args )?; - if let Some((loc, value)) = match_atom(args[0].clone())? { - return Ok(Some(Rc::new(BodyForm::Quoted(SExp::QuotedString(loc,b'\"',value))))); - } else { - eprintln!("pp helper returned {}", decode_string(name)); - return Ok(Some(body.clone())); - } - } else if name == b"string-append" { - let args = reify_args( + Ok(Some(extfun.try_eval( evaluator, prog_args, env, loc, - raw_args - )?; - let mut out_vec = Vec::new(); - let mut out_loc = None; - for a in args.iter() { - if let Some((loc, mut value)) = match_quoted_string(a.clone())? { - if out_loc.is_none() { - out_loc = Some(loc); - } - out_vec.append(&mut value); - } else { - eprintln!("pp helper returned {}", decode_string(name)); - return Ok(Some(body.clone())); - } - } - return Ok(Some(Rc::new(BodyForm::Quoted(SExp::QuotedString(out_loc.unwrap_or_else(|| body.loc()),b'\"',out_vec))))); + name, + &args, + body + )?)) + } else { + Ok(None) } - - eprintln!("pp helper didn't handle {}", decode_string(name)); - Ok(None) } } -impl PreprocessorExtension { - fn new() -> Self { PreprocessorExtension { } } -} - impl Preprocessor { pub fn new(opts: Rc) -> Self { let runner = Rc::new(DefaultProgramRunner::new()); From d96d8c21faca3c6b53b4df67953cc4bf641fe117 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 13 Mar 2023 08:21:05 -0700 Subject: [PATCH 07/81] Adding predicates and more functions from scheme --- src/compiler/preprocessor/macros.rs | 416 ++++++++++++++++++ .../{preprocessor.rs => preprocessor/mod.rs} | 225 +--------- src/compiler/sexp.rs | 2 +- src/tests/compiler/compiler.rs | 46 +- src/tests/compiler/mod.rs | 1 + src/tests/compiler/preprocessor.rs | 97 ++++ 6 files changed, 520 insertions(+), 267 deletions(-) create mode 100644 src/compiler/preprocessor/macros.rs rename src/compiler/{preprocessor.rs => preprocessor/mod.rs} (67%) create mode 100644 src/tests/compiler/preprocessor.rs diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs new file mode 100644 index 000000000..6af7d5226 --- /dev/null +++ b/src/compiler/preprocessor/macros.rs @@ -0,0 +1,416 @@ +use std::borrow::Borrow; +use std::collections::HashMap; +use std::rc::Rc; + +use clvmr::allocator::Allocator; + +use crate::classic::clvm::__type_compatibility__::bi_one; + +use crate::compiler::comptypes::{BodyForm, CompileErr}; +use crate::compiler::evaluate::{EvalExtension, Evaluator}; +use crate::compiler::sexp::{SExp, decode_string}; +use crate::compiler::srcloc::{Srcloc}; +use crate::util::{Number, number_from_u8}; + +// If the bodyform represents a constant, only match a quoted string. +fn match_quoted_string(body: Rc) -> Result)>, CompileErr> { + if let BodyForm::Quoted(SExp::QuotedString(al,_,an)) = body.borrow() { + Ok(Some((al.clone(),an.clone()))) + } else if let BodyForm::Value(SExp::QuotedString(al,_,an)) = body.borrow() { + Ok(Some((al.clone(),an.clone()))) + } else if let BodyForm::Quoted(_) = body.borrow() { + Err(CompileErr(body.loc(), "string required".to_string())) + } else { + Ok(None) + } +} + +fn match_atom(body: Rc) -> Result)>, CompileErr> { + if let BodyForm::Quoted(SExp::Atom(al,an)) = body.borrow() { + Ok(Some((al.clone(),an.clone()))) + } else if let BodyForm::Quoted(_) = body.borrow() { + Err(CompileErr(body.loc(), "atom required".to_string())) + } else { + Ok(None) + } +} + +enum MatchedNumber { + MatchedInt(Srcloc, Number), + MatchedHex(Srcloc, Vec) +} + +fn match_number(body: Rc) -> Result, CompileErr> { + if let BodyForm::Quoted(SExp::Integer(il,n)) = body.borrow() { + Ok(Some(MatchedNumber::MatchedInt(il.clone(), n.clone()))) + } else if let BodyForm::Quoted(SExp::QuotedString(ql,b'x',b)) = body.borrow() { + Ok(Some(MatchedNumber::MatchedHex(ql.clone(), b.clone()))) + } else if let BodyForm::Quoted(_) = body.borrow() { + Err(CompileErr(body.loc(), "number required".to_string())) + } else { + Ok(None) + } +} + +fn reify_args( + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + args: &[Rc] +) -> Result>, CompileErr> { + let mut allocator = Allocator::new(); + let mut converted_args = Vec::new(); + for a in args.iter() { + let shrunk = evaluator.shrink_bodyform( + &mut allocator, + prog_args.clone(), + env, + a.clone(), + false, + None + )?; + converted_args.push(shrunk); + } + Ok(converted_args) +} + +/// A container for a function to evaluate in advanced preprocessor macros. +/// We use this trait (which is very similar to the extension trait in Evaluator) +/// as a definite handler for a specific named form, so optional returns aren't +/// needed. These are held in a collection and looked up. To be maximally +/// conservative with typing and lifetime, we hold these via Rc. +pub trait ExtensionFunction { + fn required_args(&self) -> Option; + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr>; +} + +struct StringQ { } + +impl StringQ { + fn new() -> Rc { Rc::new(StringQ { }) } +} + +impl ExtensionFunction for StringQ { + fn required_args(&self) -> Option { Some(1) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + let res = + match match_quoted_string(args[0].clone()) { + Ok(Some(_)) => { + SExp::Integer(loc.clone(), bi_one()) + } + Ok(None) => { + return Ok(body.clone()); + } + Err(_) => { + SExp::Nil(loc.clone()) + } + }; + + Ok(Rc::new(BodyForm::Quoted(res))) + } +} + +struct NumberQ { } + +impl NumberQ { + fn new() -> Rc { Rc::new(NumberQ { }) } +} + +impl ExtensionFunction for NumberQ { + fn required_args(&self) -> Option { Some(1) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + todo!(); + } +} + +struct SymbolQ { } + +impl SymbolQ { + fn new() -> Rc { Rc::new(SymbolQ { }) } +} + +impl ExtensionFunction for SymbolQ { + fn required_args(&self) -> Option { Some(1) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + todo!(); + } +} + +struct SymbolToString { } + +impl SymbolToString { + fn new() -> Rc { Rc::new(SymbolToString { }) } +} + +impl ExtensionFunction for SymbolToString { + fn required_args(&self) -> Option { Some(1) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + if let Some((loc, value)) = match_atom(args[0].clone())? { + return Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString(loc,b'\"',value)))); + } else { + return Ok(body.clone()); + } + } +} + +struct StringToSymbol { } + +impl StringToSymbol { + fn new() -> Rc { Rc::new(StringToSymbol { }) } +} + +impl ExtensionFunction for StringToSymbol { + fn required_args(&self) -> Option { Some(1) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + if let Some((loc, value)) = match_quoted_string(args[0].clone())? { + return Ok(Rc::new(BodyForm::Quoted(SExp::Atom(loc,value)))); + } else { + eprintln!("pp helper returned {}", decode_string(name)); + return Ok(body.clone()); + } + } +} + +struct StringAppend { } + +impl StringAppend { + fn new() -> Rc { Rc::new(StringAppend { }) } +} + +impl ExtensionFunction for StringAppend { + fn required_args(&self) -> Option { None } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + let mut out_vec = Vec::new(); + let mut out_loc = None; + for a in args.iter() { + if let Some((loc, mut value)) = match_quoted_string(a.clone())? { + if out_loc.is_none() { + out_loc = Some(loc); + } + out_vec.append(&mut value); + } else { + eprintln!("pp helper returned {}", decode_string(name)); + return Ok(body.clone()); + } + } + return Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString(out_loc.unwrap_or_else(|| body.loc()),b'\"',out_vec)))); + } +} + +struct NumberToString { } + +impl NumberToString { + fn new() -> Rc { Rc::new(NumberToString { }) } +} + +impl ExtensionFunction for NumberToString { + fn required_args(&self) -> Option { Some(1) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + let match_res = match_number(args[0].clone())?; + let (use_loc, int_val) = + match &match_res { + Some(MatchedNumber::MatchedInt(l,i)) => { (l.clone(), i.clone()) } + Some(MatchedNumber::MatchedHex(l,h)) => { + (l.clone(), number_from_u8(&h)) + } + _ => { + return Ok(body.clone()); + } + }; + Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString(use_loc, b'\"', int_val.to_string().as_bytes().to_vec())))) + } +} + +struct StringToNumber { } + +impl StringToNumber { + fn new() -> Rc { Rc::new(StringToNumber { }) } +} + +impl ExtensionFunction for StringToNumber { + fn required_args(&self) -> Option { Some(1) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + todo!(); + } +} + +/// An evaluator extension for the preprocessor. +/// +/// Implements scheme like conversion functions for handling chialisp programs and +/// bits of them. +/// +/// These functions are provided: +/// +/// Queries +/// +/// pair? +/// string? +/// number? +/// symbol? +/// +/// Basic conversion +/// +/// string->symbol +/// symbol->string +/// string->number +/// number->string +/// +/// Working with values +/// +/// string-append s0 s1 ... +/// substring s start end +/// first +/// rest +/// cons +pub struct PreprocessorExtension { + extfuns: HashMap, Rc> +} + +impl PreprocessorExtension { + pub fn new() -> Self { + let extfuns = [ + (b"string?".to_vec(), StringQ::new()), + (b"number?".to_vec(), NumberQ::new()), + (b"symbol?".to_vec(), SymbolQ::new()), + + (b"string->symbol".to_vec(), StringToSymbol::new()), + (b"symbol->string".to_vec(), SymbolToString::new()), + (b"string->number".to_vec(), StringToNumber::new()), + (b"number->string".to_vec(), NumberToString::new()), + + (b"string-append".to_vec(), StringAppend::new()), + ]; + PreprocessorExtension { extfuns: HashMap::from(extfuns) } + } +} + +impl EvalExtension for PreprocessorExtension { + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + raw_args: &[Rc], + body: Rc, + ) -> Result>, CompileErr> { + if let Some(extfun) = self.extfuns.get(name) { + if let Some(n) = extfun.required_args() { + if raw_args.len() != n { + return Err(CompileErr(loc.clone(), format!("{} requires {} args", decode_string(name), n))); + } + } + + eprintln!("try function {}", decode_string(name)); + let args = reify_args( + evaluator, + prog_args.clone(), + env, + loc, + raw_args + )?; + Ok(Some(extfun.try_eval( + evaluator, + prog_args, + env, + loc, + name, + &args, + body + )?)) + } else { + Ok(None) + } + } +} diff --git a/src/compiler/preprocessor.rs b/src/compiler/preprocessor/mod.rs similarity index 67% rename from src/compiler/preprocessor.rs rename to src/compiler/preprocessor/mod.rs index a228ce6c7..831851348 100644 --- a/src/compiler/preprocessor.rs +++ b/src/compiler/preprocessor/mod.rs @@ -1,3 +1,5 @@ +mod macros; + use std::borrow::Borrow; use std::collections::HashMap; use std::rc::Rc; @@ -11,8 +13,9 @@ use crate::compiler::comptypes::{BodyForm, CompileErr, CompilerOpts, HelperForm, use crate::compiler::evaluate::{ArgInputs, create_argument_captures, dequote, EvalExtension, Evaluator}; use crate::compiler::frontend::compile_helperform; use crate::compiler::sexp::{Atom, decode_string, enlist, First, NodeSel, parse_sexp, SelectNode, SExp, ThisNode}; +use crate::compiler::preprocessor::macros::PreprocessorExtension; use crate::compiler::srcloc::Srcloc; -use crate::util::ErrInto; +use crate::util::{ErrInto, Number, number_from_u8}; struct Preprocessor { opts: Rc, @@ -20,226 +23,6 @@ struct Preprocessor { helpers: Vec, } -// If the bodyform represents a constant, only match a quoted string. -fn match_quoted_string(body: Rc) -> Result)>, CompileErr> { - if let BodyForm::Quoted(SExp::QuotedString(al,_,an)) = body.borrow() { - Ok(Some((al.clone(),an.clone()))) - } else if let BodyForm::Value(SExp::QuotedString(al,_,an)) = body.borrow() { - Ok(Some((al.clone(),an.clone()))) - } else if let BodyForm::Quoted(_) = body.borrow() { - Err(CompileErr(body.loc(), "string required".to_string())) - } else { - Ok(None) - } -} - -fn match_atom(body: Rc) -> Result)>, CompileErr> { - if let BodyForm::Quoted(SExp::Atom(al,an)) = body.borrow() { - Ok(Some((al.clone(),an.clone()))) - } else if let BodyForm::Quoted(_) = body.borrow() { - Err(CompileErr(body.loc(), "atom required".to_string())) - } else { - Ok(None) - } -} - -fn reify_args( - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - args: &[Rc] -) -> Result>, CompileErr> { - let mut allocator = Allocator::new(); - let mut converted_args = Vec::new(); - for a in args.iter() { - let shrunk = evaluator.shrink_bodyform( - &mut allocator, - prog_args.clone(), - env, - a.clone(), - false, - None - )?; - converted_args.push(shrunk); - } - Ok(converted_args) -} - -/// A function to evaluate in preprocessor macros. -trait ExtensionFunction { - fn try_eval( - &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr>; -} - -struct SymbolToString { } - -impl SymbolToString { - fn new() -> Rc { Rc::new(SymbolToString { }) } -} - -impl ExtensionFunction for SymbolToString { - fn try_eval( - &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { - if let Some((loc, value)) = match_atom(args[0].clone())? { - return Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString(loc,b'\"',value)))); - } else { - return Ok(body.clone()); - } - } -} - -struct StringToSymbol { } - -impl StringToSymbol { - fn new() -> Rc { Rc::new(StringToSymbol { }) } -} - -impl ExtensionFunction for StringToSymbol { - fn try_eval( - &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { - if let Some((loc, value)) = match_quoted_string(args[0].clone())? { - return Ok(Rc::new(BodyForm::Quoted(SExp::Atom(loc,value)))); - } else { - eprintln!("pp helper returned {}", decode_string(name)); - return Ok(body.clone()); - } - } -} - -struct StringAppend { } - -impl StringAppend { - fn new() -> Rc { Rc::new(StringAppend { }) } -} - -impl ExtensionFunction for StringAppend { - fn try_eval( - &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { - let mut out_vec = Vec::new(); - let mut out_loc = None; - for a in args.iter() { - if let Some((loc, mut value)) = match_quoted_string(a.clone())? { - if out_loc.is_none() { - out_loc = Some(loc); - } - out_vec.append(&mut value); - } else { - eprintln!("pp helper returned {}", decode_string(name)); - return Ok(body.clone()); - } - } - return Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString(out_loc.unwrap_or_else(|| body.loc()),b'\"',out_vec)))); - } -} - -/// An evaluator extension for the preprocessor. -/// -/// Implements scheme like conversion functions for handling chialisp programs and -/// bits of them. -/// -/// These functions are provided: -/// -/// Basic conversion -/// -/// string->symbol -/// symbol->string -/// string->number -/// number->string -/// -/// Generate fresh symbols -/// -/// gensym -/// -/// Working with values -/// -/// string-append s0 s1 ... -/// substring s start end -/// first -/// rest -/// cons -struct PreprocessorExtension { - extfuns: HashMap, Rc> -} - -impl PreprocessorExtension { - fn new() -> Self { - let extfuns = [ - (b"string->symbol".to_vec(), StringToSymbol::new()), - (b"symbol->string".to_vec(), SymbolToString::new()), - (b"string-append".to_vec(), StringAppend::new()), - ]; - PreprocessorExtension { extfuns: HashMap::from(extfuns) } - } -} - -impl EvalExtension for PreprocessorExtension { - fn try_eval( - &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], - raw_args: &[Rc], - body: Rc, - ) -> Result>, CompileErr> { - if let Some(extfun) = self.extfuns.get(name) { - eprintln!("try function {}", decode_string(name)); - let args = reify_args( - evaluator, - prog_args.clone(), - env, - loc, - raw_args - )?; - Ok(Some(extfun.try_eval( - evaluator, - prog_args, - env, - loc, - name, - &args, - body - )?)) - } else { - Ok(None) - } - } -} - impl Preprocessor { pub fn new(opts: Rc) -> Self { let runner = Rc::new(DefaultProgramRunner::new()); diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index f593754a3..d01104085 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -264,7 +264,7 @@ fn normalize_int(v: Vec, base: u32) -> Number { fn from_hex(l: Srcloc, v: &[u8]) -> SExp { let mut result = vec![0; (v.len() - 2) / 2]; hex2bin(&v[2..], &mut result).expect("should convert from hex"); - SExp::QuotedString(l, b'"', result) + SExp::QuotedString(l, b'x', result) } fn make_atom(l: Srcloc, v: Vec) -> SExp { diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index 833d1fe00..e4883706b 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -58,7 +58,7 @@ fn run_string_maybe_opt( }) } -fn run_string(content: &String, args: &String) -> Result, CompileErr> { +pub fn run_string(content: &String, args: &String) -> Result, CompileErr> { run_string_maybe_opt(content, args, false) } @@ -1192,47 +1192,3 @@ fn test_inline_out_of_bounds_diagnostic() { assert!(false); } } - -#[test] -fn test_defmac_basic_0() { - let prog = indoc! {" - (mod (X) - (defmac double-arg (A) (list (string->symbol (string-append (symbol->string A) \"1\")) (string->symbol (string-append (symbol->string A) \"2\")))) - (defun strange (double-arg X) (+ X1 X2)) - (strange X (* 2 X)) - ) - "} - .to_string(); - let res = run_string(&prog, &"(3)".to_string()).unwrap(); - assert_eq!(res.to_string(), "9"); -} - -#[test] -fn test_defmac_basic_shared_constant() { - let prog = indoc! {" - (mod (X) - (defconstant twostring \"2\") - (defmac double-arg (A) (list (string->symbol (string-append (symbol->string A) \"1\")) (string->symbol (string-append (symbol->string A) twostring)))) - (defun strange (double-arg X) (+ X1 X2)) - (c twostring (strange X (* 2 X))) - ) - "} - .to_string(); - let res = run_string(&prog, &"(3)".to_string()).unwrap(); - assert_eq!(res.to_string(), "(\"2\" . 9)"); -} - -#[test] -fn test_defmac_basic_shared_constant_not_string_with_string_operator() { - let prog = indoc! {" - (mod (X) - (defconstant twostring 2) - (defmac double-arg (A) (list (string->symbol (string-append (symbol->string A) \"1\")) (string->symbol (string-append (symbol->string A) twostring)))) - (defun strange (double-arg X) (+ X1 X2)) - (c twostring (strange X (* 2 X))) - ) - "} - .to_string(); - let res = run_string(&prog, &"(3)".to_string()); - assert!(res.is_err()); -} diff --git a/src/tests/compiler/mod.rs b/src/tests/compiler/mod.rs index 09b9beff2..dfb8e6edc 100644 --- a/src/tests/compiler/mod.rs +++ b/src/tests/compiler/mod.rs @@ -7,6 +7,7 @@ use crate::compiler::srcloc::{Srcloc, Until}; mod clvm; mod compiler; mod evaluate; +mod preprocessor; mod repl; mod srcloc; mod usecheck; diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs new file mode 100644 index 000000000..6605e4f9a --- /dev/null +++ b/src/tests/compiler/preprocessor.rs @@ -0,0 +1,97 @@ +use crate::tests::compiler::compiler::run_string; + +#[test] +fn test_defmac_basic_0() { + let prog = indoc! {" + (mod (X) + (defmac double-arg (A) (list (string->symbol (string-append (symbol->string A) \"1\")) (string->symbol (string-append (symbol->string A) \"2\")))) + (defun strange (double-arg X) (+ X1 X2)) + (strange X (* 2 X)) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()).unwrap(); + assert_eq!(res.to_string(), "9"); +} + +#[test] +fn test_defmac_basic_shared_constant() { + let prog = indoc! {" + (mod (X) + (defconstant twostring \"2\") + (defmac double-arg (A) (list (string->symbol (string-append (symbol->string A) \"1\")) (string->symbol (string-append (symbol->string A) twostring)))) + (defun strange (double-arg X) (+ X1 X2)) + (c twostring (strange X (* 2 X))) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(\"2\" . 9)"); +} + +#[test] +fn test_defmac_basic_shared_constant_not_string_with_string_operator() { + let prog = indoc! {" + (mod (X) + (defconstant twostring 2) + (defmac double-arg (A) (list (string->symbol (string-append (symbol->string A) \"1\")) (string->symbol (string-append (symbol->string A) twostring)))) + (defun strange (double-arg X) (+ X1 X2)) + (c twostring (strange X (* 2 X))) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()); + assert!(res.is_err()); +} + +#[test] +fn test_defmac_basic_shared_constant_not_string_with_string_operator_fun() { + let prog = indoc! {" + (mod (X) + (defconstant twostring \"2\") + (defun make-arg-list (A) (list (string->symbol (string-append (symbol->string A) \"1\")) (string->symbol (string-append (symbol->string A) twostring)))) + (defmac double-arg (A) (make-arg-list A)) + (defun strange (double-arg X) (+ X1 X2)) + (c twostring (strange X (* 2 X))) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(\"2\" . 9)"); +} + +#[test] +fn test_defmac_basic_test_is_string_pos() { + let prog = indoc! {" + (mod (X) + (defmac classify (S) + (if (string? S) + (qq (c 1 (unquote S))) + (qq (c 2 (unquote S))) + ) + ) + (c X (classify \"test\")) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(3 1 . \"test\")"); +} + +#[test] +fn test_defmac_basic_test_is_string_neg() { + let prog = indoc! {" + (mod (X) + (defmac classify (S) + (if (string? S) + (qq (c 1 (unquote S))) + (qq (c 2 (unquote S))) + ) + ) + (c X (classify test)) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(3 2 . test)"); +} From 7b65fe624a897ee52b0b93fe60a8f9ecb17f63eb Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 13 Mar 2023 08:38:10 -0700 Subject: [PATCH 08/81] More tests --- src/compiler/preprocessor/macros.rs | 62 +++++++++++++++++++++---- src/tests/compiler/preprocessor.rs | 72 +++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 9 deletions(-) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 6af7d5226..490244e77 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -14,14 +14,32 @@ use crate::util::{Number, number_from_u8}; // If the bodyform represents a constant, only match a quoted string. fn match_quoted_string(body: Rc) -> Result)>, CompileErr> { - if let BodyForm::Quoted(SExp::QuotedString(al,_,an)) = body.borrow() { - Ok(Some((al.clone(),an.clone()))) - } else if let BodyForm::Value(SExp::QuotedString(al,_,an)) = body.borrow() { - Ok(Some((al.clone(),an.clone()))) - } else if let BodyForm::Quoted(_) = body.borrow() { - Err(CompileErr(body.loc(), "string required".to_string())) + let is_string = + match body.borrow() { + BodyForm::Quoted(SExp::QuotedString(al,b'x',an)) => { + None + } + BodyForm::Quoted(SExp::QuotedString(al,_,an)) => { + Some((al.clone(), an.clone())) + } + BodyForm::Value(SExp::QuotedString(al,b'x',an)) => { + None + } + BodyForm::Value(SExp::QuotedString(al,_,an)) => { + Some((al.clone(), an.clone())) + } + BodyForm::Quoted(_) => { + None + } + _ => { + return Ok(None); + } + }; + + if let Some((loc, s)) = is_string { + Ok(Some((loc, s))) } else { - Ok(None) + Err(CompileErr(body.loc(), "string required".to_string())) } } @@ -149,7 +167,20 @@ impl ExtensionFunction for NumberQ { args: &[Rc], body: Rc, ) -> Result, CompileErr> { - todo!(); + let res = + match match_number(args[0].clone()) { + Ok(Some(_)) => { + SExp::Integer(loc.clone(), bi_one()) + } + Ok(None) => { + return Ok(body.clone()); + } + Err(_) => { + SExp::Nil(loc.clone()) + } + }; + + Ok(Rc::new(BodyForm::Quoted(res))) } } @@ -172,7 +203,20 @@ impl ExtensionFunction for SymbolQ { args: &[Rc], body: Rc, ) -> Result, CompileErr> { - todo!(); + let res = + match match_atom(args[0].clone()) { + Ok(Some(_)) => { + SExp::Integer(loc.clone(), bi_one()) + } + Ok(None) => { + return Ok(body.clone()); + } + Err(_) => { + SExp::Nil(loc.clone()) + } + }; + + Ok(Rc::new(BodyForm::Quoted(res))) } } diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 6605e4f9a..45bd15f6c 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -95,3 +95,75 @@ fn test_defmac_basic_test_is_string_neg() { let res = run_string(&prog, &"(3)".to_string()).unwrap(); assert_eq!(res.to_string(), "(3 2 . test)"); } + +#[test] +fn test_defmac_basic_test_is_symbol_pos() { + let prog = indoc! {" + (mod (X) + (defmac classify (S) + (if (symbol? S) + (qq (c 1 (unquote S))) + (qq (c 2 (unquote S))) + ) + ) + (c X (classify test)) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(3 1 . test)"); +} + +#[test] +fn test_defmac_basic_test_is_symbol_neg() { + let prog = indoc! {" + (mod (X) + (defmac classify (S) + (if (symbol? S) + (qq (c 1 (unquote S))) + (qq (c 2 (unquote S))) + ) + ) + (c X (classify \"test\")) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(3 2 . \"test\")"); +} + +#[test] +fn test_defmac_basic_test_is_number_pos() { + let prog = indoc! {" + (mod (X) + (defmac classify (S) + (if (number? S) + (qq (c 1 (unquote S))) + (qq (c 2 (unquote S))) + ) + ) + (c X (classify 7)) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(3 1 . 7)"); +} + +#[test] +fn test_defmac_basic_test_is_number_neg() { + let prog = indoc! {" + (mod (X) + (defmac classify (S) + (if (number? S) + (qq (c 1 (unquote S))) + (qq (c 2 (unquote S))) + ) + ) + (c X (classify \"test\")) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(3 2 . \"test\")"); +} From 397f481aa31b2d9facf65a60def7654621af5ad3 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 13 Mar 2023 13:37:15 -0700 Subject: [PATCH 09/81] WIP, getting there --- src/compiler/preprocessor/macros.rs | 24 +++++++++ src/tests/compiler/preprocessor.rs | 79 +++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 490244e77..5e23bfde2 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -368,6 +368,29 @@ impl ExtensionFunction for StringToNumber { } } +struct Substring { } + +impl Substring { + fn new() -> Rc { Rc::new(Substring { }) } +} + +impl ExtensionFunction for Substring { + fn required_args(&self) -> Option { Some(3) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + todo!(); + } +} + /// An evaluator extension for the preprocessor. /// /// Implements scheme like conversion functions for handling chialisp programs and @@ -413,6 +436,7 @@ impl PreprocessorExtension { (b"number->string".to_vec(), NumberToString::new()), (b"string-append".to_vec(), StringAppend::new()), + (b"substring".to_vec(), Substring::new()), ]; PreprocessorExtension { extfuns: HashMap::from(extfuns) } } diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 45bd15f6c..a4ff430fc 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -167,3 +167,82 @@ fn test_defmac_basic_test_is_number_neg() { let res = run_string(&prog, &"(3)".to_string()).unwrap(); assert_eq!(res.to_string(), "(3 2 . \"test\")"); } + +#[test] +fn test_defmac_create_cond_form() { + let prog = indoc! {" + ;; Make a simple list match ala ocaml/elm/haskell. + ;; + ;; The real version will be more elaborate. This is a test case and a demo. + (mod (X) + (defun list-nth (L N) + (if N + (list-nth (r L) (- N 1)) + (f L) + ) + ) + + (defun list-len (L N) + (if L + (list-len (r L) (+ N 1)) + N + ) + ) + + ;; From a list of bindings or constants, build a matcher for the elements. + (defun gather-bindings-list (expr tomatch) + (if (not expr) + () + () + ) + ) + + (defun gather-bindings (condition expr rest) + (qq + (if (= (list-len (unquote expr)) (unquote (list-len condition))) + (unquote + (let* + ( + (bindings-and-checks (gather-bindings-and-checks condition expr)) + (bindings (f bindings-and-checks)) + (checks (r bindings-and-checks)) + ) + (qq (if checks (let bindings rest))) + ) + ) + () + ) + ) + ) + + ;; For each pair, emit an expression that checks for the requested pattern + ;; and returns either the requested result or allows the following matchers + ;; to run. + (defun handle-matches (expr matches) + (if (not matches) + (qq (x (unquote expr))) + (let + ( + (condition (f (f expr))) + (code (f (r (f expr)))) + ) + (qq (if (unquote (make-match-expr condition expr)) (unquote code) (unquote (handle-matches (r matches))))) + ) + ) + ) + + ;; match as below. + (defmac match (expr . matches) (handle-matches expr matches)) + + (match X + ((16 x y) (c 1 (+ x y))) + ((3 () b c) c) + ((3 (q . 1) b c) b) + (x x) + ) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3 () 1001 1002)".to_string()).unwrap(); + assert_eq!(res.to_string(), "1002"); +} From cd148b7a98e79efcddbafd84b42ebdc00bb976c0 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 14 Mar 2023 14:39:35 -0700 Subject: [PATCH 10/81] WIP, getting there --- src/compiler/evaluate.rs | 28 +++--- src/compiler/preprocessor/macros.rs | 137 ++++++++++++++++++++++--- src/compiler/preprocessor/mod.rs | 6 +- src/tests/compiler/preprocessor.rs | 148 +++++++++++++++++++++------- 4 files changed, 258 insertions(+), 61 deletions(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index cbbb6faa3..7d53e8a2c 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -704,20 +704,6 @@ impl<'info> Evaluator { let compiled_borrowed: &SExp = compiled.borrow(); Ok(Rc::new(BodyForm::Quoted(compiled_borrowed.clone()))) } else { - for ext in self.extensions.iter() { - if let Some(res) = ext.try_eval( - self, - prog_args.clone(), - env, - &l, - call_name, - &arguments_to_convert, - body.clone() - )? { - return Ok(res); - } - } - let pres = self .lookup_prim(l.clone(), call_name) .map(|prim| { @@ -910,6 +896,20 @@ impl<'info> Evaluator { env: &HashMap, Rc>, only_inline: bool, ) -> Result, CompileErr> { + for ext in self.extensions.iter() { + if let Some(res) = ext.try_eval( + self, + prog_args.clone(), + env, + &l, + call_name, + &arguments_to_convert, + body.clone() + )? { + return Ok(res); + } + } + let helper = select_helper(&self.helpers, call_name); match helper { Some(HelperForm::Defmacro(mac)) => self.invoke_macro_expansion( diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 5e23bfde2..8aea3ccfc 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -6,8 +6,10 @@ use clvmr::allocator::Allocator; use crate::classic::clvm::__type_compatibility__::bi_one; +use crate::compiler::clvm::truthy; use crate::compiler::comptypes::{BodyForm, CompileErr}; use crate::compiler::evaluate::{EvalExtension, Evaluator}; +use crate::compiler::preprocessor::dequote; use crate::compiler::sexp::{SExp, decode_string}; use crate::compiler::srcloc::{Srcloc}; use crate::util::{Number, number_from_u8}; @@ -80,6 +82,10 @@ fn reify_args( let mut allocator = Allocator::new(); let mut converted_args = Vec::new(); for a in args.iter() { + eprintln!("shrink {}", a.to_sexp()); + for (n,e) in env.iter() { + eprintln!("- {} = {}", decode_string(&n), e.to_sexp()); + } let shrunk = evaluator.shrink_bodyform( &mut allocator, prog_args.clone(), @@ -99,6 +105,7 @@ fn reify_args( /// needed. These are held in a collection and looked up. To be maximally /// conservative with typing and lifetime, we hold these via Rc. pub trait ExtensionFunction { + fn want_interp(&self) -> bool { true } fn required_args(&self) -> Option; fn try_eval( &self, @@ -391,6 +398,109 @@ impl ExtensionFunction for Substring { } } +struct List { } + +impl List { + fn new() -> Rc { Rc::new(List { }) } +} + +impl ExtensionFunction for List { + fn required_args(&self) -> Option { None } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + let mut res = SExp::Nil(loc.clone()); + for (n,e) in env.iter() { + eprintln!("- {} = {}", decode_string(&n), e.to_sexp()); + } + for a in args.iter().rev() { + eprintln!("list arg {}", a.to_sexp()); + if let Ok(unquoted) = dequote(loc.clone(), a.clone()) { + res = SExp::Cons( + loc.clone(), + unquoted, + Rc::new(res) + ); + } else { + return Ok(body.clone()); + } + } + let list_res = BodyForm::Quoted(res); + eprintln!("list_res {}", list_res.to_sexp()); + Ok(Rc::new(list_res)) + } +} + +struct If { } + +impl If { + fn new() -> Rc { Rc::new(If { }) } +} + +impl ExtensionFunction for If { + fn want_interp(&self) -> bool { false } + + fn required_args(&self) -> Option { Some(3) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + let mut allocator = Allocator::new(); + let cond_result = + evaluator.shrink_bodyform( + &mut allocator, + prog_args.clone(), + env, + args[0].clone(), + false, + None + )?; + + if let Ok(unquoted) = dequote(body.loc(), cond_result) { + eprintln!("unquoted {}", unquoted); + if truthy(unquoted) { + eprintln!("truthy, expand {}", args[1].to_sexp()); + evaluator.shrink_bodyform( + &mut allocator, + prog_args.clone(), + env, + args[1].clone(), + false, + None + ) + } else { + eprintln!("falsey, expand {}", args[2].to_sexp()); + evaluator.shrink_bodyform( + &mut allocator, + prog_args.clone(), + env, + args[2].clone(), + false, + None + ) + } + } else { + eprintln!("can't reduce if {}", body.to_sexp()); + Ok(body.clone()) + } + } +} + /// An evaluator extension for the preprocessor. /// /// Implements scheme like conversion functions for handling chialisp programs and @@ -416,9 +526,6 @@ impl ExtensionFunction for Substring { /// /// string-append s0 s1 ... /// substring s start end -/// first -/// rest -/// cons pub struct PreprocessorExtension { extfuns: HashMap, Rc> } @@ -426,6 +533,9 @@ pub struct PreprocessorExtension { impl PreprocessorExtension { pub fn new() -> Self { let extfuns = [ + (b"if".to_vec(), If::new()), + (b"list".to_vec(), List::new()), + (b"string?".to_vec(), StringQ::new()), (b"number?".to_vec(), NumberQ::new()), (b"symbol?".to_vec(), SymbolQ::new()), @@ -460,14 +570,19 @@ impl EvalExtension for PreprocessorExtension { } } - eprintln!("try function {}", decode_string(name)); - let args = reify_args( - evaluator, - prog_args.clone(), - env, - loc, - raw_args - )?; + eprintln!("try function {}", body.to_sexp()); + let args = + if extfun.want_interp() { + reify_args( + evaluator, + prog_args.clone(), + env, + loc, + raw_args + )? + } else { + raw_args.to_vec() + }; Ok(Some(extfun.try_eval( evaluator, prog_args, diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 831851348..3cc3960ed 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -138,8 +138,12 @@ impl Preprocessor { None )?; - if let Ok(unquoted) = dequote(body.loc(), res) { + if let Ok(unquoted) = dequote(body.loc(), res.clone()) { + eprintln!("expand macro {}", unquoted); return Ok(unquoted); + } else { + eprintln!("bad expand? {}", res.to_sexp()); + todo!(); } } } diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index a4ff430fc..3787b67c7 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -169,7 +169,34 @@ fn test_defmac_basic_test_is_number_neg() { } #[test] -fn test_defmac_create_cond_form() { +fn test_defmac_extension_from_function() { + let prog = indoc! {" + (mod (X) + (defun FX (X) (symbol->string X)) + (defmac F (X) (FX X)) + (c 3 (F X)) + ) + "} + .to_string(); + let res = run_string(&prog, &"(3)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(3 . \"X\")"); +} + +#[test] +fn test_defmac_if_extension() { + let prog = indoc! {" + (mod (X) + (defun FX (X) (if X (number->string 1) 2)) + (defmac F (X) (c 1 (FX X))) + (F X) + ) + "}.to_string(); + let res = run_string(&prog, &"(9)".to_string()).unwrap(); + assert_eq!(res.to_string(), "\"1\""); +} + +#[test] +fn test_defmac_create_match_form() { let prog = indoc! {" ;; Make a simple list match ala ocaml/elm/haskell. ;; @@ -189,50 +216,101 @@ fn test_defmac_create_cond_form() { ) ) - ;; From a list of bindings or constants, build a matcher for the elements. - (defun gather-bindings-list (expr tomatch) - (if (not expr) - () - () + (defun funcall (name args) (c (string->symbol name) args)) + (defun quoted (X) (c 1 X)) + (defun equals (A B) (funcall \"=\" (list A B))) + (defun emit-list-nth (L N) (funcall \"list-nth\" (list L N))) + (defun emit-list-len (L N) (funcall \"list-len\" (list L N))) + (defun emit-if (C T E) (funcall \"if\" (list C T E))) + (defun emit-let (bindings body) (funcall \"let\" (list bindings body))) + + ;; Determine the size of each list as well as the constants and bindings + ;; Return either a list of (number-of-elts matches bindings) or symbol. + (defun build-matches-and-bindings (n pattern matches bindings) + (if (not pattern) + (list n matches bindings) + (if (l pattern) + (if (symbol? (f pattern)) + (build-matches-and-bindings (+ n 1) (r pattern) matches (c (c n (f pattern)) bindings)) + (build-matches-and-bindings (+ n 1) (r pattern) (c (c n (f pattern)) matches) bindings) + ) + pattern + ) + ) ) - ) - (defun gather-bindings (condition expr rest) - (qq - (if (= (list-len (unquote expr)) (unquote (list-len condition))) - (unquote - (let* - ( - (bindings-and-checks (gather-bindings-and-checks condition expr)) - (bindings (f bindings-and-checks)) - (checks (r bindings-and-checks)) - ) - (qq (if checks (let bindings rest))) + ;; Emit code that matches each match list for length and constants. + (defun write-match-code (expr matches) + (if (not matches) + (quoted 1) + (if (l (f matches)) + (let* + ( + (offset (f (f matches))) + (desire (r (f matches))) + (this-match (equals (quoted desire) (emit-list-nth expr (quoted offset)))) + ) + (if (not (r matches)) + (list this-match) + (c this-match (write-match-code expr (r matches))) + ) + ) + (quoted 1) + ) + ) + ) + + ;; Generate let bindings for the bindings indicated in the match. + (defun let-bindings (expr bindings) + (if (not bindings) + () + (let + ((n (f (f bindings))) + (binding (r (f bindings))) + ) + (c (list binding (emit-list-nth expr n)) (let-bindings expr (r bindings))) ) ) - () - ) ) - ) - ;; For each pair, emit an expression that checks for the requested pattern - ;; and returns either the requested result or allows the following matchers - ;; to run. - (defun handle-matches (expr matches) - (if (not matches) - (qq (x (unquote expr))) - (let - ( - (condition (f (f expr))) - (code (f (r (f expr)))) + ;; Generate if expressions that match the indicates matches and return + ;; the indicated code with bindings installed. + (defun match-if (expr clauses) + (if (not clauses) + (list 8) + (let + ((extracted-clause-data (build-matches-and-bindings 0 (f (f clauses)) () ())) + (code (f (r (f clauses)))) + ) + (if (l extracted-clause-data) + (let + ( + (number-of-elts (f extracted-clause-data)) + (matches (list-nth extracted-clause-data 1)) + (bindings (list-nth extracted-clause-data 2)) + ) + (emit-if + (emit-if + (equals (emit-list-len list expr) (quoted number-of-elts)) + ;; then + (list (c (string->symbol \"logand\") (write-match-code expr matches))) + ;; else + () + ) + ;; then + (emit-let (let-bindings expr bindings) code) + ;; else + (match-if expr (r clauses)) + ) + ) + (emit-let (list (list extracted-clause-data expr)) code) + ) + ) ) - (qq (if (unquote (make-match-expr condition expr)) (unquote code) (unquote (handle-matches (r matches))))) - ) ) - ) ;; match as below. - (defmac match (expr . matches) (handle-matches expr matches)) + (defmac match (expr . matches) (match-if expr matches)) (match X ((16 x y) (c 1 (+ x y))) From 4ad9bd2315c647e3fe8f3cc3a2c834c065a3eb8e Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 20 Mar 2023 14:18:01 -0700 Subject: [PATCH 11/81] Working big example --- src/compiler/preprocessor/macros.rs | 138 ++++++++++++++++++++++------ src/tests/compiler/preprocessor.rs | 7 +- 2 files changed, 116 insertions(+), 29 deletions(-) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 8aea3ccfc..827a3b041 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -82,10 +82,6 @@ fn reify_args( let mut allocator = Allocator::new(); let mut converted_args = Vec::new(); for a in args.iter() { - eprintln!("shrink {}", a.to_sexp()); - for (n,e) in env.iter() { - eprintln!("- {} = {}", decode_string(&n), e.to_sexp()); - } let shrunk = evaluator.shrink_bodyform( &mut allocator, prog_args.clone(), @@ -276,7 +272,7 @@ impl ExtensionFunction for StringToSymbol { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { return Ok(Rc::new(BodyForm::Quoted(SExp::Atom(loc,value)))); } else { - eprintln!("pp helper returned {}", decode_string(name)); + eprintln!("pp helper {} returned {} given {}", decode_string(name), body.to_sexp(), args[0].to_sexp()); return Ok(body.clone()); } } @@ -417,25 +413,113 @@ impl ExtensionFunction for List { args: &[Rc], body: Rc, ) -> Result, CompileErr> { - let mut res = SExp::Nil(loc.clone()); - for (n,e) in env.iter() { - eprintln!("- {} = {}", decode_string(&n), e.to_sexp()); - } + let mut allocator = Allocator::new(); + let mut res = Rc::new(BodyForm::Quoted(SExp::Nil(loc.clone()))); for a in args.iter().rev() { - eprintln!("list arg {}", a.to_sexp()); - if let Ok(unquoted) = dequote(loc.clone(), a.clone()) { - res = SExp::Cons( - loc.clone(), - unquoted, - Rc::new(res) - ); - } else { - return Ok(body.clone()); - } + res = Rc::new(BodyForm::Call( + loc.clone(), + vec![ + Rc::new(BodyForm::Value(SExp::Atom(loc.clone(), b"c".to_vec()))), + a.clone(), + res + ] + )); + } + evaluator.shrink_bodyform( + &mut allocator, + prog_args.clone(), + env, + res, + false, + None + ) + } +} + +struct Cons { } + +impl Cons { + fn new() -> Rc { Rc::new(Cons { }) } +} + +impl ExtensionFunction for Cons { + fn required_args(&self) -> Option { Some(2) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + if let (BodyForm::Quoted(a), BodyForm::Quoted(b)) = (args[0].borrow(), args[1].borrow()) { + Ok(Rc::new(BodyForm::Quoted(SExp::Cons(loc.clone(), Rc::new(a.clone()), Rc::new(b.clone()))))) + } else { + Ok(body.clone()) + } + } +} + +struct First { } + +impl First { + fn new() -> Rc { Rc::new(First { }) } +} + +impl ExtensionFunction for First { + fn required_args(&self) -> Option { Some(1) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + if let BodyForm::Quoted(SExp::Cons(_,a,b)) = args[0].borrow() { + let a_borrowed: &SExp = a.borrow(); + Ok(Rc::new(BodyForm::Quoted(a_borrowed.clone()))) + } else if let BodyForm::Quoted(_) = args[0].borrow() { + Err(CompileErr(loc.clone(), "bad cons in first".to_string())) + } else { + Ok(body.clone()) + } + } +} + +struct Rest { } + +impl Rest { + fn new() -> Rc { Rc::new(Rest { }) } +} + +impl ExtensionFunction for Rest { + fn required_args(&self) -> Option { Some(1) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + if let BodyForm::Quoted(SExp::Cons(_,a,b)) = args[0].borrow() { + let a_borrowed: &SExp = b.borrow(); + Ok(Rc::new(BodyForm::Quoted(a_borrowed.clone()))) + } else if let BodyForm::Quoted(_) = args[0].borrow() { + Err(CompileErr(loc.clone(), "bad cons in rest".to_string())) + } else { + Ok(body.clone()) } - let list_res = BodyForm::Quoted(res); - eprintln!("list_res {}", list_res.to_sexp()); - Ok(Rc::new(list_res)) } } @@ -472,9 +556,7 @@ impl ExtensionFunction for If { )?; if let Ok(unquoted) = dequote(body.loc(), cond_result) { - eprintln!("unquoted {}", unquoted); if truthy(unquoted) { - eprintln!("truthy, expand {}", args[1].to_sexp()); evaluator.shrink_bodyform( &mut allocator, prog_args.clone(), @@ -484,7 +566,6 @@ impl ExtensionFunction for If { None ) } else { - eprintln!("falsey, expand {}", args[2].to_sexp()); evaluator.shrink_bodyform( &mut allocator, prog_args.clone(), @@ -495,7 +576,6 @@ impl ExtensionFunction for If { ) } } else { - eprintln!("can't reduce if {}", body.to_sexp()); Ok(body.clone()) } } @@ -535,6 +615,9 @@ impl PreprocessorExtension { let extfuns = [ (b"if".to_vec(), If::new()), (b"list".to_vec(), List::new()), + (b"c".to_vec(), Cons::new()), + (b"f".to_vec(), First::new()), + (b"r".to_vec(), Rest::new()), (b"string?".to_vec(), StringQ::new()), (b"number?".to_vec(), NumberQ::new()), @@ -571,6 +654,9 @@ impl EvalExtension for PreprocessorExtension { } eprintln!("try function {}", body.to_sexp()); + for (n,v) in env.iter() { + eprintln!("- {} = {}", decode_string(&n), v.to_sexp()); + } let args = if extfun.want_interp() { reify_args( diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 3787b67c7..6abfa29c8 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -201,7 +201,8 @@ fn test_defmac_create_match_form() { ;; Make a simple list match ala ocaml/elm/haskell. ;; ;; The real version will be more elaborate. This is a test case and a demo. - (mod (X) + (mod X + (include *standard-cl-21*) (defun list-nth (L N) (if N (list-nth (r L) (- N 1)) @@ -291,9 +292,9 @@ fn test_defmac_create_match_form() { ) (emit-if (emit-if - (equals (emit-list-len list expr) (quoted number-of-elts)) + (equals (emit-list-len expr 0) (quoted number-of-elts)) ;; then - (list (c (string->symbol \"logand\") (write-match-code expr matches))) + (c (string->symbol \"logand\") (write-match-code expr matches)) ;; else () ) From 5e15b2b6feeefc3366513706ca950baad0e1b865 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 21 Mar 2023 09:29:17 -0700 Subject: [PATCH 12/81] Adding tests of added things --- src/tests/compiler/preprocessor.rs | 72 ++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 6abfa29c8..5ece97456 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -325,3 +325,75 @@ fn test_defmac_create_match_form() { let res = run_string(&prog, &"(3 () 1001 1002)".to_string()).unwrap(); assert_eq!(res.to_string(), "1002"); } + +#[test] +fn test_defmac_stringq() { + let prog = indoc! {" + (mod () + (defmac is-string (X) (string? X)) + (list (is-string X) (is-string \"X\") (is-string 3)) + ) + "}.to_string(); + let res = run_string(&prog, &"()".to_string()).unwrap(); + assert_eq!(res.to_string(), "(() 1 ())"); +} + +#[test] +fn test_defmac_numberq() { + let prog = indoc! {" + (mod () + (defmac is-number (X) (number? X)) + (list (is-number X) (is-number \"X\") (is-number 3)) + ) + "}.to_string(); + let res = run_string(&prog, &"()".to_string()).unwrap(); + assert_eq!(res.to_string(), "(() () 1)"); +} + +#[test] +fn test_defmac_symbolq() { + let prog = indoc! {" + (mod () + (defmac is-symbol (X) (symbol? X)) + (list (is-symbol X) (is-symbol \"X\") (is-symbol 3)) + ) + "}.to_string(); + let res = run_string(&prog, &"()".to_string()).unwrap(); + assert_eq!(res.to_string(), "(1 () ())"); +} + +#[test] +fn test_defmac_string_to_symbol() { + let prog = indoc! {" + (mod () + (defmac is-symbol (X) (symbol? X)) + (list (is-symbol X) (is-symbol \"X\") (is-symbol 3)) + ) + "}.to_string(); + let res = run_string(&prog, &"()".to_string()).unwrap(); + assert_eq!(res.to_string(), "(1 () ())"); +} + +#[test] +fn test_defmac_string_to_symbol_converts() { + let prog = indoc! {" + (mod (X) + (defmac let_pi (code) (qq (let (((unquote (string->symbol \"pi\")) 31415)) (unquote code)))) + (let_pi (+ pi X)) + ) + "}.to_string(); + let res = run_string(&prog, &"(5)".to_string()).unwrap(); + assert_eq!(res.to_string(), "31420"); +} + +#[test] +fn test_defmac_string_needs_conversion() { + let prog = indoc! {" + (mod (X) + (defmac let_pi (code) (qq (let ((\"pi\" 31415)) (unquote code)))) + (let_pi (+ pi X)) + ) + "}.to_string(); + let res = run_string(&prog, &"(5)".to_string()); + assert!(res.is_err()); +} From 3a1431c4f54fb34731351d62d812a526d321011d Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 21 Mar 2023 10:19:12 -0700 Subject: [PATCH 13/81] Finish substring, more tests --- src/compiler/preprocessor/macros.rs | 60 ++++++++++++++++++++++++----- src/tests/compiler/preprocessor.rs | 16 ++++++++ 2 files changed, 67 insertions(+), 9 deletions(-) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 827a3b041..b0b45ab36 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -3,8 +3,9 @@ use std::collections::HashMap; use std::rc::Rc; use clvmr::allocator::Allocator; +use num_traits::ToPrimitive; -use crate::classic::clvm::__type_compatibility__::bi_one; +use crate::classic::clvm::__type_compatibility__::{bi_one, bi_zero}; use crate::compiler::clvm::truthy; use crate::compiler::comptypes::{BodyForm, CompileErr}; @@ -61,14 +62,37 @@ enum MatchedNumber { } fn match_number(body: Rc) -> Result, CompileErr> { - if let BodyForm::Quoted(SExp::Integer(il,n)) = body.borrow() { - Ok(Some(MatchedNumber::MatchedInt(il.clone(), n.clone()))) - } else if let BodyForm::Quoted(SExp::QuotedString(ql,b'x',b)) = body.borrow() { - Ok(Some(MatchedNumber::MatchedHex(ql.clone(), b.clone()))) - } else if let BodyForm::Quoted(_) = body.borrow() { - Err(CompileErr(body.loc(), "number required".to_string())) + match body.borrow() { + BodyForm::Quoted(SExp::Integer(il,n)) => { + Ok(Some(MatchedNumber::MatchedInt(il.clone(), n.clone()))) + } + BodyForm::Quoted(SExp::QuotedString(ql,b'x',b)) => { + Ok(Some(MatchedNumber::MatchedHex(ql.clone(), b.clone()))) + } + BodyForm::Quoted(SExp::Nil(il)) => { + Ok(Some(MatchedNumber::MatchedInt(il.clone(), bi_zero()))) + } + BodyForm::Quoted(_) => { + Err(CompileErr(body.loc(), "number required".to_string())) + } + _ => Ok(None) + } +} + +fn numeric_value(body: Rc) -> Result { + match match_number(body.clone())? { + Some(MatchedNumber::MatchedInt(_, n)) => Ok(n.clone()), + Some(MatchedNumber::MatchedHex(_, h)) => Ok(number_from_u8(&h)), + _ => Err(CompileErr(body.loc(), "Not a number".to_string())) + } +} + +fn usize_value(body: Rc) -> Result { + let n = numeric_value(body.clone())?; + if let Some(res) = n.to_usize() { + Ok(res) } else { - Ok(None) + Err(CompileErr(body.loc(), "Value out of range".to_string())) } } @@ -390,7 +414,25 @@ impl ExtensionFunction for Substring { args: &[Rc], body: Rc, ) -> Result, CompileErr> { - todo!(); + let start_element = usize_value(args[1].clone())?; + let end_element = usize_value(args[2].clone())?; + + match args[0].borrow() { + BodyForm::Quoted(SExp::QuotedString(l,ch,s)) => { + if start_element > end_element || start_element > s.len() || end_element > s.len() { + return Err(CompileErr(l.clone(), "start greater than end in substring".to_string())); + } + let result_value: Vec = + s.iter().take(end_element).skip(start_element).copied().collect(); + Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString(l.clone(), *ch, result_value)))) + } + BodyForm::Quoted(_) => { + Err(CompileErr(body.loc(), "Not a string".to_string())) + } + _ => { + Ok(body.clone()) + } + } } } diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 5ece97456..286ace546 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -397,3 +397,19 @@ fn test_defmac_string_needs_conversion() { let res = run_string(&prog, &"(5)".to_string()); assert!(res.is_err()); } + +#[test] +fn test_defmac_string_substr_0() { + let prog = indoc! {" + (mod (X) + (defmac first-letter-of (Q) + (let ((first-character (substring (symbol->string Q) 0 1))) + (qq (c (unquote first-character) (unquote (string->symbol first-character)))) + ) + ) + (first-letter-of Xanadu) + ) + "}.to_string(); + let res = run_string(&prog, &"(5999)".to_string()).unwrap(); + assert_eq!(res.to_string(), "(\"X\" . 5999)"); +} From 514ca1bcf56c3c14f3bada0614c3412eb488aaa7 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 21 Mar 2023 12:27:10 -0700 Subject: [PATCH 14/81] More --- src/compiler/preprocessor/macros.rs | 35 +++++++++++++++++++++++++++++ src/tests/compiler/preprocessor.rs | 19 ++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index b0b45ab36..2d94b4553 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::rc::Rc; use clvmr::allocator::Allocator; +use num_bigint::ToBigInt; use num_traits::ToPrimitive; use crate::classic::clvm::__type_compatibility__::{bi_one, bi_zero}; @@ -395,6 +396,39 @@ impl ExtensionFunction for StringToNumber { } } +struct StringLength { } + +impl StringLength { + fn new() -> Rc { Rc::new(StringLength { }) } +} + +impl ExtensionFunction for StringLength { + fn required_args(&self) -> Option { Some(1) } + + fn try_eval( + &self, + evaluator: &Evaluator, + prog_args: Rc, + env: &HashMap, Rc>, + loc: &Srcloc, + name: &[u8], + args: &[Rc], + body: Rc, + ) -> Result, CompileErr> { + if let Some((loc, mut value)) = match_quoted_string(args[0].clone())? { + if let Some(len_bi) = value.len().to_bigint() { + Ok(Rc::new(BodyForm::Quoted(SExp::Integer(loc.clone(), len_bi)))) + } else { + Err(CompileErr(loc.clone(), "Error getting string length".to_string())) + } + } else { + eprintln!("pp helper returned {}", decode_string(name)); + Ok(body.clone()) + } + } +} + + struct Substring { } impl Substring { @@ -671,6 +705,7 @@ impl PreprocessorExtension { (b"number->string".to_vec(), NumberToString::new()), (b"string-append".to_vec(), StringAppend::new()), + (b"string-length".to_vec(), StringLength::new()), (b"substring".to_vec(), Substring::new()), ]; PreprocessorExtension { extfuns: HashMap::from(extfuns) } diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 286ace546..ad48f9898 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -413,3 +413,22 @@ fn test_defmac_string_substr_0() { let res = run_string(&prog, &"(5999)".to_string()).unwrap(); assert_eq!(res.to_string(), "(\"X\" . 5999)"); } + +#[test] +fn test_defmac_string_substr_1() { + let prog = indoc! {" + (mod (test_variable_name) + (defmac bind-tail-of-symbol (N Q CODE) + (let* + ((stringified (symbol->string Q)) + (slen (string-length stringified)) + (suffix (string->symbol (substring stringified N slen)))) + (qq (let (((unquote suffix) (r (unquote Q)))) (unquote CODE))) + ) + ) + (bind-tail-of-symbol 5 test_variable_name (c 9999 variable_name)) + ) + "}.to_string(); + let res = run_string(&prog, &"((87 89 91))".to_string()).unwrap(); + assert_eq!(res.to_string(), "(9999 89 91)"); +} From 51bce1b625d6dbdb7793d1ecd25d61d6588452c6 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 21 Mar 2023 15:09:13 -0700 Subject: [PATCH 15/81] more --- src/compiler/preprocessor/macros.rs | 10 ++++- src/tests/compiler/preprocessor.rs | 67 +++++++++++++++++++++++++++-- 2 files changed, 72 insertions(+), 5 deletions(-) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 2d94b4553..ac1c68add 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -392,7 +392,15 @@ impl ExtensionFunction for StringToNumber { args: &[Rc], body: Rc, ) -> Result, CompileErr> { - todo!(); + if let Some((loc, mut value)) = match_quoted_string(args[0].clone())? { + if let Ok(cvt_bi) = decode_string(&value).parse::() { + Ok(Rc::new(BodyForm::Quoted(SExp::Integer(loc.clone(), cvt_bi)))) + } else { + Err(CompileErr(loc.clone(), "bad number".to_string())) + } + } else { + Err(CompileErr(loc.clone(), "should be given a string".to_string())) + } } } diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index ad48f9898..40495c8df 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -415,7 +415,7 @@ fn test_defmac_string_substr_0() { } #[test] -fn test_defmac_string_substr_1() { +fn test_defmac_string_substr_bad() { let prog = indoc! {" (mod (test_variable_name) (defmac bind-tail-of-symbol (N Q CODE) @@ -426,9 +426,68 @@ fn test_defmac_string_substr_1() { (qq (let (((unquote suffix) (r (unquote Q)))) (unquote CODE))) ) ) - (bind-tail-of-symbol 5 test_variable_name (c 9999 variable_name)) + (bind-tail-of-symbol 100 test_variable_name (c 9999 variable_name)) ) "}.to_string(); - let res = run_string(&prog, &"((87 89 91))".to_string()).unwrap(); - assert_eq!(res.to_string(), "(9999 89 91)"); + let res = run_string(&prog, &"((87 89 91))".to_string()); + assert!(res.is_err()); +} + +#[test] +fn test_defmac_string_to_number_0() { + let prog = indoc! {" + (mod (X_7) + (defmac add-n-to (X) + (let + ((stringified (symbol->string X)) + (slen (string-length stringified)) + (number-part (substring stringified (- slen 1) slen)) + (numeric-value (string->number number-part))) + (qq (+ (unquote numeric-value) (unquote X))) + ) + ) + (add-n-to X_7) + ) + "}.to_string(); + let res = run_string(&prog, &"(31)".to_string()).unwrap(); + assert_eq!(res.to_string(), "38"); +} + +#[test] +fn test_defmac_string_to_number_bad() { + let prog = indoc! {" + (mod (X_A) + (defmac add-n-to (X) + (let + ((stringified (symbol->string X)) + (slen (string-length stringified)) + (number-part (substring stringified (- slen 1) slen)) + (numeric-value (string->number number-part))) + (qq (+ (unquote numeric-value) (unquote X))) + ) + ) + (add-n-to X_A) + ) + "}.to_string(); + let res = run_string(&prog, &"(31)".to_string()); + assert!(res.is_err()); +} + +#[test] +fn test_defmac_number_to_string() { + let prog = indoc! {" + (mod (Q) + (defmac with-my-length (X) + (let* + ((stringified (symbol->string X)) + (slen (string-length stringified))) + (string->symbol (string-append stringified \"-\" (number->string slen))) + ) + ) + (defun F (Xanadu-6) (+ (with-my-length Xanadu) 99)) + (F Q) + ) + "}.to_string(); + let res = run_string(&prog, &"(37)".to_string()).unwrap(); + assert_eq!(res.to_string(), "136"); } From dfcc9c664c04001ee4043ac6ce6c52f3f0e0bd42 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 21 Mar 2023 15:56:54 -0700 Subject: [PATCH 16/81] clippy --- src/compiler/preprocessor/macros.rs | 154 ++++++++++++++-------------- src/compiler/preprocessor/mod.rs | 8 +- 2 files changed, 83 insertions(+), 79 deletions(-) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index ac1c68add..76d622424 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -20,13 +20,13 @@ use crate::util::{Number, number_from_u8}; fn match_quoted_string(body: Rc) -> Result)>, CompileErr> { let is_string = match body.borrow() { - BodyForm::Quoted(SExp::QuotedString(al,b'x',an)) => { + BodyForm::Quoted(SExp::QuotedString(_,b'x',_)) => { None } BodyForm::Quoted(SExp::QuotedString(al,_,an)) => { Some((al.clone(), an.clone())) } - BodyForm::Value(SExp::QuotedString(al,b'x',an)) => { + BodyForm::Value(SExp::QuotedString(_,b'x',_)) => { None } BodyForm::Value(SExp::QuotedString(al,_,an)) => { @@ -101,7 +101,6 @@ fn reify_args( evaluator: &Evaluator, prog_args: Rc, env: &HashMap, Rc>, - loc: &Srcloc, args: &[Rc] ) -> Result>, CompileErr> { let mut allocator = Allocator::new(); @@ -151,11 +150,11 @@ impl ExtensionFunction for StringQ { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, loc: &Srcloc, - name: &[u8], + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { @@ -187,11 +186,11 @@ impl ExtensionFunction for NumberQ { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, loc: &Srcloc, - name: &[u8], + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { @@ -223,11 +222,11 @@ impl ExtensionFunction for SymbolQ { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, loc: &Srcloc, - name: &[u8], + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { @@ -259,11 +258,11 @@ impl ExtensionFunction for SymbolToString { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, + _loc: &Srcloc, + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { @@ -286,18 +285,17 @@ impl ExtensionFunction for StringToSymbol { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, + _loc: &Srcloc, + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { return Ok(Rc::new(BodyForm::Quoted(SExp::Atom(loc,value)))); } else { - eprintln!("pp helper {} returned {} given {}", decode_string(name), body.to_sexp(), args[0].to_sexp()); return Ok(body.clone()); } } @@ -314,11 +312,11 @@ impl ExtensionFunction for StringAppend { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, + _loc: &Srcloc, + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { @@ -331,7 +329,6 @@ impl ExtensionFunction for StringAppend { } out_vec.append(&mut value); } else { - eprintln!("pp helper returned {}", decode_string(name)); return Ok(body.clone()); } } @@ -350,11 +347,11 @@ impl ExtensionFunction for NumberToString { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, + _loc: &Srcloc, + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { @@ -384,15 +381,15 @@ impl ExtensionFunction for StringToNumber { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, loc: &Srcloc, - name: &[u8], + _name: &[u8], args: &[Rc], - body: Rc, + _body: Rc, ) -> Result, CompileErr> { - if let Some((loc, mut value)) = match_quoted_string(args[0].clone())? { + if let Some((loc, value)) = match_quoted_string(args[0].clone())? { if let Ok(cvt_bi) = decode_string(&value).parse::() { Ok(Rc::new(BodyForm::Quoted(SExp::Integer(loc.clone(), cvt_bi)))) } else { @@ -415,22 +412,21 @@ impl ExtensionFunction for StringLength { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, + _loc: &Srcloc, + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { - if let Some((loc, mut value)) = match_quoted_string(args[0].clone())? { + if let Some((loc, value)) = match_quoted_string(args[0].clone())? { if let Some(len_bi) = value.len().to_bigint() { Ok(Rc::new(BodyForm::Quoted(SExp::Integer(loc.clone(), len_bi)))) } else { Err(CompileErr(loc.clone(), "Error getting string length".to_string())) } } else { - eprintln!("pp helper returned {}", decode_string(name)); Ok(body.clone()) } } @@ -448,11 +444,11 @@ impl ExtensionFunction for Substring { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, + _loc: &Srcloc, + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { @@ -493,9 +489,9 @@ impl ExtensionFunction for List { prog_args: Rc, env: &HashMap, Rc>, loc: &Srcloc, - name: &[u8], + _name: &[u8], args: &[Rc], - body: Rc, + _body: Rc, ) -> Result, CompileErr> { let mut allocator = Allocator::new(); let mut res = Rc::new(BodyForm::Quoted(SExp::Nil(loc.clone()))); @@ -531,11 +527,11 @@ impl ExtensionFunction for Cons { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, loc: &Srcloc, - name: &[u8], + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { @@ -558,15 +554,15 @@ impl ExtensionFunction for First { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, loc: &Srcloc, - name: &[u8], + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { - if let BodyForm::Quoted(SExp::Cons(_,a,b)) = args[0].borrow() { + if let BodyForm::Quoted(SExp::Cons(_,a,_)) = args[0].borrow() { let a_borrowed: &SExp = a.borrow(); Ok(Rc::new(BodyForm::Quoted(a_borrowed.clone()))) } else if let BodyForm::Quoted(_) = args[0].borrow() { @@ -588,15 +584,15 @@ impl ExtensionFunction for Rest { fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, + _evaluator: &Evaluator, + _prog_args: Rc, + _env: &HashMap, Rc>, loc: &Srcloc, - name: &[u8], + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { - if let BodyForm::Quoted(SExp::Cons(_,a,b)) = args[0].borrow() { + if let BodyForm::Quoted(SExp::Cons(_,_,b)) = args[0].borrow() { let a_borrowed: &SExp = b.borrow(); Ok(Rc::new(BodyForm::Quoted(a_borrowed.clone()))) } else if let BodyForm::Quoted(_) = args[0].borrow() { @@ -623,8 +619,8 @@ impl ExtensionFunction for If { evaluator: &Evaluator, prog_args: Rc, env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], + _loc: &Srcloc, + _name: &[u8], args: &[Rc], body: Rc, ) -> Result, CompileErr> { @@ -672,9 +668,16 @@ impl ExtensionFunction for If { /// /// These functions are provided: /// +/// Enhanced versions of builtin macros: +/// +/// if -- first class short circuiting, no round trip to CLVM space +/// list -- simple own implementation +/// c -- cons preserving exact input values +/// f -- first and rest exactly preserving part of cons. +/// r -- +/// /// Queries /// -/// pair? /// string? /// number? /// symbol? @@ -689,7 +692,9 @@ impl ExtensionFunction for If { /// Working with values /// /// string-append s0 s1 ... +/// string-length /// substring s start end +/// pub struct PreprocessorExtension { extfuns: HashMap, Rc> } @@ -748,7 +753,6 @@ impl EvalExtension for PreprocessorExtension { evaluator, prog_args.clone(), env, - loc, raw_args )? } else { diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index a24c31d95..2ddb1358d 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -10,12 +10,12 @@ use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; use crate::compiler::compiler::KNOWN_DIALECTS; use crate::compiler::comptypes::{BodyForm, CompileErr, CompilerOpts, HelperForm, IncludeDesc}; -use crate::compiler::evaluate::{ArgInputs, create_argument_captures, dequote, EvalExtension, Evaluator}; +use crate::compiler::evaluate::{ArgInputs, create_argument_captures, dequote, Evaluator}; use crate::compiler::frontend::compile_helperform; -use crate::compiler::sexp::{Atom, decode_string, enlist, First, NodeSel, parse_sexp, SelectNode, SExp, ThisNode}; +use crate::compiler::sexp::{Atom, decode_string, enlist, NodeSel, parse_sexp, SelectNode, SExp, ThisNode}; use crate::compiler::preprocessor::macros::PreprocessorExtension; use crate::compiler::srcloc::Srcloc; -use crate::util::{ErrInto, Number, number_from_u8}; +use crate::util::ErrInto; struct Preprocessor { opts: Rc, @@ -101,7 +101,7 @@ impl Preprocessor { let first_expanded = self.expand_macros(f.clone())?; let rest_expanded = self.expand_macros(r.clone())?; let new_self = Rc::new(SExp::Cons(l.clone(), first_expanded, rest_expanded)); - if let Ok(NodeSel::Cons((nl, name), args)) = NodeSel::Cons( + if let Ok(NodeSel::Cons((_, name), args)) = NodeSel::Cons( Atom::Here(()), ThisNode::Here ).select_nodes(new_self.clone()) { // See if it's a form that calls one of our macros. From 4b1b14bff10105b9da1fb8c9cbe847a06f36460e Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 22 Mar 2023 09:12:49 -0700 Subject: [PATCH 17/81] fmt --- src/compiler/comptypes.rs | 1 - src/compiler/evaluate.rs | 4 +- src/compiler/preprocessor/macros.rs | 443 +++++++++++++++------------- src/compiler/preprocessor/mod.rs | 80 ++--- src/compiler/sexp.rs | 9 +- src/tests/compiler/preprocessor.rs | 36 ++- 6 files changed, 316 insertions(+), 257 deletions(-) diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index a36ba2c76..c23454172 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -755,4 +755,3 @@ pub fn join_vecs_to_string(sep: Vec, vecs: &[Vec]) -> String { decode_string(&s) } - diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index a7e2b1f55..f5fcf4ef4 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -628,7 +628,7 @@ impl<'info> Evaluator { helpers, mash_conditions: false, ignore_exn: false, - extensions: Vec::new() + extensions: Vec::new(), } } @@ -944,7 +944,7 @@ impl<'info> Evaluator { &l, call_name, &arguments_to_convert, - body.clone() + body.clone(), )? { return Ok(res); } diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 76d622424..f04f0db36 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -12,33 +12,22 @@ use crate::compiler::clvm::truthy; use crate::compiler::comptypes::{BodyForm, CompileErr}; use crate::compiler::evaluate::{EvalExtension, Evaluator}; use crate::compiler::preprocessor::dequote; -use crate::compiler::sexp::{SExp, decode_string}; -use crate::compiler::srcloc::{Srcloc}; -use crate::util::{Number, number_from_u8}; +use crate::compiler::sexp::{decode_string, SExp}; +use crate::compiler::srcloc::Srcloc; +use crate::util::{number_from_u8, Number}; // If the bodyform represents a constant, only match a quoted string. fn match_quoted_string(body: Rc) -> Result)>, CompileErr> { - let is_string = - match body.borrow() { - BodyForm::Quoted(SExp::QuotedString(_,b'x',_)) => { - None - } - BodyForm::Quoted(SExp::QuotedString(al,_,an)) => { - Some((al.clone(), an.clone())) - } - BodyForm::Value(SExp::QuotedString(_,b'x',_)) => { - None - } - BodyForm::Value(SExp::QuotedString(al,_,an)) => { - Some((al.clone(), an.clone())) - } - BodyForm::Quoted(_) => { - None - } - _ => { - return Ok(None); - } - }; + let is_string = match body.borrow() { + BodyForm::Quoted(SExp::QuotedString(_, b'x', _)) => None, + BodyForm::Quoted(SExp::QuotedString(al, _, an)) => Some((al.clone(), an.clone())), + BodyForm::Value(SExp::QuotedString(_, b'x', _)) => None, + BodyForm::Value(SExp::QuotedString(al, _, an)) => Some((al.clone(), an.clone())), + BodyForm::Quoted(_) => None, + _ => { + return Ok(None); + } + }; if let Some((loc, s)) = is_string { Ok(Some((loc, s))) @@ -48,8 +37,8 @@ fn match_quoted_string(body: Rc) -> Result)>, } fn match_atom(body: Rc) -> Result)>, CompileErr> { - if let BodyForm::Quoted(SExp::Atom(al,an)) = body.borrow() { - Ok(Some((al.clone(),an.clone()))) + if let BodyForm::Quoted(SExp::Atom(al, an)) = body.borrow() { + Ok(Some((al.clone(), an.clone()))) } else if let BodyForm::Quoted(_) = body.borrow() { Err(CompileErr(body.loc(), "atom required".to_string())) } else { @@ -59,24 +48,22 @@ fn match_atom(body: Rc) -> Result)>, CompileEr enum MatchedNumber { MatchedInt(Srcloc, Number), - MatchedHex(Srcloc, Vec) + MatchedHex(Srcloc, Vec), } fn match_number(body: Rc) -> Result, CompileErr> { match body.borrow() { - BodyForm::Quoted(SExp::Integer(il,n)) => { + BodyForm::Quoted(SExp::Integer(il, n)) => { Ok(Some(MatchedNumber::MatchedInt(il.clone(), n.clone()))) } - BodyForm::Quoted(SExp::QuotedString(ql,b'x',b)) => { + BodyForm::Quoted(SExp::QuotedString(ql, b'x', b)) => { Ok(Some(MatchedNumber::MatchedHex(ql.clone(), b.clone()))) } BodyForm::Quoted(SExp::Nil(il)) => { Ok(Some(MatchedNumber::MatchedInt(il.clone(), bi_zero()))) } - BodyForm::Quoted(_) => { - Err(CompileErr(body.loc(), "number required".to_string())) - } - _ => Ok(None) + BodyForm::Quoted(_) => Err(CompileErr(body.loc(), "number required".to_string())), + _ => Ok(None), } } @@ -84,7 +71,7 @@ fn numeric_value(body: Rc) -> Result { match match_number(body.clone())? { Some(MatchedNumber::MatchedInt(_, n)) => Ok(n.clone()), Some(MatchedNumber::MatchedHex(_, h)) => Ok(number_from_u8(&h)), - _ => Err(CompileErr(body.loc(), "Not a number".to_string())) + _ => Err(CompileErr(body.loc(), "Not a number".to_string())), } } @@ -101,7 +88,7 @@ fn reify_args( evaluator: &Evaluator, prog_args: Rc, env: &HashMap, Rc>, - args: &[Rc] + args: &[Rc], ) -> Result>, CompileErr> { let mut allocator = Allocator::new(); let mut converted_args = Vec::new(); @@ -112,7 +99,7 @@ fn reify_args( env, a.clone(), false, - None + None, )?; converted_args.push(shrunk); } @@ -125,7 +112,9 @@ fn reify_args( /// needed. These are held in a collection and looked up. To be maximally /// conservative with typing and lifetime, we hold these via Rc. pub trait ExtensionFunction { - fn want_interp(&self) -> bool { true } + fn want_interp(&self) -> bool { + true + } fn required_args(&self) -> Option; fn try_eval( &self, @@ -139,14 +128,18 @@ pub trait ExtensionFunction { ) -> Result, CompileErr>; } -struct StringQ { } +struct StringQ {} impl StringQ { - fn new() -> Rc { Rc::new(StringQ { }) } + fn new() -> Rc { + Rc::new(StringQ {}) + } } impl ExtensionFunction for StringQ { - fn required_args(&self) -> Option { Some(1) } + fn required_args(&self) -> Option { + Some(1) + } fn try_eval( &self, @@ -158,31 +151,30 @@ impl ExtensionFunction for StringQ { args: &[Rc], body: Rc, ) -> Result, CompileErr> { - let res = - match match_quoted_string(args[0].clone()) { - Ok(Some(_)) => { - SExp::Integer(loc.clone(), bi_one()) - } - Ok(None) => { - return Ok(body.clone()); - } - Err(_) => { - SExp::Nil(loc.clone()) - } - }; + let res = match match_quoted_string(args[0].clone()) { + Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), + Ok(None) => { + return Ok(body.clone()); + } + Err(_) => SExp::Nil(loc.clone()), + }; Ok(Rc::new(BodyForm::Quoted(res))) } } -struct NumberQ { } +struct NumberQ {} impl NumberQ { - fn new() -> Rc { Rc::new(NumberQ { }) } + fn new() -> Rc { + Rc::new(NumberQ {}) + } } impl ExtensionFunction for NumberQ { - fn required_args(&self) -> Option { Some(1) } + fn required_args(&self) -> Option { + Some(1) + } fn try_eval( &self, @@ -194,31 +186,30 @@ impl ExtensionFunction for NumberQ { args: &[Rc], body: Rc, ) -> Result, CompileErr> { - let res = - match match_number(args[0].clone()) { - Ok(Some(_)) => { - SExp::Integer(loc.clone(), bi_one()) - } - Ok(None) => { - return Ok(body.clone()); - } - Err(_) => { - SExp::Nil(loc.clone()) - } - }; + let res = match match_number(args[0].clone()) { + Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), + Ok(None) => { + return Ok(body.clone()); + } + Err(_) => SExp::Nil(loc.clone()), + }; Ok(Rc::new(BodyForm::Quoted(res))) } } -struct SymbolQ { } +struct SymbolQ {} impl SymbolQ { - fn new() -> Rc { Rc::new(SymbolQ { }) } + fn new() -> Rc { + Rc::new(SymbolQ {}) + } } impl ExtensionFunction for SymbolQ { - fn required_args(&self) -> Option { Some(1) } + fn required_args(&self) -> Option { + Some(1) + } fn try_eval( &self, @@ -230,31 +221,30 @@ impl ExtensionFunction for SymbolQ { args: &[Rc], body: Rc, ) -> Result, CompileErr> { - let res = - match match_atom(args[0].clone()) { - Ok(Some(_)) => { - SExp::Integer(loc.clone(), bi_one()) - } - Ok(None) => { - return Ok(body.clone()); - } - Err(_) => { - SExp::Nil(loc.clone()) - } - }; + let res = match match_atom(args[0].clone()) { + Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), + Ok(None) => { + return Ok(body.clone()); + } + Err(_) => SExp::Nil(loc.clone()), + }; Ok(Rc::new(BodyForm::Quoted(res))) } } -struct SymbolToString { } +struct SymbolToString {} impl SymbolToString { - fn new() -> Rc { Rc::new(SymbolToString { }) } + fn new() -> Rc { + Rc::new(SymbolToString {}) + } } impl ExtensionFunction for SymbolToString { - fn required_args(&self) -> Option { Some(1) } + fn required_args(&self) -> Option { + Some(1) + } fn try_eval( &self, @@ -267,21 +257,27 @@ impl ExtensionFunction for SymbolToString { body: Rc, ) -> Result, CompileErr> { if let Some((loc, value)) = match_atom(args[0].clone())? { - return Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString(loc,b'\"',value)))); + return Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( + loc, b'\"', value, + )))); } else { return Ok(body.clone()); } } } -struct StringToSymbol { } +struct StringToSymbol {} impl StringToSymbol { - fn new() -> Rc { Rc::new(StringToSymbol { }) } + fn new() -> Rc { + Rc::new(StringToSymbol {}) + } } impl ExtensionFunction for StringToSymbol { - fn required_args(&self) -> Option { Some(1) } + fn required_args(&self) -> Option { + Some(1) + } fn try_eval( &self, @@ -294,21 +290,25 @@ impl ExtensionFunction for StringToSymbol { body: Rc, ) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { - return Ok(Rc::new(BodyForm::Quoted(SExp::Atom(loc,value)))); + return Ok(Rc::new(BodyForm::Quoted(SExp::Atom(loc, value)))); } else { return Ok(body.clone()); } } } -struct StringAppend { } +struct StringAppend {} impl StringAppend { - fn new() -> Rc { Rc::new(StringAppend { }) } + fn new() -> Rc { + Rc::new(StringAppend {}) + } } impl ExtensionFunction for StringAppend { - fn required_args(&self) -> Option { None } + fn required_args(&self) -> Option { + None + } fn try_eval( &self, @@ -332,18 +332,26 @@ impl ExtensionFunction for StringAppend { return Ok(body.clone()); } } - return Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString(out_loc.unwrap_or_else(|| body.loc()),b'\"',out_vec)))); + return Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( + out_loc.unwrap_or_else(|| body.loc()), + b'\"', + out_vec, + )))); } } -struct NumberToString { } +struct NumberToString {} impl NumberToString { - fn new() -> Rc { Rc::new(NumberToString { }) } + fn new() -> Rc { + Rc::new(NumberToString {}) + } } impl ExtensionFunction for NumberToString { - fn required_args(&self) -> Option { Some(1) } + fn required_args(&self) -> Option { + Some(1) + } fn try_eval( &self, @@ -356,28 +364,33 @@ impl ExtensionFunction for NumberToString { body: Rc, ) -> Result, CompileErr> { let match_res = match_number(args[0].clone())?; - let (use_loc, int_val) = - match &match_res { - Some(MatchedNumber::MatchedInt(l,i)) => { (l.clone(), i.clone()) } - Some(MatchedNumber::MatchedHex(l,h)) => { - (l.clone(), number_from_u8(&h)) - } - _ => { - return Ok(body.clone()); - } - }; - Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString(use_loc, b'\"', int_val.to_string().as_bytes().to_vec())))) + let (use_loc, int_val) = match &match_res { + Some(MatchedNumber::MatchedInt(l, i)) => (l.clone(), i.clone()), + Some(MatchedNumber::MatchedHex(l, h)) => (l.clone(), number_from_u8(&h)), + _ => { + return Ok(body.clone()); + } + }; + Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( + use_loc, + b'\"', + int_val.to_string().as_bytes().to_vec(), + )))) } } -struct StringToNumber { } +struct StringToNumber {} impl StringToNumber { - fn new() -> Rc { Rc::new(StringToNumber { }) } + fn new() -> Rc { + Rc::new(StringToNumber {}) + } } impl ExtensionFunction for StringToNumber { - fn required_args(&self) -> Option { Some(1) } + fn required_args(&self) -> Option { + Some(1) + } fn try_eval( &self, @@ -391,24 +404,34 @@ impl ExtensionFunction for StringToNumber { ) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { if let Ok(cvt_bi) = decode_string(&value).parse::() { - Ok(Rc::new(BodyForm::Quoted(SExp::Integer(loc.clone(), cvt_bi)))) + Ok(Rc::new(BodyForm::Quoted(SExp::Integer( + loc.clone(), + cvt_bi, + )))) } else { Err(CompileErr(loc.clone(), "bad number".to_string())) } } else { - Err(CompileErr(loc.clone(), "should be given a string".to_string())) + Err(CompileErr( + loc.clone(), + "should be given a string".to_string(), + )) } } } -struct StringLength { } +struct StringLength {} impl StringLength { - fn new() -> Rc { Rc::new(StringLength { }) } + fn new() -> Rc { + Rc::new(StringLength {}) + } } impl ExtensionFunction for StringLength { - fn required_args(&self) -> Option { Some(1) } + fn required_args(&self) -> Option { + Some(1) + } fn try_eval( &self, @@ -422,9 +445,15 @@ impl ExtensionFunction for StringLength { ) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { if let Some(len_bi) = value.len().to_bigint() { - Ok(Rc::new(BodyForm::Quoted(SExp::Integer(loc.clone(), len_bi)))) + Ok(Rc::new(BodyForm::Quoted(SExp::Integer( + loc.clone(), + len_bi, + )))) } else { - Err(CompileErr(loc.clone(), "Error getting string length".to_string())) + Err(CompileErr( + loc.clone(), + "Error getting string length".to_string(), + )) } } else { Ok(body.clone()) @@ -432,15 +461,18 @@ impl ExtensionFunction for StringLength { } } - -struct Substring { } +struct Substring {} impl Substring { - fn new() -> Rc { Rc::new(Substring { }) } + fn new() -> Rc { + Rc::new(Substring {}) + } } impl ExtensionFunction for Substring { - fn required_args(&self) -> Option { Some(3) } + fn required_args(&self) -> Option { + Some(3) + } fn try_eval( &self, @@ -456,32 +488,43 @@ impl ExtensionFunction for Substring { let end_element = usize_value(args[2].clone())?; match args[0].borrow() { - BodyForm::Quoted(SExp::QuotedString(l,ch,s)) => { + BodyForm::Quoted(SExp::QuotedString(l, ch, s)) => { if start_element > end_element || start_element > s.len() || end_element > s.len() { - return Err(CompileErr(l.clone(), "start greater than end in substring".to_string())); + return Err(CompileErr( + l.clone(), + "start greater than end in substring".to_string(), + )); } - let result_value: Vec = - s.iter().take(end_element).skip(start_element).copied().collect(); - Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString(l.clone(), *ch, result_value)))) - } - BodyForm::Quoted(_) => { - Err(CompileErr(body.loc(), "Not a string".to_string())) - } - _ => { - Ok(body.clone()) + let result_value: Vec = s + .iter() + .take(end_element) + .skip(start_element) + .copied() + .collect(); + Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( + l.clone(), + *ch, + result_value, + )))) } + BodyForm::Quoted(_) => Err(CompileErr(body.loc(), "Not a string".to_string())), + _ => Ok(body.clone()), } } } -struct List { } +struct List {} impl List { - fn new() -> Rc { Rc::new(List { }) } + fn new() -> Rc { + Rc::new(List {}) + } } impl ExtensionFunction for List { - fn required_args(&self) -> Option { None } + fn required_args(&self) -> Option { + None + } fn try_eval( &self, @@ -501,29 +544,26 @@ impl ExtensionFunction for List { vec![ Rc::new(BodyForm::Value(SExp::Atom(loc.clone(), b"c".to_vec()))), a.clone(), - res - ] + res, + ], )); } - evaluator.shrink_bodyform( - &mut allocator, - prog_args.clone(), - env, - res, - false, - None - ) + evaluator.shrink_bodyform(&mut allocator, prog_args.clone(), env, res, false, None) } } -struct Cons { } +struct Cons {} impl Cons { - fn new() -> Rc { Rc::new(Cons { }) } + fn new() -> Rc { + Rc::new(Cons {}) + } } impl ExtensionFunction for Cons { - fn required_args(&self) -> Option { Some(2) } + fn required_args(&self) -> Option { + Some(2) + } fn try_eval( &self, @@ -536,21 +576,29 @@ impl ExtensionFunction for Cons { body: Rc, ) -> Result, CompileErr> { if let (BodyForm::Quoted(a), BodyForm::Quoted(b)) = (args[0].borrow(), args[1].borrow()) { - Ok(Rc::new(BodyForm::Quoted(SExp::Cons(loc.clone(), Rc::new(a.clone()), Rc::new(b.clone()))))) + Ok(Rc::new(BodyForm::Quoted(SExp::Cons( + loc.clone(), + Rc::new(a.clone()), + Rc::new(b.clone()), + )))) } else { Ok(body.clone()) } } } -struct First { } +struct First {} impl First { - fn new() -> Rc { Rc::new(First { }) } + fn new() -> Rc { + Rc::new(First {}) + } } impl ExtensionFunction for First { - fn required_args(&self) -> Option { Some(1) } + fn required_args(&self) -> Option { + Some(1) + } fn try_eval( &self, @@ -562,7 +610,7 @@ impl ExtensionFunction for First { args: &[Rc], body: Rc, ) -> Result, CompileErr> { - if let BodyForm::Quoted(SExp::Cons(_,a,_)) = args[0].borrow() { + if let BodyForm::Quoted(SExp::Cons(_, a, _)) = args[0].borrow() { let a_borrowed: &SExp = a.borrow(); Ok(Rc::new(BodyForm::Quoted(a_borrowed.clone()))) } else if let BodyForm::Quoted(_) = args[0].borrow() { @@ -573,14 +621,18 @@ impl ExtensionFunction for First { } } -struct Rest { } +struct Rest {} impl Rest { - fn new() -> Rc { Rc::new(Rest { }) } + fn new() -> Rc { + Rc::new(Rest {}) + } } impl ExtensionFunction for Rest { - fn required_args(&self) -> Option { Some(1) } + fn required_args(&self) -> Option { + Some(1) + } fn try_eval( &self, @@ -592,7 +644,7 @@ impl ExtensionFunction for Rest { args: &[Rc], body: Rc, ) -> Result, CompileErr> { - if let BodyForm::Quoted(SExp::Cons(_,_,b)) = args[0].borrow() { + if let BodyForm::Quoted(SExp::Cons(_, _, b)) = args[0].borrow() { let a_borrowed: &SExp = b.borrow(); Ok(Rc::new(BodyForm::Quoted(a_borrowed.clone()))) } else if let BodyForm::Quoted(_) = args[0].borrow() { @@ -603,16 +655,22 @@ impl ExtensionFunction for Rest { } } -struct If { } +struct If {} impl If { - fn new() -> Rc { Rc::new(If { }) } + fn new() -> Rc { + Rc::new(If {}) + } } impl ExtensionFunction for If { - fn want_interp(&self) -> bool { false } + fn want_interp(&self) -> bool { + false + } - fn required_args(&self) -> Option { Some(3) } + fn required_args(&self) -> Option { + Some(3) + } fn try_eval( &self, @@ -625,15 +683,14 @@ impl ExtensionFunction for If { body: Rc, ) -> Result, CompileErr> { let mut allocator = Allocator::new(); - let cond_result = - evaluator.shrink_bodyform( - &mut allocator, - prog_args.clone(), - env, - args[0].clone(), - false, - None - )?; + let cond_result = evaluator.shrink_bodyform( + &mut allocator, + prog_args.clone(), + env, + args[0].clone(), + false, + None, + )?; if let Ok(unquoted) = dequote(body.loc(), cond_result) { if truthy(unquoted) { @@ -643,7 +700,7 @@ impl ExtensionFunction for If { env, args[1].clone(), false, - None + None, ) } else { evaluator.shrink_bodyform( @@ -652,7 +709,7 @@ impl ExtensionFunction for If { env, args[2].clone(), false, - None + None, ) } } else { @@ -696,7 +753,7 @@ impl ExtensionFunction for If { /// substring s start end /// pub struct PreprocessorExtension { - extfuns: HashMap, Rc> + extfuns: HashMap, Rc>, } impl PreprocessorExtension { @@ -707,21 +764,20 @@ impl PreprocessorExtension { (b"c".to_vec(), Cons::new()), (b"f".to_vec(), First::new()), (b"r".to_vec(), Rest::new()), - (b"string?".to_vec(), StringQ::new()), (b"number?".to_vec(), NumberQ::new()), (b"symbol?".to_vec(), SymbolQ::new()), - (b"string->symbol".to_vec(), StringToSymbol::new()), (b"symbol->string".to_vec(), SymbolToString::new()), (b"string->number".to_vec(), StringToNumber::new()), (b"number->string".to_vec(), NumberToString::new()), - (b"string-append".to_vec(), StringAppend::new()), (b"string-length".to_vec(), StringLength::new()), (b"substring".to_vec(), Substring::new()), ]; - PreprocessorExtension { extfuns: HashMap::from(extfuns) } + PreprocessorExtension { + extfuns: HashMap::from(extfuns), + } } } @@ -739,33 +795,24 @@ impl EvalExtension for PreprocessorExtension { if let Some(extfun) = self.extfuns.get(name) { if let Some(n) = extfun.required_args() { if raw_args.len() != n { - return Err(CompileErr(loc.clone(), format!("{} requires {} args", decode_string(name), n))); + return Err(CompileErr( + loc.clone(), + format!("{} requires {} args", decode_string(name), n), + )); } } eprintln!("try function {}", body.to_sexp()); - for (n,v) in env.iter() { + for (n, v) in env.iter() { eprintln!("- {} = {}", decode_string(&n), v.to_sexp()); } - let args = - if extfun.want_interp() { - reify_args( - evaluator, - prog_args.clone(), - env, - raw_args - )? - } else { - raw_args.to_vec() - }; + let args = if extfun.want_interp() { + reify_args(evaluator, prog_args.clone(), env, raw_args)? + } else { + raw_args.to_vec() + }; Ok(Some(extfun.try_eval( - evaluator, - prog_args, - env, - loc, - name, - &args, - body + evaluator, prog_args, env, loc, name, &args, body, )?)) } else { Ok(None) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 2ddb1358d..da6cad96c 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -10,10 +10,12 @@ use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; use crate::compiler::compiler::KNOWN_DIALECTS; use crate::compiler::comptypes::{BodyForm, CompileErr, CompilerOpts, HelperForm, IncludeDesc}; -use crate::compiler::evaluate::{ArgInputs, create_argument_captures, dequote, Evaluator}; +use crate::compiler::evaluate::{create_argument_captures, dequote, ArgInputs, Evaluator}; use crate::compiler::frontend::compile_helperform; -use crate::compiler::sexp::{Atom, decode_string, enlist, NodeSel, parse_sexp, SelectNode, SExp, ThisNode}; use crate::compiler::preprocessor::macros::PreprocessorExtension; +use crate::compiler::sexp::{ + decode_string, enlist, parse_sexp, Atom, NodeSel, SExp, SelectNode, ThisNode, +}; use crate::compiler::srcloc::Srcloc; use crate::util::ErrInto; @@ -31,17 +33,16 @@ impl Preprocessor { Preprocessor { opts: opts.clone(), evaluator: eval, - helpers: Vec::new() + helpers: Vec::new(), } } /// Given a specification of an include file, load up the forms inside it and /// return them (or an error if the file couldn't be read or wasn't a list). - pub fn process_include( - &mut self, - include: IncludeDesc, - ) -> Result>, CompileErr> { - let filename_and_content = self.opts.read_new_file(self.opts.filename(), decode_string(&include.name))?; + pub fn process_include(&mut self, include: IncludeDesc) -> Result>, CompileErr> { + let filename_and_content = self + .opts + .read_new_file(self.opts.filename(), decode_string(&include.name))?; let content = filename_and_content.1; let start_of_file = Srcloc::start(&decode_string(&include.name)); @@ -61,7 +62,7 @@ impl Preprocessor { fn recurse_dependencies( &mut self, includes: &mut Vec, - desc: IncludeDesc + desc: IncludeDesc, ) -> Result<(), CompileErr> { let name_string = decode_string(&desc.name); if KNOWN_DIALECTS.contains_key(&name_string) { @@ -91,23 +92,21 @@ impl Preprocessor { // Check for and apply preprocessor level macros. // This is maximally permissive. - fn expand_macros( - &mut self, - body: Rc - ) -> Result, CompileErr> { + fn expand_macros(&mut self, body: Rc) -> Result, CompileErr> { eprintln!("expand_macros {}", body); - if let SExp::Cons(l,f,r) = body.borrow() { + if let SExp::Cons(l, f, r) = body.borrow() { // First expand inner macros. let first_expanded = self.expand_macros(f.clone())?; let rest_expanded = self.expand_macros(r.clone())?; let new_self = Rc::new(SExp::Cons(l.clone(), first_expanded, rest_expanded)); - if let Ok(NodeSel::Cons((_, name), args)) = NodeSel::Cons( - Atom::Here(()), ThisNode::Here - ).select_nodes(new_self.clone()) { + if let Ok(NodeSel::Cons((_, name), args)) = + NodeSel::Cons(Atom::Here(()), ThisNode::Here).select_nodes(new_self.clone()) + { // See if it's a form that calls one of our macros. for m in self.helpers.iter() { eprintln!("want {} helper {}", decode_string(&name), m.to_sexp()); - if let HelperForm::Defun(_,mdata) = &m { // We record upfront macros + if let HelperForm::Defun(_, mdata) = &m { + // We record upfront macros if mdata.name != name { continue; } @@ -128,7 +127,7 @@ impl Preprocessor { create_argument_captures( &mut macro_arg_env, &ArgInputs::Whole(Rc::new(BodyForm::Quoted(args_borrowed.clone()))), - mdata.args.clone() + mdata.args.clone(), )?; let res = self.evaluator.shrink_bodyform( @@ -137,7 +136,7 @@ impl Preprocessor { ¯o_arg_env, mdata.body.clone(), false, - None + None, )?; if let Ok(unquoted) = dequote(body.loc(), res.clone()) { @@ -158,26 +157,28 @@ impl Preprocessor { } // If it's a defmac (preprocessor level macro), add it to the evaulator. - fn decode_macro( - &mut self, - definition: Rc - ) -> Result, CompileErr> { + fn decode_macro(&mut self, definition: Rc) -> Result, CompileErr> { eprintln!("decode_macro {definition}"); if let Ok(NodeSel::Cons( (defmac_loc, kw), - NodeSel::Cons( - (nl, name), - NodeSel::Cons(args,body) - ) + NodeSel::Cons((nl, name), NodeSel::Cons(args, body)), )) = NodeSel::Cons( Atom::Here(()), NodeSel::Cons( Atom::Here(()), - NodeSel::Cons(ThisNode::Here, ThisNode::Here) - ) - ).select_nodes(definition.clone()) { + NodeSel::Cons(ThisNode::Here, ThisNode::Here), + ), + ) + .select_nodes(definition.clone()) + { let is_defmac = kw == b"defmac"; - if is_defmac || kw == b"defmacro" || kw == b"defun" || kw == b"defun-inline" || kw == b"defconst" || kw == b"defconstant" { + if is_defmac + || kw == b"defmacro" + || kw == b"defun" + || kw == b"defun-inline" + || kw == b"defconst" + || kw == b"defconstant" + { if is_defmac { let target_defun = Rc::new(SExp::Cons( defmac_loc.clone(), @@ -185,10 +186,8 @@ impl Preprocessor { Rc::new(SExp::Cons( nl.clone(), Rc::new(SExp::Atom(nl.clone(), name.clone())), - Rc::new(SExp::Cons( - args.loc(), args.clone(), body.clone() - )) - )) + Rc::new(SExp::Cons(args.loc(), args.clone(), body.clone())), + )), )); eprintln!("target_defun {target_defun}"); if let Some(helper) = compile_helperform(self.opts.clone(), target_defun)? { @@ -196,7 +195,10 @@ impl Preprocessor { self.evaluator.add_helper(&helper); self.helpers.push(helper); } else { - return Err(CompileErr(definition.loc(), "defmac found but couldn't be converted to function".to_string())); + return Err(CompileErr( + definition.loc(), + "defmac found but couldn't be converted to function".to_string(), + )); } } else if let Some(helper) = compile_helperform(self.opts.clone(), definition)? { self.evaluator.add_helper(&helper); @@ -312,7 +314,7 @@ impl Preprocessor { cmod }; - while let SExp::Cons(_,f,r) = tocompile.borrow() { + while let SExp::Cons(_, f, r) = tocompile.borrow() { let mut lst = self.process_pp_form(includes, f.clone())?; result.append(&mut lst); tocompile = r.clone(); @@ -328,7 +330,7 @@ impl Preprocessor { pub fn preprocess( opts: Rc, includes: &mut Vec, - cmod: Rc + cmod: Rc, ) -> Result>, CompileErr> { let mut p = Preprocessor::new(opts); p.run(includes, cmod) diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 8c2da6bc4..c1b4d009c 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -868,10 +868,9 @@ impl SelectNode, E> for ThisNode { } impl SelectNode<(Srcloc, Vec), (Srcloc, String)> for Atom<()> { - fn select_nodes(&self, s: Rc) -> Result<(Srcloc, Vec), (Srcloc, String)> - { - if let SExp::Atom(loc,name) = s.borrow() { - return Ok((loc.clone(),name.clone())); + fn select_nodes(&self, s: Rc) -> Result<(Srcloc, Vec), (Srcloc, String)> { + if let SExp::Atom(loc, name) = s.borrow() { + return Ok((loc.clone(), name.clone())); } Err((s.loc(), "Not an atom".to_string())) @@ -881,7 +880,7 @@ impl SelectNode<(Srcloc, Vec), (Srcloc, String)> for Atom<()> { impl SelectNode for Atom<&str> { fn select_nodes(&self, s: Rc) -> Result { let Atom::Here(name) = self; - if let Ok((l,n)) = Atom::Here(()).select_nodes(s.clone()) { + if let Ok((l, n)) = Atom::Here(()).select_nodes(s.clone()) { if n == name.as_bytes() { return Ok(l.clone()); } diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 40495c8df..5ccfd906d 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -190,7 +190,8 @@ fn test_defmac_if_extension() { (defmac F (X) (c 1 (FX X))) (F X) ) - "}.to_string(); + "} + .to_string(); let res = run_string(&prog, &"(9)".to_string()).unwrap(); assert_eq!(res.to_string(), "\"1\""); } @@ -333,7 +334,8 @@ fn test_defmac_stringq() { (defmac is-string (X) (string? X)) (list (is-string X) (is-string \"X\") (is-string 3)) ) - "}.to_string(); + "} + .to_string(); let res = run_string(&prog, &"()".to_string()).unwrap(); assert_eq!(res.to_string(), "(() 1 ())"); } @@ -345,7 +347,8 @@ fn test_defmac_numberq() { (defmac is-number (X) (number? X)) (list (is-number X) (is-number \"X\") (is-number 3)) ) - "}.to_string(); + "} + .to_string(); let res = run_string(&prog, &"()".to_string()).unwrap(); assert_eq!(res.to_string(), "(() () 1)"); } @@ -357,7 +360,8 @@ fn test_defmac_symbolq() { (defmac is-symbol (X) (symbol? X)) (list (is-symbol X) (is-symbol \"X\") (is-symbol 3)) ) - "}.to_string(); + "} + .to_string(); let res = run_string(&prog, &"()".to_string()).unwrap(); assert_eq!(res.to_string(), "(1 () ())"); } @@ -369,7 +373,8 @@ fn test_defmac_string_to_symbol() { (defmac is-symbol (X) (symbol? X)) (list (is-symbol X) (is-symbol \"X\") (is-symbol 3)) ) - "}.to_string(); + "} + .to_string(); let res = run_string(&prog, &"()".to_string()).unwrap(); assert_eq!(res.to_string(), "(1 () ())"); } @@ -381,7 +386,8 @@ fn test_defmac_string_to_symbol_converts() { (defmac let_pi (code) (qq (let (((unquote (string->symbol \"pi\")) 31415)) (unquote code)))) (let_pi (+ pi X)) ) - "}.to_string(); + "} + .to_string(); let res = run_string(&prog, &"(5)".to_string()).unwrap(); assert_eq!(res.to_string(), "31420"); } @@ -393,7 +399,8 @@ fn test_defmac_string_needs_conversion() { (defmac let_pi (code) (qq (let ((\"pi\" 31415)) (unquote code)))) (let_pi (+ pi X)) ) - "}.to_string(); + "} + .to_string(); let res = run_string(&prog, &"(5)".to_string()); assert!(res.is_err()); } @@ -409,7 +416,8 @@ fn test_defmac_string_substr_0() { ) (first-letter-of Xanadu) ) - "}.to_string(); + "} + .to_string(); let res = run_string(&prog, &"(5999)".to_string()).unwrap(); assert_eq!(res.to_string(), "(\"X\" . 5999)"); } @@ -428,7 +436,8 @@ fn test_defmac_string_substr_bad() { ) (bind-tail-of-symbol 100 test_variable_name (c 9999 variable_name)) ) - "}.to_string(); + "} + .to_string(); let res = run_string(&prog, &"((87 89 91))".to_string()); assert!(res.is_err()); } @@ -448,7 +457,8 @@ fn test_defmac_string_to_number_0() { ) (add-n-to X_7) ) - "}.to_string(); + "} + .to_string(); let res = run_string(&prog, &"(31)".to_string()).unwrap(); assert_eq!(res.to_string(), "38"); } @@ -468,7 +478,8 @@ fn test_defmac_string_to_number_bad() { ) (add-n-to X_A) ) - "}.to_string(); + "} + .to_string(); let res = run_string(&prog, &"(31)".to_string()); assert!(res.is_err()); } @@ -487,7 +498,8 @@ fn test_defmac_number_to_string() { (defun F (Xanadu-6) (+ (with-my-length Xanadu) 99)) (F Q) ) - "}.to_string(); + "} + .to_string(); let res = run_string(&prog, &"(37)".to_string()).unwrap(); assert_eq!(res.to_string(), "136"); } From fdf81e9a7baa819efc4e404bba77adcea92ac48d Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 22 Mar 2023 09:37:36 -0700 Subject: [PATCH 18/81] Tune up, should be ready to pr --- src/compiler/evaluate.rs | 3 +- src/compiler/preprocessor/macros.rs | 117 ++++++++++++++-------------- src/compiler/preprocessor/mod.rs | 24 ++---- src/compiler/sexp.rs | 2 +- 4 files changed, 66 insertions(+), 80 deletions(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index f5fcf4ef4..893f20489 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -95,6 +95,7 @@ pub enum ArgInputs { /// I also anticipate using EvalExtensions to analyze and control code shrinking /// during some kinds of optimization. pub trait EvalExtension { + #[allow(clippy::too_many_arguments)] fn try_eval( &self, evaluator: &Evaluator, @@ -943,7 +944,7 @@ impl<'info> Evaluator { env, &l, call_name, - &arguments_to_convert, + arguments_to_convert, body.clone(), )? { return Ok(res); diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index f04f0db36..7e2dbc652 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -69,7 +69,7 @@ fn match_number(body: Rc) -> Result, CompileErr> fn numeric_value(body: Rc) -> Result { match match_number(body.clone())? { - Some(MatchedNumber::MatchedInt(_, n)) => Ok(n.clone()), + Some(MatchedNumber::MatchedInt(_, n)) => Ok(n), Some(MatchedNumber::MatchedHex(_, h)) => Ok(number_from_u8(&h)), _ => Err(CompileErr(body.loc(), "Not a number".to_string())), } @@ -116,6 +116,7 @@ pub trait ExtensionFunction { true } fn required_args(&self) -> Option; + #[allow(clippy::too_many_arguments)] fn try_eval( &self, evaluator: &Evaluator, @@ -131,7 +132,7 @@ pub trait ExtensionFunction { struct StringQ {} impl StringQ { - fn new() -> Rc { + fn create() -> Rc { Rc::new(StringQ {}) } } @@ -154,7 +155,7 @@ impl ExtensionFunction for StringQ { let res = match match_quoted_string(args[0].clone()) { Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), Ok(None) => { - return Ok(body.clone()); + return Ok(body); } Err(_) => SExp::Nil(loc.clone()), }; @@ -166,7 +167,7 @@ impl ExtensionFunction for StringQ { struct NumberQ {} impl NumberQ { - fn new() -> Rc { + fn create() -> Rc { Rc::new(NumberQ {}) } } @@ -189,7 +190,7 @@ impl ExtensionFunction for NumberQ { let res = match match_number(args[0].clone()) { Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), Ok(None) => { - return Ok(body.clone()); + return Ok(body); } Err(_) => SExp::Nil(loc.clone()), }; @@ -201,7 +202,7 @@ impl ExtensionFunction for NumberQ { struct SymbolQ {} impl SymbolQ { - fn new() -> Rc { + fn create() -> Rc { Rc::new(SymbolQ {}) } } @@ -224,7 +225,7 @@ impl ExtensionFunction for SymbolQ { let res = match match_atom(args[0].clone()) { Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), Ok(None) => { - return Ok(body.clone()); + return Ok(body); } Err(_) => SExp::Nil(loc.clone()), }; @@ -236,7 +237,7 @@ impl ExtensionFunction for SymbolQ { struct SymbolToString {} impl SymbolToString { - fn new() -> Rc { + fn create() -> Rc { Rc::new(SymbolToString {}) } } @@ -257,11 +258,11 @@ impl ExtensionFunction for SymbolToString { body: Rc, ) -> Result, CompileErr> { if let Some((loc, value)) = match_atom(args[0].clone())? { - return Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( + Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( loc, b'\"', value, - )))); + )))) } else { - return Ok(body.clone()); + Ok(body) } } } @@ -269,7 +270,7 @@ impl ExtensionFunction for SymbolToString { struct StringToSymbol {} impl StringToSymbol { - fn new() -> Rc { + fn create() -> Rc { Rc::new(StringToSymbol {}) } } @@ -290,9 +291,9 @@ impl ExtensionFunction for StringToSymbol { body: Rc, ) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { - return Ok(Rc::new(BodyForm::Quoted(SExp::Atom(loc, value)))); + Ok(Rc::new(BodyForm::Quoted(SExp::Atom(loc, value)))) } else { - return Ok(body.clone()); + Ok(body) } } } @@ -300,7 +301,7 @@ impl ExtensionFunction for StringToSymbol { struct StringAppend {} impl StringAppend { - fn new() -> Rc { + fn create() -> Rc { Rc::new(StringAppend {}) } } @@ -329,21 +330,21 @@ impl ExtensionFunction for StringAppend { } out_vec.append(&mut value); } else { - return Ok(body.clone()); + return Ok(body); } } - return Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( + Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( out_loc.unwrap_or_else(|| body.loc()), b'\"', out_vec, - )))); + )))) } } struct NumberToString {} impl NumberToString { - fn new() -> Rc { + fn create() -> Rc { Rc::new(NumberToString {}) } } @@ -366,9 +367,9 @@ impl ExtensionFunction for NumberToString { let match_res = match_number(args[0].clone())?; let (use_loc, int_val) = match &match_res { Some(MatchedNumber::MatchedInt(l, i)) => (l.clone(), i.clone()), - Some(MatchedNumber::MatchedHex(l, h)) => (l.clone(), number_from_u8(&h)), + Some(MatchedNumber::MatchedHex(l, h)) => (l.clone(), number_from_u8(h)), _ => { - return Ok(body.clone()); + return Ok(body); } }; Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( @@ -382,7 +383,7 @@ impl ExtensionFunction for NumberToString { struct StringToNumber {} impl StringToNumber { - fn new() -> Rc { + fn create() -> Rc { Rc::new(StringToNumber {}) } } @@ -405,11 +406,11 @@ impl ExtensionFunction for StringToNumber { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { if let Ok(cvt_bi) = decode_string(&value).parse::() { Ok(Rc::new(BodyForm::Quoted(SExp::Integer( - loc.clone(), + loc, cvt_bi, )))) } else { - Err(CompileErr(loc.clone(), "bad number".to_string())) + Err(CompileErr(loc, "bad number".to_string())) } } else { Err(CompileErr( @@ -423,7 +424,7 @@ impl ExtensionFunction for StringToNumber { struct StringLength {} impl StringLength { - fn new() -> Rc { + fn create() -> Rc { Rc::new(StringLength {}) } } @@ -446,17 +447,17 @@ impl ExtensionFunction for StringLength { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { if let Some(len_bi) = value.len().to_bigint() { Ok(Rc::new(BodyForm::Quoted(SExp::Integer( - loc.clone(), + loc, len_bi, )))) } else { Err(CompileErr( - loc.clone(), + loc, "Error getting string length".to_string(), )) } } else { - Ok(body.clone()) + Ok(body) } } } @@ -464,7 +465,7 @@ impl ExtensionFunction for StringLength { struct Substring {} impl Substring { - fn new() -> Rc { + fn create() -> Rc { Rc::new(Substring {}) } } @@ -508,7 +509,7 @@ impl ExtensionFunction for Substring { )))) } BodyForm::Quoted(_) => Err(CompileErr(body.loc(), "Not a string".to_string())), - _ => Ok(body.clone()), + _ => Ok(body), } } } @@ -516,7 +517,7 @@ impl ExtensionFunction for Substring { struct List {} impl List { - fn new() -> Rc { + fn create() -> Rc { Rc::new(List {}) } } @@ -548,14 +549,14 @@ impl ExtensionFunction for List { ], )); } - evaluator.shrink_bodyform(&mut allocator, prog_args.clone(), env, res, false, None) + evaluator.shrink_bodyform(&mut allocator, prog_args, env, res, false, None) } } struct Cons {} impl Cons { - fn new() -> Rc { + fn create() -> Rc { Rc::new(Cons {}) } } @@ -582,7 +583,7 @@ impl ExtensionFunction for Cons { Rc::new(b.clone()), )))) } else { - Ok(body.clone()) + Ok(body) } } } @@ -590,7 +591,7 @@ impl ExtensionFunction for Cons { struct First {} impl First { - fn new() -> Rc { + fn create() -> Rc { Rc::new(First {}) } } @@ -616,7 +617,7 @@ impl ExtensionFunction for First { } else if let BodyForm::Quoted(_) = args[0].borrow() { Err(CompileErr(loc.clone(), "bad cons in first".to_string())) } else { - Ok(body.clone()) + Ok(body) } } } @@ -624,7 +625,7 @@ impl ExtensionFunction for First { struct Rest {} impl Rest { - fn new() -> Rc { + fn create() -> Rc { Rc::new(Rest {}) } } @@ -650,7 +651,7 @@ impl ExtensionFunction for Rest { } else if let BodyForm::Quoted(_) = args[0].borrow() { Err(CompileErr(loc.clone(), "bad cons in rest".to_string())) } else { - Ok(body.clone()) + Ok(body) } } } @@ -658,7 +659,7 @@ impl ExtensionFunction for Rest { struct If {} impl If { - fn new() -> Rc { + fn create() -> Rc { Rc::new(If {}) } } @@ -696,7 +697,7 @@ impl ExtensionFunction for If { if truthy(unquoted) { evaluator.shrink_bodyform( &mut allocator, - prog_args.clone(), + prog_args, env, args[1].clone(), false, @@ -705,7 +706,7 @@ impl ExtensionFunction for If { } else { evaluator.shrink_bodyform( &mut allocator, - prog_args.clone(), + prog_args, env, args[2].clone(), false, @@ -759,21 +760,21 @@ pub struct PreprocessorExtension { impl PreprocessorExtension { pub fn new() -> Self { let extfuns = [ - (b"if".to_vec(), If::new()), - (b"list".to_vec(), List::new()), - (b"c".to_vec(), Cons::new()), - (b"f".to_vec(), First::new()), - (b"r".to_vec(), Rest::new()), - (b"string?".to_vec(), StringQ::new()), - (b"number?".to_vec(), NumberQ::new()), - (b"symbol?".to_vec(), SymbolQ::new()), - (b"string->symbol".to_vec(), StringToSymbol::new()), - (b"symbol->string".to_vec(), SymbolToString::new()), - (b"string->number".to_vec(), StringToNumber::new()), - (b"number->string".to_vec(), NumberToString::new()), - (b"string-append".to_vec(), StringAppend::new()), - (b"string-length".to_vec(), StringLength::new()), - (b"substring".to_vec(), Substring::new()), + (b"if".to_vec(), If::create()), + (b"list".to_vec(), List::create()), + (b"c".to_vec(), Cons::create()), + (b"f".to_vec(), First::create()), + (b"r".to_vec(), Rest::create()), + (b"string?".to_vec(), StringQ::create()), + (b"number?".to_vec(), NumberQ::create()), + (b"symbol?".to_vec(), SymbolQ::create()), + (b"string->symbol".to_vec(), StringToSymbol::create()), + (b"symbol->string".to_vec(), SymbolToString::create()), + (b"string->number".to_vec(), StringToNumber::create()), + (b"number->string".to_vec(), NumberToString::create()), + (b"string-append".to_vec(), StringAppend::create()), + (b"string-length".to_vec(), StringLength::create()), + (b"substring".to_vec(), Substring::create()), ]; PreprocessorExtension { extfuns: HashMap::from(extfuns), @@ -802,10 +803,6 @@ impl EvalExtension for PreprocessorExtension { } } - eprintln!("try function {}", body.to_sexp()); - for (n, v) in env.iter() { - eprintln!("- {} = {}", decode_string(&n), v.to_sexp()); - } let args = if extfun.want_interp() { reify_args(evaluator, prog_args.clone(), env, raw_args)? } else { diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index da6cad96c..d8af94fc5 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -93,7 +93,6 @@ impl Preprocessor { // Check for and apply preprocessor level macros. // This is maximally permissive. fn expand_macros(&mut self, body: Rc) -> Result, CompileErr> { - eprintln!("expand_macros {}", body); if let SExp::Cons(l, f, r) = body.borrow() { // First expand inner macros. let first_expanded = self.expand_macros(f.clone())?; @@ -104,18 +103,12 @@ impl Preprocessor { { // See if it's a form that calls one of our macros. for m in self.helpers.iter() { - eprintln!("want {} helper {}", decode_string(&name), m.to_sexp()); if let HelperForm::Defun(_, mdata) = &m { // We record upfront macros if mdata.name != name { continue; } - eprintln!("expanding macro {}", m.to_sexp()); - for h in self.helpers.iter() { - eprintln!("- {}", decode_string(h.name())); - } - // as inline defuns because they're closest to that // semantically. let mut allocator = Allocator::new(); @@ -139,12 +132,10 @@ impl Preprocessor { None, )?; - if let Ok(unquoted) = dequote(body.loc(), res.clone()) { - eprintln!("expand macro {}", unquoted); + if let Ok(unquoted) = dequote(body.loc(), res) { return Ok(unquoted); } else { - eprintln!("bad expand? {}", res.to_sexp()); - todo!(); + return Err(CompileErr(body.loc(), "Failed to fully evaluate macro".to_string())); } } } @@ -158,7 +149,6 @@ impl Preprocessor { // If it's a defmac (preprocessor level macro), add it to the evaulator. fn decode_macro(&mut self, definition: Rc) -> Result, CompileErr> { - eprintln!("decode_macro {definition}"); if let Ok(NodeSel::Cons( (defmac_loc, kw), NodeSel::Cons((nl, name), NodeSel::Cons(args, body)), @@ -182,16 +172,14 @@ impl Preprocessor { if is_defmac { let target_defun = Rc::new(SExp::Cons( defmac_loc.clone(), - Rc::new(SExp::atom_from_string(defmac_loc.clone(), "defun")), + Rc::new(SExp::atom_from_string(defmac_loc, "defun")), Rc::new(SExp::Cons( nl.clone(), - Rc::new(SExp::Atom(nl.clone(), name.clone())), - Rc::new(SExp::Cons(args.loc(), args.clone(), body.clone())), + Rc::new(SExp::Atom(nl, name)), + Rc::new(SExp::Cons(args.loc(), args.clone(), body)), )), )); - eprintln!("target_defun {target_defun}"); if let Some(helper) = compile_helperform(self.opts.clone(), target_defun)? { - eprintln!("add helper {}", helper.to_sexp()); self.evaluator.add_helper(&helper); self.helpers.push(helper); } else { @@ -350,7 +338,7 @@ pub fn gather_dependencies( let mut p = Preprocessor::new(no_stdenv_opts); let loc = Srcloc::start(real_input_path); - let parsed = parse_sexp(loc.clone(), file_content.bytes())?; + let parsed = parse_sexp(loc, file_content.bytes())?; if parsed.is_empty() { return Ok(vec![]); diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index c1b4d009c..4e9dcb2d0 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -882,7 +882,7 @@ impl SelectNode for Atom<&str> { let Atom::Here(name) = self; if let Ok((l, n)) = Atom::Here(()).select_nodes(s.clone()) { if n == name.as_bytes() { - return Ok(l.clone()); + return Ok(l); } } From 8237c6af5f97b6df7db590e87629abec9ca3e145 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 22 Mar 2023 09:51:30 -0700 Subject: [PATCH 19/81] Fmt --- src/compiler/preprocessor/macros.rs | 15 +++------------ src/compiler/preprocessor/mod.rs | 5 ++++- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 7e2dbc652..28abf435d 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -405,10 +405,7 @@ impl ExtensionFunction for StringToNumber { ) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { if let Ok(cvt_bi) = decode_string(&value).parse::() { - Ok(Rc::new(BodyForm::Quoted(SExp::Integer( - loc, - cvt_bi, - )))) + Ok(Rc::new(BodyForm::Quoted(SExp::Integer(loc, cvt_bi)))) } else { Err(CompileErr(loc, "bad number".to_string())) } @@ -446,15 +443,9 @@ impl ExtensionFunction for StringLength { ) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { if let Some(len_bi) = value.len().to_bigint() { - Ok(Rc::new(BodyForm::Quoted(SExp::Integer( - loc, - len_bi, - )))) + Ok(Rc::new(BodyForm::Quoted(SExp::Integer(loc, len_bi)))) } else { - Err(CompileErr( - loc, - "Error getting string length".to_string(), - )) + Err(CompileErr(loc, "Error getting string length".to_string())) } } else { Ok(body) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index d8af94fc5..77d77d94d 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -135,7 +135,10 @@ impl Preprocessor { if let Ok(unquoted) = dequote(body.loc(), res) { return Ok(unquoted); } else { - return Err(CompileErr(body.loc(), "Failed to fully evaluate macro".to_string())); + return Err(CompileErr( + body.loc(), + "Failed to fully evaluate macro".to_string(), + )); } } } From 0aa7ad511b9e54e90c9697f0d5e1432f77dea2f1 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 23 May 2023 17:40:33 -0700 Subject: [PATCH 20/81] Start strict --- src/classic/clvm_tools/clvmc.rs | 29 ++++++++++++++++------------- src/classic/clvm_tools/cmds.rs | 6 +++--- src/compiler/compiler.rs | 12 +++++++++++- src/compiler/comptypes.rs | 11 +++++++++++ src/tests/classic/stage_2.rs | 8 +++++++- 5 files changed, 48 insertions(+), 18 deletions(-) diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index b2ac08bab..e18805338 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -22,10 +22,8 @@ use crate::classic::platform::distutils::dep_util::newer; use crate::compiler::clvm::convert_to_clvm_rs; use crate::compiler::compiler::compile_file; -use crate::compiler::compiler::run_optimizer; -use crate::compiler::compiler::DefaultCompilerOpts; -use crate::compiler::comptypes::CompileErr; -use crate::compiler::comptypes::CompilerOpts; +use crate::compiler::compiler::{DefaultCompilerOpts, run_optimizer}; +use crate::compiler::comptypes::{AcceptedDialect, CompileErr, CompilerOpts}; use crate::compiler::runtypes::RunFailure; fn include_dialect( @@ -56,15 +54,19 @@ pub fn write_sym_output( .map(|_| ()) } -pub fn detect_modern(allocator: &mut Allocator, sexp: NodePtr) -> Option { +pub fn detect_modern(allocator: &mut Allocator, sexp: NodePtr) -> AcceptedDialect { let mut dialects = HashMap::new(); dialects.insert("*standard-cl-21*".as_bytes().to_vec(), 21); dialects.insert("*standard-cl-22*".as_bytes().to_vec(), 22); - proper_list(allocator, sexp, true).and_then(|l| { + let mut result = Default::default(); + + if let Some(l) = proper_list(allocator, sexp, true) { for elt in l.iter() { - if let Some(dialect) = detect_modern(allocator, *elt) { - return Some(dialect); + let detect_modern_result = detect_modern(allocator, *elt); + if detect_modern_result.stepping.is_some() { + result = detect_modern_result; + break; } match proper_list(allocator, *elt, true) { @@ -78,14 +80,15 @@ pub fn detect_modern(allocator: &mut Allocator, sexp: NodePtr) -> Option { } if let Some(dialect) = include_dialect(allocator, &dialects, &e) { - return Some(dialect); + result.stepping = Some(dialect); + break; } } } } + } - None - }) + result } pub fn compile_clvm_text( @@ -98,8 +101,8 @@ pub fn compile_clvm_text( ) -> Result { let ir_src = read_ir(text).map_err(|s| EvalErr(allocator.null(), s.to_string()))?; let assembled_sexp = assemble_from_ir(allocator, Rc::new(ir_src))?; - - if let Some(dialect) = detect_modern(allocator, assembled_sexp) { + let dialect = detect_modern(allocator, assembled_sexp); + if let Some(dialect) = dialect.stepping { let runner = Rc::new(DefaultProgramRunner::new()); let opts = opts.set_optimize(true).set_frontend_opt(dialect > 21); diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index 79eb746d7..dea8a5746 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -1145,9 +1145,9 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .map(|a| matches!(a, ArgumentValue::ArgBool(true))) .unwrap_or(false); - let dialect = input_sexp.and_then(|i| detect_modern(&mut allocator, i)); + let dialect = input_sexp.map(|i| detect_modern(&mut allocator, i)); let mut stderr_output = |s: String| { - if dialect.is_some() { + if dialect.as_ref().and_then(|d| d.stepping).is_some() { eprintln!("{s}"); } else { stdout.write_str(&s); @@ -1183,7 +1183,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .unwrap_or_else(|| "main.sym".to_string()); // In testing: short circuit for modern compilation. - if let Some(dialect) = dialect { + if let Some(dialect) = dialect.and_then(|d| d.stepping) { let do_optimize = parsed_args .get("optimize") .map(|x| matches!(x, ArgumentValue::ArgBool(true))) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 7639f88b5..0bab69b35 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -14,7 +14,7 @@ use crate::classic::clvm_tools::stages::stage_2::optimize::optimize_sexp; use crate::compiler::clvm::{convert_from_clvm_rs, convert_to_clvm_rs, sha256tree}; use crate::compiler::codegen::{codegen, hoist_body_let_binding, process_helper_let_bindings}; use crate::compiler::comptypes::{ - CompileErr, CompileForm, CompilerOpts, DefunData, HelperForm, PrimaryCodegen, + AcceptedDialect, CompileErr, CompileForm, CompilerOpts, DefunData, HelperForm, PrimaryCodegen, }; use crate::compiler::evaluate::{build_reflex_captures, Evaluator, EVAL_STACK_LIMIT}; use crate::compiler::frontend::frontend; @@ -74,6 +74,7 @@ pub struct DefaultCompilerOpts { pub frontend_check_live: bool, pub start_env: Option>, pub prim_map: Rc, Rc>>, + pub dialect: AcceptedDialect, known_dialects: Rc>, } @@ -228,6 +229,9 @@ impl CompilerOpts for DefaultCompilerOpts { fn code_generator(&self) -> Option { self.code_generator.clone() } + fn dialect(&self) -> AcceptedDialect { + self.dialect.clone() + } fn in_defun(&self) -> bool { self.in_defun } @@ -253,6 +257,11 @@ impl CompilerOpts for DefaultCompilerOpts { self.include_dirs.clone() } + fn set_dialect(&self, dialect: AcceptedDialect) -> Rc { + let mut copy = self.clone(); + copy.dialect = dialect; + Rc::new(copy) + } fn set_search_paths(&self, dirs: &[String]) -> Rc { let mut copy = self.clone(); copy.include_dirs = dirs.to_owned(); @@ -349,6 +358,7 @@ impl DefaultCompilerOpts { frontend_opt: false, frontend_check_live: true, start_env: None, + dialect: Default::default(), prim_map: create_prim_map(), known_dialects: Rc::new(KNOWN_DIALECTS.clone()), } diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 0b7b89e1a..7febd5b2f 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -30,6 +30,13 @@ impl From<(Srcloc, String)> for CompileErr { #[derive(Clone, Debug)] pub struct CompiledCode(pub Srcloc, pub Rc); +/// Specifying how the language is spoken. +#[derive(Clone, Debug, Default)] +pub struct AcceptedDialect { + pub stepping: Option, + pub strict: bool, +} + /// A description of an inlined function for use during inline expansion. /// This is used only by PrimaryCodegen. #[derive(Clone, Debug)] @@ -309,6 +316,8 @@ pub trait CompilerOpts { /// complex constants, and into (com ...) forms. This allows the CompilerOpts /// to carry this info across boundaries into a new context. fn code_generator(&self) -> Option; + /// Get the dialect declared in the toplevel program. + fn dialect(&self) -> AcceptedDialect; /// Specifies whether code is being generated on behalf of an inner defun in /// the program. fn in_defun(&self) -> bool; @@ -333,6 +342,8 @@ pub trait CompilerOpts { /// Specifies the search paths we're carrying. fn get_search_paths(&self) -> Vec; + /// Set the dialect. + fn set_dialect(&self, dialect: AcceptedDialect) -> Rc; /// Set search paths. fn set_search_paths(&self, dirs: &[String]) -> Rc; /// Set whether we're compiling on behalf of a defun. diff --git a/src/tests/classic/stage_2.rs b/src/tests/classic/stage_2.rs index 0d80b005a..d7055c6d3 100644 --- a/src/tests/classic/stage_2.rs +++ b/src/tests/classic/stage_2.rs @@ -17,7 +17,7 @@ use crate::classic::clvm_tools::stages::stage_2::helpers::{brun, evaluate, quote use crate::classic::clvm_tools::stages::stage_2::operators::run_program_for_search_paths; use crate::classic::clvm_tools::stages::stage_2::reader::{process_embed_file, read_file}; -use crate::compiler::comptypes::{CompileErr, CompilerOpts, PrimaryCodegen}; +use crate::compiler::comptypes::{AcceptedDialect, CompileErr, CompilerOpts, PrimaryCodegen}; use crate::compiler::sexp::{decode_string, SExp}; use crate::compiler::srcloc::Srcloc; @@ -319,6 +319,9 @@ impl CompilerOpts for TestCompilerOptsPresentsOwnFiles { fn code_generator(&self) -> Option { None } + fn dialect(&self) -> AcceptedDialect { + Default::default() + } fn in_defun(&self) -> bool { false } @@ -343,6 +346,9 @@ impl CompilerOpts for TestCompilerOptsPresentsOwnFiles { fn get_search_paths(&self) -> Vec { vec![".".to_string()] } + fn set_dialect(&self, _dialect: AcceptedDialect) -> Rc { + Rc::new(self.clone()) + } fn set_search_paths(&self, _dirs: &[String]) -> Rc { Rc::new(self.clone()) } From f82fe4c1102244c0be5a2649224d64191a85183c Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 23 May 2023 16:50:26 -0700 Subject: [PATCH 21/81] fmt + clippy --- src/classic/clvm_tools/clvmc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index e18805338..0eeffed9f 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -22,7 +22,7 @@ use crate::classic::platform::distutils::dep_util::newer; use crate::compiler::clvm::convert_to_clvm_rs; use crate::compiler::compiler::compile_file; -use crate::compiler::compiler::{DefaultCompilerOpts, run_optimizer}; +use crate::compiler::compiler::{run_optimizer, DefaultCompilerOpts}; use crate::compiler::comptypes::{AcceptedDialect, CompileErr, CompilerOpts}; use crate::compiler::runtypes::RunFailure; From ca4e65a7985733ff52183c2c32113288fcd8ba27 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 25 May 2023 17:29:02 -0700 Subject: [PATCH 22/81] Merge up the easy parts --- resources/tests/strict/strict-list-fail.clsp | 4 + resources/tests/strict/strict-list-pass.clsp | 4 + resources/tests/strict/strict-test-fail.clsp | 9 ++ resources/tests/strict/strict-test-pass.clsp | 7 ++ src/classic/clvm_tools/clvmc.rs | 68 ++--------- src/classic/clvm_tools/cmds.rs | 15 ++- src/compiler/codegen.rs | 44 ++++--- src/compiler/compiler.rs | 62 ++++++---- src/compiler/comptypes.rs | 56 ++++++--- src/compiler/dialect.rs | 119 +++++++++++++++++++ src/compiler/frontend.rs | 2 + src/compiler/mod.rs | 3 +- src/compiler/preprocessor/mod.rs | 7 +- src/compiler/rename.rs | 12 +- src/tests/classic/run.rs | 108 +++++++++++++++++ src/tests/classic/stage_2.rs | 3 +- 16 files changed, 391 insertions(+), 132 deletions(-) create mode 100644 resources/tests/strict/strict-list-fail.clsp create mode 100644 resources/tests/strict/strict-list-pass.clsp create mode 100644 resources/tests/strict/strict-test-fail.clsp create mode 100644 resources/tests/strict/strict-test-pass.clsp create mode 100644 src/compiler/dialect.rs diff --git a/resources/tests/strict/strict-list-fail.clsp b/resources/tests/strict/strict-list-fail.clsp new file mode 100644 index 000000000..9e4e42502 --- /dev/null +++ b/resources/tests/strict/strict-list-fail.clsp @@ -0,0 +1,4 @@ +(mod (X) + (include *strict-cl-21*) + (list X (+ X 1) (+ X2)) + ) diff --git a/resources/tests/strict/strict-list-pass.clsp b/resources/tests/strict/strict-list-pass.clsp new file mode 100644 index 000000000..efbba4770 --- /dev/null +++ b/resources/tests/strict/strict-list-pass.clsp @@ -0,0 +1,4 @@ +(mod (X) + (include *strict-cl-21*) + (list X (+ X 1) (+ X 2)) + ) diff --git a/resources/tests/strict/strict-test-fail.clsp b/resources/tests/strict/strict-test-fail.clsp new file mode 100644 index 000000000..4ca88390b --- /dev/null +++ b/resources/tests/strict/strict-test-fail.clsp @@ -0,0 +1,9 @@ +(mod (X) + (include *strict-cl-21*) + ;; This wouldn't be able to be rejected because X1 is coming from a macro + ;; expansion. This should fail in strict but succeed wrong non-strict. + (if X + (+ X1 2) + 5 + ) + ) diff --git a/resources/tests/strict/strict-test-pass.clsp b/resources/tests/strict/strict-test-pass.clsp new file mode 100644 index 000000000..6dfbc39de --- /dev/null +++ b/resources/tests/strict/strict-test-pass.clsp @@ -0,0 +1,7 @@ +(mod (X) + (include *strict-cl-21*) + (if X + (+ X 2) + 5 + ) + ) diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index 0eeffed9f..f8ae6a6bc 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -6,12 +6,11 @@ use std::rc::Rc; use tempfile::NamedTempFile; -use clvm_rs::allocator::{Allocator, NodePtr, SExp}; +use clvm_rs::allocator::{Allocator, NodePtr}; use clvm_rs::reduction::EvalErr; use crate::classic::clvm::__type_compatibility__::Stream; use crate::classic::clvm::serialize::sexp_to_stream; -use crate::classic::clvm::sexp::proper_list; use crate::classic::clvm_tools::binutils::{assemble_from_ir, disassemble}; use crate::classic::clvm_tools::ir::reader::read_ir; use crate::classic::clvm_tools::stages::run; @@ -23,25 +22,10 @@ use crate::classic::platform::distutils::dep_util::newer; use crate::compiler::clvm::convert_to_clvm_rs; use crate::compiler::compiler::compile_file; use crate::compiler::compiler::{run_optimizer, DefaultCompilerOpts}; -use crate::compiler::comptypes::{AcceptedDialect, CompileErr, CompilerOpts}; +use crate::compiler::comptypes::{CompileErr, CompilerOpts}; +use crate::compiler::dialect::detect_modern; use crate::compiler::runtypes::RunFailure; -fn include_dialect( - allocator: &mut Allocator, - dialects: &HashMap, i32>, - e: &[NodePtr], -) -> Option { - if let (SExp::Atom(inc), SExp::Atom(name)) = (allocator.sexp(e[0]), allocator.sexp(e[1])) { - if allocator.buf(&inc) == "include".as_bytes().to_vec() { - if let Some(dialect) = dialects.get(allocator.buf(&name)) { - return Some(*dialect); - } - } - } - - None -} - pub fn write_sym_output( compiled_lookup: &HashMap, path: &str, @@ -54,44 +38,7 @@ pub fn write_sym_output( .map(|_| ()) } -pub fn detect_modern(allocator: &mut Allocator, sexp: NodePtr) -> AcceptedDialect { - let mut dialects = HashMap::new(); - dialects.insert("*standard-cl-21*".as_bytes().to_vec(), 21); - dialects.insert("*standard-cl-22*".as_bytes().to_vec(), 22); - - let mut result = Default::default(); - - if let Some(l) = proper_list(allocator, sexp, true) { - for elt in l.iter() { - let detect_modern_result = detect_modern(allocator, *elt); - if detect_modern_result.stepping.is_some() { - result = detect_modern_result; - break; - } - - match proper_list(allocator, *elt, true) { - None => { - continue; - } - - Some(e) => { - if e.len() != 2 { - continue; - } - - if let Some(dialect) = include_dialect(allocator, &dialects, &e) { - result.stepping = Some(dialect); - break; - } - } - } - } - } - - result -} - -pub fn compile_clvm_text( +pub fn compile_clvm_text_maybe_opt( allocator: &mut Allocator, opts: Rc, symbol_table: &mut HashMap, @@ -102,9 +49,12 @@ pub fn compile_clvm_text( let ir_src = read_ir(text).map_err(|s| EvalErr(allocator.null(), s.to_string()))?; let assembled_sexp = assemble_from_ir(allocator, Rc::new(ir_src))?; let dialect = detect_modern(allocator, assembled_sexp); - if let Some(dialect) = dialect.stepping { + if let Some(stepping) = dialect.stepping { let runner = Rc::new(DefaultProgramRunner::new()); - let opts = opts.set_optimize(true).set_frontend_opt(dialect > 21); + let opts = opts + .set_dialect(dialect) + .set_optimize(do_optimize) + .set_frontend_opt(stepping > 21); let unopt_res = compile_file(allocator, runner.clone(), opts, text, symbol_table); let res = unopt_res.and_then(|x| run_optimizer(allocator, runner, Rc::new(x))); diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index dea8a5746..7238d0de1 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -27,7 +27,7 @@ use crate::classic::clvm::keyword_from_atom; use crate::classic::clvm::serialize::{sexp_from_stream, sexp_to_stream, SimpleCreateCLVMObject}; use crate::classic::clvm::sexp::{enlist, proper_list, sexp_as_bin}; use crate::classic::clvm_tools::binutils::{assemble_from_ir, disassemble, disassemble_with_kw}; -use crate::classic::clvm_tools::clvmc::{detect_modern, write_sym_output}; +use crate::classic::clvm_tools::clvmc::write_sym_output; use crate::classic::clvm_tools::debug::check_unused; use crate::classic::clvm_tools::debug::{ program_hash_from_program_env_cons, start_log_after, trace_pre_eval, trace_to_table, @@ -53,6 +53,8 @@ use crate::compiler::clvm::start_step; use crate::compiler::compiler::{compile_file, run_optimizer, DefaultCompilerOpts}; use crate::compiler::comptypes::{CompileErr, CompilerOpts}; use crate::compiler::debug::build_symbol_table_mut; +use crate::compiler::dialect::detect_modern; +use crate::compiler::frontend::frontend; use crate::compiler::preprocessor::gather_dependencies; use crate::compiler::prims; use crate::compiler::runtypes::RunFailure; @@ -910,6 +912,12 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .set_type(Rc::new(PathJoin {})) .set_default(ArgumentValue::ArgString(None, "main.sym".to_string())), ); + parser.add_argument( + vec!["--strict".to_string()], + Argument::new() + .set_action(TArgOptionAction::StoreTrue) + .set_help("For modern dialects, don't treat unknown names as constants".to_string()), + ); if tool_name == "run" { parser.add_argument( @@ -1183,7 +1191,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .unwrap_or_else(|| "main.sym".to_string()); // In testing: short circuit for modern compilation. - if let Some(dialect) = dialect.and_then(|d| d.stepping) { + if let Some(stepping) = dialect.as_ref().and_then(|d| d.stepping) { let do_optimize = parsed_args .get("optimize") .map(|x| matches!(x, ArgumentValue::ArgBool(true))) @@ -1191,9 +1199,10 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let runner = Rc::new(DefaultProgramRunner::new()); let use_filename = input_file.unwrap_or_else(|| "*command*".to_string()); let opts = Rc::new(DefaultCompilerOpts::new(&use_filename)) + .set_dialect(dialect.unwrap_or_else(|| Default::default())) .set_optimize(do_optimize) .set_search_paths(&search_paths) - .set_frontend_opt(dialect > 21); + .set_frontend_opt(stepping > 21); let mut symbol_table = HashMap::new(); let unopt_res = compile_file( diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 20fff0062..55de7d0e0 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -526,6 +526,10 @@ pub fn generate_expr_code( create_name_lookup(compiler, l.clone(), atom) .map(|f| Ok(CompiledCode(l.clone(), f))) .unwrap_or_else(|_| { + if opts.dialect().strict { + return Err(CompileErr(l.clone(), format!("Unbound use of {} as a variable name", decode_string(atom)))); + } + // Pass through atoms that don't look up on behalf of // macros, as it's possible that a macro returned // something that's canonically a name in number form. @@ -539,20 +543,32 @@ pub fn generate_expr_code( }) } } - // Since macros are in this language and the runtime has - // a very narrow data representation, we'll need to - // accomodate bare numbers coming back in place of identifiers. - // I'm considering ways to make this better. - SExp::Integer(l, i) => generate_expr_code( - allocator, - runner, - opts, - compiler, - Rc::new(BodyForm::Value(SExp::Atom( - l.clone(), - u8_from_number(i.clone()), - ))), - ), + SExp::Integer(l, i) => { + if opts.dialect().strict { + return generate_expr_code( + allocator, + runner, + opts, + compiler, + Rc::new(BodyForm::Quoted(SExp::Integer(l.clone(), i.clone()))), + ); + } + + // Since macros are in this language and the runtime has + // a very narrow data representation, we'll need to + // accomodate bare numbers coming back in place of identifiers, + // but only in legacy non-strict mode. + generate_expr_code( + allocator, + runner, + opts, + compiler, + Rc::new(BodyForm::Value(SExp::Atom( + l.clone(), + u8_from_number(i.clone()), + ))), + ) + } _ => Ok(CompiledCode( v.loc(), Rc::new(primquote(v.loc(), Rc::new(v.clone()))), diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 0bab69b35..13114ce0a 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -14,8 +14,9 @@ use crate::classic::clvm_tools::stages::stage_2::optimize::optimize_sexp; use crate::compiler::clvm::{convert_from_clvm_rs, convert_to_clvm_rs, sha256tree}; use crate::compiler::codegen::{codegen, hoist_body_let_binding, process_helper_let_bindings}; use crate::compiler::comptypes::{ - AcceptedDialect, CompileErr, CompileForm, CompilerOpts, DefunData, HelperForm, PrimaryCodegen, + CompileErr, CompileForm, CompilerOpts, DefunData, HelperForm, PrimaryCodegen, }; +use crate::compiler::dialect::{AcceptedDialect, KNOWN_DIALECTS}; use crate::compiler::evaluate::{build_reflex_captures, Evaluator, EVAL_STACK_LIMIT}; use crate::compiler::frontend::frontend; use crate::compiler::prims; @@ -25,24 +26,6 @@ use crate::compiler::srcloc::Srcloc; use crate::util::Number; lazy_static! { - pub static ref KNOWN_DIALECTS: HashMap = { - let mut known_dialects: HashMap = HashMap::new(); - known_dialects.insert( - "*standard-cl-21*".to_string(), - indoc! {"( - (defconstant *chialisp-version* 21) - )"} - .to_string(), - ); - known_dialects.insert( - "*standard-cl-22*".to_string(), - indoc! {"( - (defconstant *chialisp-version* 22) - )"} - .to_string(), - ); - known_dialects - }; pub static ref STANDARD_MACROS: String = { indoc! {"( (defmacro if (A B C) (qq (a (i (unquote A) (com (unquote B)) (com (unquote C))) @))) @@ -60,6 +43,34 @@ lazy_static! { "} .to_string() }; + + pub static ref ADVANCED_MACROS: String = { + indoc! {"( + (defmac if (A B C) + (qq (a (i (unquote A) (com (unquote B)) (com (unquote C))) @)) + ) + + (defun __chia__compile-list (args) + (if args + (c 4 (c (f args) (c (__chia__compile-list (r args)) ()))) + () + ) + ) + + (defmac list ARGS (__chia__compile-list ARGS)) + + (defun-inline / (A B) (f (divmod A B))) + (defun-inline c* (A B) (c A B)) + (defun-inline a* (A B) (a A B)) + (defun-inline coerce (X) : (Any -> Any) X) + (defun-inline explode (X) : (forall a ((Exec a) -> a)) X) + (defun-inline bless (X) : (forall a ((Pair a Unit) -> (Exec a))) (coerce X)) + (defun-inline lift (X V) : (forall a (forall b ((Pair (Exec a) (Pair b Unit)) -> (Exec (Pair a b))))) (coerce X)) + (defun-inline unlift (X) : (forall a (forall b ((Pair (Exec (Pair a b)) Unit) -> (Exec b)))) (coerce X)) + ) + "} + .to_string() + }; } #[derive(Clone, Debug)] @@ -75,8 +86,6 @@ pub struct DefaultCompilerOpts { pub start_env: Option>, pub prim_map: Rc, Rc>>, pub dialect: AcceptedDialect, - - known_dialects: Rc>, } pub fn create_prim_map() -> Rc, Rc>> { @@ -309,9 +318,13 @@ impl CompilerOpts for DefaultCompilerOpts { filename: String, ) -> Result<(String, String), CompileErr> { if filename == "*macros*" { - return Ok((filename, STANDARD_MACROS.clone())); - } else if let Some(content) = self.known_dialects.get(&filename) { - return Ok((filename, content.to_string())); + if self.dialect().strict { + return Ok((filename, ADVANCED_MACROS.clone().as_bytes().to_vec())); + } else { + return Ok((filename, STANDARD_MACROS.clone().as_bytes().to_vec())); + } + } else if let Some(dialect) = KNOWN_DIALECTS.get(&filename) { + return Ok((filename, dialect.content.as_bytes().to_vec())); } for dir in self.include_dirs.iter() { @@ -360,7 +373,6 @@ impl DefaultCompilerOpts { start_env: None, dialect: Default::default(), prim_map: create_prim_map(), - known_dialects: Rc::new(KNOWN_DIALECTS.clone()), } } } diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 7febd5b2f..1fe5b8eef 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -9,8 +9,9 @@ use clvm_rs::allocator::Allocator; use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType}; use crate::classic::clvm_tools::stages::stage_0::TRunProgram; -use crate::compiler::clvm::sha256tree; -use crate::compiler::sexp::{decode_string, SExp}; +use crate::compiler::clvm::{sha256tree, truthy}; +use crate::compiler::dialect::AcceptedDialect; +use crate::compiler::sexp::{decode_string, enlist, SExp}; use crate::compiler::srcloc::Srcloc; /// The basic error type. It contains a Srcloc identifying coordinates of the @@ -30,13 +31,6 @@ impl From<(Srcloc, String)> for CompileErr { #[derive(Clone, Debug)] pub struct CompiledCode(pub Srcloc, pub Rc); -/// Specifying how the language is spoken. -#[derive(Clone, Debug, Default)] -pub struct AcceptedDialect { - pub stepping: Option, - pub strict: bool, -} - /// A description of an inlined function for use during inline expansion. /// This is used only by PrimaryCodegen. #[derive(Clone, Debug)] @@ -192,6 +186,8 @@ pub struct DefmacData { pub args: Rc, /// The program appearing in the macro definition. pub program: Rc, + /// Whether this is an an advanced macro. + pub advanced: bool } /// Information from a constant definition. @@ -494,6 +490,38 @@ impl CompileForm { } } +pub fn generate_defmacro_sexp(mac: &DefmacData) -> Rc { + if mac.advanced { + Rc::new(SExp::Cons( + mac.loc.clone(), + Rc::new(SExp::atom_from_string(mac.loc.clone(), "defmac")), + Rc::new(SExp::Cons( + mac.loc.clone(), + Rc::new(SExp::atom_from_vec(mac.nl.clone(), &mac.name)), + Rc::new(SExp::Cons( + mac.loc.clone(), + mac.args.clone(), + Rc::new(SExp::Cons( + mac.loc.clone(), + mac.program.exp.to_sexp(), + Rc::new(SExp::Nil(mac.loc.clone())) + )), + )), + )), + )) + } else { + Rc::new(SExp::Cons( + mac.loc.clone(), + Rc::new(SExp::atom_from_string(mac.loc.clone(), "defmacro")), + Rc::new(SExp::Cons( + mac.loc.clone(), + Rc::new(SExp::atom_from_vec(mac.nl.clone(), &mac.name)), + mac.program.to_sexp(), + )), + )) + } +} + impl HelperForm { /// Get a reference to the HelperForm's name. pub fn name(&self) -> &Vec { @@ -544,15 +572,7 @@ impl HelperForm { ], )), }, - HelperForm::Defmacro(mac) => Rc::new(SExp::Cons( - mac.loc.clone(), - Rc::new(SExp::atom_from_string(mac.loc.clone(), "defmacro")), - Rc::new(SExp::Cons( - mac.loc.clone(), - Rc::new(SExp::atom_from_vec(mac.nl.clone(), &mac.name)), - mac.program.to_sexp(), - )), - )), + HelperForm::Defmacro(mac) => generate_defmacro_sexp(&mac), HelperForm::Defun(inline, defun) => { let di_string = "defun-inline".to_string(); let d_string = "defun".to_string(); diff --git a/src/compiler/dialect.rs b/src/compiler/dialect.rs new file mode 100644 index 000000000..c5b533d87 --- /dev/null +++ b/src/compiler/dialect.rs @@ -0,0 +1,119 @@ +use std::collections::HashMap; + +use clvmr::allocator::{Allocator, NodePtr, SExp}; + +use crate::classic::clvm::sexp::proper_list; + +use crate::compiler::sexp::decode_string; + +/// Specifying how the language is spoken. +#[derive(Clone, Debug, Default)] +pub struct AcceptedDialect { + pub stepping: Option, + pub strict: bool, +} + +/// A package containing the content we should insert when a dialect include is +/// used, plus the compilation flags. +#[derive(Clone, Debug)] +pub struct DialectDescription { + pub accepted: AcceptedDialect, + pub content: String +} + +lazy_static! { + pub static ref KNOWN_DIALECTS: HashMap = { + let mut dialects: HashMap = HashMap::new(); + let dialect_list = [ + ("*standard-cl-21*", DialectDescription { + accepted: AcceptedDialect { + stepping: Some(21), + .. Default::default() + }, + content: indoc! {"( + (defconstant *chialisp-version* 21) + )"}.to_string() + }), + ("*strict-cl-21*", DialectDescription { + accepted: AcceptedDialect { + stepping: Some(21), + strict: true + }, + content: indoc! {"( + (defconstant *chialisp-version* 22) + )"}.to_string() + }), + ("*standard-cl-22*", DialectDescription { + accepted: AcceptedDialect { + stepping: Some(22), + strict: false + }, + content: indoc! {"( + (defconstant *chialisp-version* 22) + )"}.to_string() + }), + ("*standard-cl-23*", DialectDescription { + accepted: AcceptedDialect { + stepping: Some(23), + strict: true + }, + content: indoc! {"( + (defconstant *chialisp-version* 23) + )"}.to_string() + }), + ]; + for (n,v) in dialect_list.iter() { + dialects.insert(n.to_string(), v.clone()); + } + dialects + }; +} + +fn include_dialect( + allocator: &mut Allocator, + e: &[NodePtr], +) -> Option +{ + if let (SExp::Atom(inc), SExp::Atom(name)) = (allocator.sexp(e[0]), allocator.sexp(e[1])) { + if allocator.buf(&inc) == "include".as_bytes().to_vec() { + if let Some(dialect) = KNOWN_DIALECTS.get(&decode_string(&allocator.buf(&name))) { + return Some(dialect.accepted.clone()); + } + } + } + + None +} + +pub fn detect_modern(allocator: &mut Allocator, sexp: NodePtr) -> AcceptedDialect { + let mut result = Default::default(); + + if let Some(l) = proper_list(allocator, sexp, true) { + for elt in l.iter() { + let detect_modern_result = detect_modern(allocator, *elt); + if detect_modern_result.stepping.is_some() { + result = detect_modern_result; + break; + } + + match proper_list(allocator, *elt, true) { + None => { + continue; + } + + Some(e) => { + if e.len() != 2 { + continue; + } + + if let Some(dialect) = include_dialect(allocator, &e) { + result = dialect; + break; + } + } + } + } + } + + result +} diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index b493d4559..f552147fd 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -478,6 +478,7 @@ fn compile_defmacro( name, args: args.clone(), program: Rc::new(p), + advanced: false, }) }) } @@ -695,6 +696,7 @@ fn frontend_start( )) } else { let l = pre_forms[0].loc(); + eprintln!("pre_forms {}", pre_forms[0]); pre_forms[0] .proper_list() .map(|x| { diff --git a/src/compiler/mod.rs b/src/compiler/mod.rs index 32e12ad78..e37a2f70d 100644 --- a/src/compiler/mod.rs +++ b/src/compiler/mod.rs @@ -13,8 +13,9 @@ pub mod compiler; /// - CompileForm - The type of finished (mod ) forms before code generation. /// - HelperForm - The type of declarations like macros, constants and functions. pub mod comptypes; -/// pub mod debug; +/// Dialect definitions. +pub mod dialect; pub mod evaluate; pub mod frontend; pub mod gensym; diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 77d77d94d..aa4fd6595 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -8,8 +8,11 @@ use clvmr::allocator::Allocator; use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; -use crate::compiler::compiler::KNOWN_DIALECTS; -use crate::compiler::comptypes::{BodyForm, CompileErr, CompilerOpts, HelperForm, IncludeDesc}; +use crate::compiler::cldb::hex_to_modern_sexp; +use crate::compiler::dialect::KNOWN_DIALECTS; +use crate::compiler::comptypes::{ + BodyForm, CompileErr, CompilerOpts, HelperForm, IncludeDesc, +}; use crate::compiler::evaluate::{create_argument_captures, dequote, ArgInputs, Evaluator}; use crate::compiler::frontend::compile_helperform; use crate::compiler::preprocessor::macros::PreprocessorExtension; diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 9b2e42ed5..81a1d8b23 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -271,12 +271,9 @@ fn rename_in_helperform(namemap: &HashMap, Vec>, h: &HelperForm) -> body: Rc::new(rename_in_bodyform(namemap, defc.body.clone())), }), HelperForm::Defmacro(mac) => HelperForm::Defmacro(DefmacData { - loc: mac.loc.clone(), - kw: mac.kw.clone(), - nl: mac.nl.clone(), - name: mac.name.to_vec(), - args: mac.args.clone(), program: Rc::new(rename_in_compileform(namemap, mac.program.clone())), + ..mac.clone() + }), HelperForm::Defun(inline, defun) => HelperForm::Defun( *inline, @@ -315,15 +312,12 @@ fn rename_args_helperform(h: &HelperForm) -> HelperForm { let local_renamed_arg = rename_in_cons(&local_namemap, mac.args.clone()); let local_renamed_body = rename_args_compileform(mac.program.borrow()); HelperForm::Defmacro(DefmacData { - loc: mac.loc.clone(), - kw: mac.kw.clone(), - nl: mac.nl.clone(), - name: mac.name.clone(), args: local_renamed_arg, program: Rc::new(rename_in_compileform( &local_namemap, Rc::new(local_renamed_body), )), + .. mac.clone() }) } HelperForm::Defun(inline, defun) => { diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index 6a849743a..0fb736ec0 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -869,3 +869,111 @@ fn test_cost_reporting_0() { "cost = 1978\n0x6fcb06b1fe29d132bb37f3a21b86d7cf03d636bf6230aa206486bef5e68f9875" ); } + +#[test] +fn test_assign_lambda_code_generation() { + let tname = "test_assign_lambda_code_generation.sym".to_string(); + do_basic_run(&vec![ + "run".to_string(), + "--extra-syms".to_string(), + "--symbol-output-file".to_string(), + tname.clone(), + "(mod (A) (include *standard-cl-21*) (defun F (X) (+ X 1)) (assign-lambda X (F A) X))" + .to_string(), + ]); + let read_in_file = fs::read_to_string(&tname).expect("should have dropped symbols"); + fs::remove_file(&tname).expect("should have existed"); + let decoded_symbol_file: HashMap = + serde_json::from_str(&read_in_file).expect("should decode"); + let found_wanted_symbols: Vec = decoded_symbol_file + .iter() + .filter(|(_, v)| *v == "F" || v.starts_with("letbinding")) + .map(|(k, _)| k.clone()) + .collect(); + assert_eq!(found_wanted_symbols.len(), 2); + // We should have these two functions. + assert!(found_wanted_symbols + .contains(&"ccd5be506752cebf01f9930b4c108fe18058c65e1ab57a72ca0a00d9788c7ca6".to_string())); + assert!(found_wanted_symbols + .contains(&"0a5af5ae61fae2e53cb309d4d9c2c64baf0261824823008b9cf2b21b09221e44".to_string())); +} + +#[test] +fn test_assign_lambda_code_generation_inline() { + let tname = "test_assign_inline_code_generation.sym".to_string(); + do_basic_run(&vec![ + "run".to_string(), + "--extra-syms".to_string(), + "--symbol-output-file".to_string(), + tname.clone(), + "(mod (A) (include *standard-cl-21*) (defun F (X) (+ X 1)) (assign-inline X (F A) X))" + .to_string(), + ]); + let read_in_file = fs::read_to_string(&tname).expect("should have dropped symbols"); + fs::remove_file(&tname).expect("should have existed"); + let decoded_symbol_file: HashMap = + serde_json::from_str(&read_in_file).expect("should decode"); + let found_wanted_symbols: Vec = decoded_symbol_file + .iter() + .filter(|(_, v)| *v == "F" || v.starts_with("letbinding")) + .map(|(k, _)| k.clone()) + .collect(); + assert_eq!(found_wanted_symbols.len(), 1); + // We should have these two functions. + assert!(found_wanted_symbols + .contains(&"ccd5be506752cebf01f9930b4c108fe18058c65e1ab57a72ca0a00d9788c7ca6".to_string())); +} + +#[test] +fn test_assign_fancy_final_dot_rest() { + let result_prog = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/chia-gaming".to_string(), + "resources/tests/chia-gaming/test-last.clsp".to_string(), + ]); + let result = do_basic_brun(&vec!["brun".to_string(), result_prog, "()".to_string()]) + .trim() + .to_string(); + assert_eq!(result, "101"); +} + +#[test] +fn test_strict_smoke_0() { + let result = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + "resources/tests/strict/strict-test-fail.clsp".to_string(), + ]); + assert!(result.contains("Unbound")); + assert!(result.contains("X1")); +} + +#[test] +fn test_strict_smoke_1() { + let result_prog = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + "resources/tests/strict/strict-test-pass.clsp".to_string(), + ]); + let result = do_basic_brun(&vec!["brun".to_string(), result_prog, "(13)".to_string()]) + .trim() + .to_string(); + assert_eq!(result, "15"); +} + +/* +#[test] +fn test_strict_list_fail() { + let result_prog = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + "resources/tests/strict/strict-list-fail.clsp".to_string(), + ]); + eprintln!("result_prog {result_prog}"); + todo!(); +} +*/ diff --git a/src/tests/classic/stage_2.rs b/src/tests/classic/stage_2.rs index d7055c6d3..a83846b9f 100644 --- a/src/tests/classic/stage_2.rs +++ b/src/tests/classic/stage_2.rs @@ -17,7 +17,8 @@ use crate::classic::clvm_tools::stages::stage_2::helpers::{brun, evaluate, quote use crate::classic::clvm_tools::stages::stage_2::operators::run_program_for_search_paths; use crate::classic::clvm_tools::stages::stage_2::reader::{process_embed_file, read_file}; -use crate::compiler::comptypes::{AcceptedDialect, CompileErr, CompilerOpts, PrimaryCodegen}; +use crate::compiler::comptypes::{CompileErr, CompilerOpts, PrimaryCodegen}; +use crate::compiler::dialect::AcceptedDialect; use crate::compiler::sexp::{decode_string, SExp}; use crate::compiler::srcloc::Srcloc; From 7341d4085f7be51d0f7fee304c3f5f9986449e22 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 29 May 2023 16:13:00 -0700 Subject: [PATCH 23/81] fmt + clippy --- src/classic/clvm_tools/clvmc.rs | 20 ++++++++ src/classic/clvm_tools/cmds.rs | 3 +- src/compiler/codegen.rs | 8 ++- src/compiler/compiler.rs | 7 ++- src/compiler/comptypes.rs | 10 ++-- src/compiler/dialect.rs | 86 ++++++++++++++++++-------------- src/compiler/frontend.rs | 1 - src/compiler/preprocessor/mod.rs | 5 +- src/compiler/rename.rs | 3 +- src/tests/classic/run.rs | 68 ------------------------- 10 files changed, 87 insertions(+), 124 deletions(-) diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index f8ae6a6bc..73e40050e 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -40,6 +40,7 @@ pub fn write_sym_output( pub fn compile_clvm_text_maybe_opt( allocator: &mut Allocator, + do_optimize: bool, opts: Rc, symbol_table: &mut HashMap, text: &str, @@ -79,6 +80,25 @@ pub fn compile_clvm_text_maybe_opt( } } +pub fn compile_clvm_text( + allocator: &mut Allocator, + opts: Rc, + symbol_table: &mut HashMap, + text: &str, + input_path: &str, + classic_with_opts: bool, +) -> Result { + compile_clvm_text_maybe_opt( + allocator, + true, + opts, + symbol_table, + text, + input_path, + classic_with_opts, + ) +} + pub fn compile_clvm_inner( allocator: &mut Allocator, opts: Rc, diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index 7238d0de1..f8c2d7549 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -54,7 +54,6 @@ use crate::compiler::compiler::{compile_file, run_optimizer, DefaultCompilerOpts use crate::compiler::comptypes::{CompileErr, CompilerOpts}; use crate::compiler::debug::build_symbol_table_mut; use crate::compiler::dialect::detect_modern; -use crate::compiler::frontend::frontend; use crate::compiler::preprocessor::gather_dependencies; use crate::compiler::prims; use crate::compiler::runtypes::RunFailure; @@ -1199,7 +1198,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let runner = Rc::new(DefaultProgramRunner::new()); let use_filename = input_file.unwrap_or_else(|| "*command*".to_string()); let opts = Rc::new(DefaultCompilerOpts::new(&use_filename)) - .set_dialect(dialect.unwrap_or_else(|| Default::default())) + .set_dialect(dialect.unwrap_or_default()) .set_optimize(do_optimize) .set_search_paths(&search_paths) .set_frontend_opt(stepping > 21); diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 55de7d0e0..064512485 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -527,7 +527,13 @@ pub fn generate_expr_code( .map(|f| Ok(CompiledCode(l.clone(), f))) .unwrap_or_else(|_| { if opts.dialect().strict { - return Err(CompileErr(l.clone(), format!("Unbound use of {} as a variable name", decode_string(atom)))); + return Err(CompileErr( + l.clone(), + format!( + "Unbound use of {} as a variable name", + decode_string(atom) + ), + )); } // Pass through atoms that don't look up on behalf of diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 13114ce0a..ec9577096 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -43,7 +43,6 @@ lazy_static! { "} .to_string() }; - pub static ref ADVANCED_MACROS: String = { indoc! {"( (defmac if (A B C) @@ -319,12 +318,12 @@ impl CompilerOpts for DefaultCompilerOpts { ) -> Result<(String, String), CompileErr> { if filename == "*macros*" { if self.dialect().strict { - return Ok((filename, ADVANCED_MACROS.clone().as_bytes().to_vec())); + return Ok((filename, ADVANCED_MACROS.clone())); } else { - return Ok((filename, STANDARD_MACROS.clone().as_bytes().to_vec())); + return Ok((filename, STANDARD_MACROS.clone())); } } else if let Some(dialect) = KNOWN_DIALECTS.get(&filename) { - return Ok((filename, dialect.content.as_bytes().to_vec())); + return Ok((filename, dialect.content.clone())); } for dir in self.include_dirs.iter() { diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 1fe5b8eef..500590e74 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -9,9 +9,9 @@ use clvm_rs::allocator::Allocator; use crate::classic::clvm::__type_compatibility__::{Bytes, BytesFromType}; use crate::classic::clvm_tools::stages::stage_0::TRunProgram; -use crate::compiler::clvm::{sha256tree, truthy}; +use crate::compiler::clvm::sha256tree; use crate::compiler::dialect::AcceptedDialect; -use crate::compiler::sexp::{decode_string, enlist, SExp}; +use crate::compiler::sexp::{decode_string, SExp}; use crate::compiler::srcloc::Srcloc; /// The basic error type. It contains a Srcloc identifying coordinates of the @@ -187,7 +187,7 @@ pub struct DefmacData { /// The program appearing in the macro definition. pub program: Rc, /// Whether this is an an advanced macro. - pub advanced: bool + pub advanced: bool, } /// Information from a constant definition. @@ -504,7 +504,7 @@ pub fn generate_defmacro_sexp(mac: &DefmacData) -> Rc { Rc::new(SExp::Cons( mac.loc.clone(), mac.program.exp.to_sexp(), - Rc::new(SExp::Nil(mac.loc.clone())) + Rc::new(SExp::Nil(mac.loc.clone())), )), )), )), @@ -572,7 +572,7 @@ impl HelperForm { ], )), }, - HelperForm::Defmacro(mac) => generate_defmacro_sexp(&mac), + HelperForm::Defmacro(mac) => generate_defmacro_sexp(mac), HelperForm::Defun(inline, defun) => { let di_string = "defun-inline".to_string(); let d_string = "defun".to_string(); diff --git a/src/compiler/dialect.rs b/src/compiler/dialect.rs index c5b533d87..062cb47c2 100644 --- a/src/compiler/dialect.rs +++ b/src/compiler/dialect.rs @@ -18,65 +18,77 @@ pub struct AcceptedDialect { #[derive(Clone, Debug)] pub struct DialectDescription { pub accepted: AcceptedDialect, - pub content: String + pub content: String, } lazy_static! { pub static ref KNOWN_DIALECTS: HashMap = { let mut dialects: HashMap = HashMap::new(); let dialect_list = [ - ("*standard-cl-21*", DialectDescription { - accepted: AcceptedDialect { - stepping: Some(21), - .. Default::default() - }, - content: indoc! {"( + ( + "*standard-cl-21*", + DialectDescription { + accepted: AcceptedDialect { + stepping: Some(21), + ..Default::default() + }, + content: indoc! {"( (defconstant *chialisp-version* 21) - )"}.to_string() - }), - ("*strict-cl-21*", DialectDescription { - accepted: AcceptedDialect { - stepping: Some(21), - strict: true + )"} + .to_string(), }, - content: indoc! {"( + ), + ( + "*strict-cl-21*", + DialectDescription { + accepted: AcceptedDialect { + stepping: Some(21), + strict: true, + }, + content: indoc! {"( (defconstant *chialisp-version* 22) - )"}.to_string() - }), - ("*standard-cl-22*", DialectDescription { - accepted: AcceptedDialect { - stepping: Some(22), - strict: false + )"} + .to_string(), }, - content: indoc! {"( + ), + ( + "*standard-cl-22*", + DialectDescription { + accepted: AcceptedDialect { + stepping: Some(22), + strict: false, + }, + content: indoc! {"( (defconstant *chialisp-version* 22) - )"}.to_string() - }), - ("*standard-cl-23*", DialectDescription { - accepted: AcceptedDialect { - stepping: Some(23), - strict: true + )"} + .to_string(), }, - content: indoc! {"( + ), + ( + "*standard-cl-23*", + DialectDescription { + accepted: AcceptedDialect { + stepping: Some(23), + strict: true, + }, + content: indoc! {"( (defconstant *chialisp-version* 23) - )"}.to_string() - }), + )"} + .to_string(), + }, + ), ]; - for (n,v) in dialect_list.iter() { + for (n, v) in dialect_list.iter() { dialects.insert(n.to_string(), v.clone()); } dialects }; } -fn include_dialect( - allocator: &mut Allocator, - e: &[NodePtr], -) -> Option -{ +fn include_dialect(allocator: &mut Allocator, e: &[NodePtr]) -> Option { if let (SExp::Atom(inc), SExp::Atom(name)) = (allocator.sexp(e[0]), allocator.sexp(e[1])) { if allocator.buf(&inc) == "include".as_bytes().to_vec() { - if let Some(dialect) = KNOWN_DIALECTS.get(&decode_string(&allocator.buf(&name))) { + if let Some(dialect) = KNOWN_DIALECTS.get(&decode_string(allocator.buf(&name))) { return Some(dialect.accepted.clone()); } } diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index f552147fd..c4f6b80dd 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -696,7 +696,6 @@ fn frontend_start( )) } else { let l = pre_forms[0].loc(); - eprintln!("pre_forms {}", pre_forms[0]); pre_forms[0] .proper_list() .map(|x| { diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index aa4fd6595..cf33fd633 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -8,11 +8,8 @@ use clvmr::allocator::Allocator; use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; -use crate::compiler::cldb::hex_to_modern_sexp; +use crate::compiler::comptypes::{BodyForm, CompileErr, CompilerOpts, HelperForm, IncludeDesc}; use crate::compiler::dialect::KNOWN_DIALECTS; -use crate::compiler::comptypes::{ - BodyForm, CompileErr, CompilerOpts, HelperForm, IncludeDesc, -}; use crate::compiler::evaluate::{create_argument_captures, dequote, ArgInputs, Evaluator}; use crate::compiler::frontend::compile_helperform; use crate::compiler::preprocessor::macros::PreprocessorExtension; diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 81a1d8b23..73f75db1a 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -273,7 +273,6 @@ fn rename_in_helperform(namemap: &HashMap, Vec>, h: &HelperForm) -> HelperForm::Defmacro(mac) => HelperForm::Defmacro(DefmacData { program: Rc::new(rename_in_compileform(namemap, mac.program.clone())), ..mac.clone() - }), HelperForm::Defun(inline, defun) => HelperForm::Defun( *inline, @@ -317,7 +316,7 @@ fn rename_args_helperform(h: &HelperForm) -> HelperForm { &local_namemap, Rc::new(local_renamed_body), )), - .. mac.clone() + ..mac.clone() }) } HelperForm::Defun(inline, defun) => { diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index 0fb736ec0..b17bcaec8 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -870,74 +870,6 @@ fn test_cost_reporting_0() { ); } -#[test] -fn test_assign_lambda_code_generation() { - let tname = "test_assign_lambda_code_generation.sym".to_string(); - do_basic_run(&vec![ - "run".to_string(), - "--extra-syms".to_string(), - "--symbol-output-file".to_string(), - tname.clone(), - "(mod (A) (include *standard-cl-21*) (defun F (X) (+ X 1)) (assign-lambda X (F A) X))" - .to_string(), - ]); - let read_in_file = fs::read_to_string(&tname).expect("should have dropped symbols"); - fs::remove_file(&tname).expect("should have existed"); - let decoded_symbol_file: HashMap = - serde_json::from_str(&read_in_file).expect("should decode"); - let found_wanted_symbols: Vec = decoded_symbol_file - .iter() - .filter(|(_, v)| *v == "F" || v.starts_with("letbinding")) - .map(|(k, _)| k.clone()) - .collect(); - assert_eq!(found_wanted_symbols.len(), 2); - // We should have these two functions. - assert!(found_wanted_symbols - .contains(&"ccd5be506752cebf01f9930b4c108fe18058c65e1ab57a72ca0a00d9788c7ca6".to_string())); - assert!(found_wanted_symbols - .contains(&"0a5af5ae61fae2e53cb309d4d9c2c64baf0261824823008b9cf2b21b09221e44".to_string())); -} - -#[test] -fn test_assign_lambda_code_generation_inline() { - let tname = "test_assign_inline_code_generation.sym".to_string(); - do_basic_run(&vec![ - "run".to_string(), - "--extra-syms".to_string(), - "--symbol-output-file".to_string(), - tname.clone(), - "(mod (A) (include *standard-cl-21*) (defun F (X) (+ X 1)) (assign-inline X (F A) X))" - .to_string(), - ]); - let read_in_file = fs::read_to_string(&tname).expect("should have dropped symbols"); - fs::remove_file(&tname).expect("should have existed"); - let decoded_symbol_file: HashMap = - serde_json::from_str(&read_in_file).expect("should decode"); - let found_wanted_symbols: Vec = decoded_symbol_file - .iter() - .filter(|(_, v)| *v == "F" || v.starts_with("letbinding")) - .map(|(k, _)| k.clone()) - .collect(); - assert_eq!(found_wanted_symbols.len(), 1); - // We should have these two functions. - assert!(found_wanted_symbols - .contains(&"ccd5be506752cebf01f9930b4c108fe18058c65e1ab57a72ca0a00d9788c7ca6".to_string())); -} - -#[test] -fn test_assign_fancy_final_dot_rest() { - let result_prog = do_basic_run(&vec![ - "run".to_string(), - "-i".to_string(), - "resources/tests/chia-gaming".to_string(), - "resources/tests/chia-gaming/test-last.clsp".to_string(), - ]); - let result = do_basic_brun(&vec!["brun".to_string(), result_prog, "()".to_string()]) - .trim() - .to_string(); - assert_eq!(result, "101"); -} - #[test] fn test_strict_smoke_0() { let result = do_basic_run(&vec![ From 9f90b88ea25af6eb04fdfd1b08f912d8710d1baa Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 25 May 2023 17:49:45 -0700 Subject: [PATCH 24/81] Pull in a local add_helper method --- src/compiler/preprocessor/mod.rs | 52 +++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index cf33fd633..e28dbc98e 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -90,6 +90,56 @@ impl Preprocessor { Ok(()) } + /* + fn recurse_dependencies( + &mut self, + includes: &mut Vec, + desc: IncludeType, + ) -> Result<(), CompileErr> { + match desc { + IncludeType::Basic(desc) => { + let name_string = decode_string(&desc.name); + if KNOWN_DIALECTS.contains_key(&name_string) { + return Ok(()); + } + + let (full_name, content) = self.opts.read_new_file(self.opts.filename(), name_string)?; + includes.push(IncludeDesc { + name: full_name.as_bytes().to_vec(), + ..desc + }); + + let parsed = parse_sexp(Srcloc::start(&full_name), content.iter().copied())?; + if parsed.is_empty() { + return Ok(()); + } + + let program_form = parsed[0].clone(); + if let Some(l) = program_form.proper_list() { + for elt in l.iter() { + self.process_pp_form(includes, Rc::new(elt.clone()))?; + } + } + + Ok(()) + }, + _ => { todo!() } + } + } + */ + + fn add_helper(&mut self, h: HelperForm) { + for i in 0..=self.helpers.len() { + if i == self.helpers.len() { + self.helpers.push(h); + break; + } else if self.helpers[i].name() == h.name() { + self.helpers[i] = h; + break; + } + } + } + // Check for and apply preprocessor level macros. // This is maximally permissive. fn expand_macros(&mut self, body: Rc) -> Result, CompileErr> { @@ -184,7 +234,7 @@ impl Preprocessor { )); if let Some(helper) = compile_helperform(self.opts.clone(), target_defun)? { self.evaluator.add_helper(&helper); - self.helpers.push(helper); + self.add_helper(helper.clone()); } else { return Err(CompileErr( definition.loc(), From cd764e4de54ba0bfe5675f2b555917340d971ab0 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 25 May 2023 20:39:28 -0700 Subject: [PATCH 25/81] Speed up, reduce heap traffic --- src/compiler/preprocessor/mod.rs | 39 +++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index e28dbc98e..86122194f 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -142,14 +142,37 @@ impl Preprocessor { // Check for and apply preprocessor level macros. // This is maximally permissive. - fn expand_macros(&mut self, body: Rc) -> Result, CompileErr> { + fn expand_macros(&mut self, body: Rc, start: bool) -> Result>, CompileErr> { if let SExp::Cons(l, f, r) = body.borrow() { // First expand inner macros. - let first_expanded = self.expand_macros(f.clone())?; - let rest_expanded = self.expand_macros(r.clone())?; - let new_self = Rc::new(SExp::Cons(l.clone(), first_expanded, rest_expanded)); + let first_expanded = self.expand_macros(f.clone(), true)?; + let rest_expanded = self.expand_macros(r.clone(), false)?; + let new_self = + match (first_expanded, rest_expanded) { + (None, None) => None, + (Some(f), None) => Some(Rc::new(SExp::Cons( + l.clone(), + f.clone(), + r.clone() + ))), + (None, Some(r)) => Some(Rc::new(SExp::Cons( + l.clone(), + f.clone(), + r.clone() + ))), + (Some(f),Some(r)) => Some(Rc::new(SExp::Cons( + l.clone(), + f.clone(), + r.clone() + ))) + }; + + if !start { + return Ok(new_self); + } + if let Ok(NodeSel::Cons((_, name), args)) = - NodeSel::Cons(Atom::Here(()), ThisNode::Here).select_nodes(new_self.clone()) + NodeSel::Cons(Atom::Here(()), ThisNode::Here).select_nodes(new_self.clone().unwrap_or_else(|| body.clone())) { // See if it's a form that calls one of our macros. for m in self.helpers.iter() { @@ -183,7 +206,7 @@ impl Preprocessor { )?; if let Ok(unquoted) = dequote(body.loc(), res) { - return Ok(unquoted); + return Ok(Some(unquoted)); } else { return Err(CompileErr( body.loc(), @@ -197,7 +220,7 @@ impl Preprocessor { return Ok(new_self); } - Ok(body) + Ok(None) } // If it's a defmac (preprocessor level macro), add it to the evaulator. @@ -256,7 +279,7 @@ impl Preprocessor { includes: &mut Vec, unexpanded_body: Rc, ) -> Result>, CompileErr> { - let body = self.expand_macros(unexpanded_body)?; + let body = self.expand_macros(unexpanded_body.clone(), true)?.unwrap_or_else(|| unexpanded_body.clone()); // Support using the preprocessor to collect dependencies recursively. let included: Option = body .proper_list() From 8e254c24beca952a51fab9ed08d269133dfaf3d8 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 29 May 2023 17:05:46 -0700 Subject: [PATCH 26/81] fmt + clippy --- src/compiler/preprocessor/mod.rs | 38 ++++++++++++++------------------ 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 86122194f..ba6f5c9cd 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -142,37 +142,29 @@ impl Preprocessor { // Check for and apply preprocessor level macros. // This is maximally permissive. - fn expand_macros(&mut self, body: Rc, start: bool) -> Result>, CompileErr> { + fn expand_macros( + &mut self, + body: Rc, + start: bool, + ) -> Result>, CompileErr> { if let SExp::Cons(l, f, r) = body.borrow() { // First expand inner macros. let first_expanded = self.expand_macros(f.clone(), true)?; let rest_expanded = self.expand_macros(r.clone(), false)?; - let new_self = - match (first_expanded, rest_expanded) { - (None, None) => None, - (Some(f), None) => Some(Rc::new(SExp::Cons( - l.clone(), - f.clone(), - r.clone() - ))), - (None, Some(r)) => Some(Rc::new(SExp::Cons( - l.clone(), - f.clone(), - r.clone() - ))), - (Some(f),Some(r)) => Some(Rc::new(SExp::Cons( - l.clone(), - f.clone(), - r.clone() - ))) - }; + let new_self = match (first_expanded, rest_expanded) { + (None, None) => None, + (Some(f), None) => Some(Rc::new(SExp::Cons(l.clone(), f, r.clone()))), + (None, Some(r)) => Some(Rc::new(SExp::Cons(l.clone(), f.clone(), r))), + (Some(f), Some(r)) => Some(Rc::new(SExp::Cons(l.clone(), f, r))), + }; if !start { return Ok(new_self); } if let Ok(NodeSel::Cons((_, name), args)) = - NodeSel::Cons(Atom::Here(()), ThisNode::Here).select_nodes(new_self.clone().unwrap_or_else(|| body.clone())) + NodeSel::Cons(Atom::Here(()), ThisNode::Here) + .select_nodes(new_self.clone().unwrap_or_else(|| body.clone())) { // See if it's a form that calls one of our macros. for m in self.helpers.iter() { @@ -279,7 +271,9 @@ impl Preprocessor { includes: &mut Vec, unexpanded_body: Rc, ) -> Result>, CompileErr> { - let body = self.expand_macros(unexpanded_body.clone(), true)?.unwrap_or_else(|| unexpanded_body.clone()); + let body = self + .expand_macros(unexpanded_body.clone(), true)? + .unwrap_or_else(|| unexpanded_body.clone()); // Support using the preprocessor to collect dependencies recursively. let included: Option = body .proper_list() From 0d193ff17dbfe5da58762a927aa1bba6eb78b4f8 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 25 May 2023 20:52:36 -0700 Subject: [PATCH 27/81] Rename defuns so they don't overlap real uses when they occur --- src/compiler/preprocessor/mod.rs | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index ba6f5c9cd..a8db832b4 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -25,6 +25,27 @@ struct Preprocessor { helpers: Vec, } +fn compose_defconst(loc: Srcloc, name: &[u8], sexp: Rc) -> Rc { + Rc::new(enlist( + loc.clone(), + vec![ + Rc::new(SExp::Atom(loc.clone(), b"defconst".to_vec())), + Rc::new(SExp::Atom(loc.clone(), name.to_vec())), + Rc::new(SExp::Cons( + loc.clone(), + Rc::new(SExp::Atom(loc, vec![1])), + sexp, + )), + ], + )) +} + +fn make_defmac_name(name: &[u8]) -> Vec { + let mut res = b"__chia__defmac__".to_vec(); + res.append(&mut name.to_vec()); + res +} + impl Preprocessor { pub fn new(opts: Rc) -> Self { let runner = Rc::new(DefaultProgramRunner::new()); @@ -166,11 +187,13 @@ impl Preprocessor { NodeSel::Cons(Atom::Here(()), ThisNode::Here) .select_nodes(new_self.clone().unwrap_or_else(|| body.clone())) { + let defmac_name = make_defmac_name(&name); + // See if it's a form that calls one of our macros. for m in self.helpers.iter() { if let HelperForm::Defun(_, mdata) = &m { // We record upfront macros - if mdata.name != name { + if mdata.name != defmac_name { continue; } @@ -243,7 +266,7 @@ impl Preprocessor { Rc::new(SExp::atom_from_string(defmac_loc, "defun")), Rc::new(SExp::Cons( nl.clone(), - Rc::new(SExp::Atom(nl, name)), + Rc::new(SExp::Atom(nl, make_defmac_name(&name))), Rc::new(SExp::Cons(args.loc(), args.clone(), body)), )), )); From afaf3f0f949adaf709366afafb66a18d283fcad3 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 25 May 2023 20:56:54 -0700 Subject: [PATCH 28/81] cherry-pick + fmt + clippy --- src/compiler/preprocessor/mod.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index a8db832b4..a671a76fe 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -25,6 +25,7 @@ struct Preprocessor { helpers: Vec, } +/* fn compose_defconst(loc: Srcloc, name: &[u8], sexp: Rc) -> Rc { Rc::new(enlist( loc.clone(), @@ -39,6 +40,7 @@ fn compose_defconst(loc: Srcloc, name: &[u8], sexp: Rc) -> Rc { ], )) } +*/ fn make_defmac_name(name: &[u8]) -> Vec { let mut res = b"__chia__defmac__".to_vec(); @@ -150,6 +152,7 @@ impl Preprocessor { */ fn add_helper(&mut self, h: HelperForm) { + self.evaluator.add_helper(&h); for i in 0..=self.helpers.len() { if i == self.helpers.len() { self.helpers.push(h); @@ -271,8 +274,7 @@ impl Preprocessor { )), )); if let Some(helper) = compile_helperform(self.opts.clone(), target_defun)? { - self.evaluator.add_helper(&helper); - self.add_helper(helper.clone()); + self.add_helper(helper); } else { return Err(CompileErr( definition.loc(), @@ -280,7 +282,7 @@ impl Preprocessor { )); } } else if let Some(helper) = compile_helperform(self.opts.clone(), definition)? { - self.evaluator.add_helper(&helper); + self.add_helper(helper); } } } From ae6472d444f109fe0a13a536a71820a8751a4f82 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 25 May 2023 21:01:34 -0700 Subject: [PATCH 29/81] Remove persistent evaluator, use a local one. now there's only one helper list and we control it from here. --- src/compiler/preprocessor/mod.rs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index a671a76fe..373bbffd0 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -6,7 +6,10 @@ use std::rc::Rc; use clvmr::allocator::Allocator; -use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; +use crate::classic::clvm_tools::clvmc::compile_clvm_text_maybe_opt; +use crate::classic::clvm_tools::stages::stage_0::{DefaultProgramRunner, TRunProgram}; +use crate::compiler::clvm::convert_from_clvm_rs; +ne helper list and we control it from here.) use crate::compiler::comptypes::{BodyForm, CompileErr, CompilerOpts, HelperForm, IncludeDesc}; use crate::compiler::dialect::KNOWN_DIALECTS; @@ -21,7 +24,7 @@ use crate::util::ErrInto; struct Preprocessor { opts: Rc, - evaluator: Evaluator, + runner: Rc, helpers: Vec, } @@ -51,11 +54,9 @@ fn make_defmac_name(name: &[u8]) -> Vec { impl Preprocessor { pub fn new(opts: Rc) -> Self { let runner = Rc::new(DefaultProgramRunner::new()); - let mut eval = Evaluator::new(opts.clone(), runner, Vec::new()); - eval.add_extension(Rc::new(PreprocessorExtension::new())); Preprocessor { opts: opts.clone(), - evaluator: eval, + runner, helpers: Vec::new(), } } @@ -152,7 +153,6 @@ impl Preprocessor { */ fn add_helper(&mut self, h: HelperForm) { - self.evaluator.add_helper(&h); for i in 0..=self.helpers.len() { if i == self.helpers.len() { self.helpers.push(h); @@ -214,7 +214,9 @@ impl Preprocessor { mdata.args.clone(), )?; - let res = self.evaluator.shrink_bodyform( + let mut eval = Evaluator::new(self.opts.clone(), self.runner.clone(), self.helpers.clone()); + eval.add_extension(Rc::new(PreprocessorExtension::new())); + let res = eval.shrink_bodyform( &mut allocator, mdata.args.clone(), ¯o_arg_env, From affb0445de1283d41f32f60e58fdd8853c440d8a Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 29 May 2023 17:30:30 -0700 Subject: [PATCH 30/81] fmt --- src/compiler/preprocessor/mod.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 373bbffd0..0e3e37d9a 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -6,10 +6,7 @@ use std::rc::Rc; use clvmr::allocator::Allocator; -use crate::classic::clvm_tools::clvmc::compile_clvm_text_maybe_opt; use crate::classic::clvm_tools::stages::stage_0::{DefaultProgramRunner, TRunProgram}; -use crate::compiler::clvm::convert_from_clvm_rs; -ne helper list and we control it from here.) use crate::compiler::comptypes::{BodyForm, CompileErr, CompilerOpts, HelperForm, IncludeDesc}; use crate::compiler::dialect::KNOWN_DIALECTS; @@ -214,7 +211,11 @@ impl Preprocessor { mdata.args.clone(), )?; - let mut eval = Evaluator::new(self.opts.clone(), self.runner.clone(), self.helpers.clone()); + let mut eval = Evaluator::new( + self.opts.clone(), + self.runner.clone(), + self.helpers.clone(), + ); eval.add_extension(Rc::new(PreprocessorExtension::new())); let res = eval.shrink_bodyform( &mut allocator, From 4396883139abe0cb5473fb465dac0d5f43fe9929 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 26 May 2023 14:12:56 -0700 Subject: [PATCH 31/81] more moves --- src/compiler/evaluate.rs | 61 ++++++++++++++++++++++---------- src/compiler/preprocessor/mod.rs | 7 ++++ 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 893f20489..66d13bcdc 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -17,7 +17,7 @@ use crate::compiler::comptypes::{ }; use crate::compiler::frontend::frontend; use crate::compiler::runtypes::RunFailure; -use crate::compiler::sexp::SExp; +use crate::compiler::sexp::{enlist, SExp}; use crate::compiler::srcloc::Srcloc; use crate::compiler::stackvisit::{HasDepthLimit, VisitedMarker}; use crate::util::{number_from_u8, u8_from_number, Number}; @@ -700,6 +700,45 @@ impl<'info> Evaluator { } } + fn defmac_ordering(&self) -> bool { + let dialect = self.opts.dialect(); + dialect.strict || dialect.stepping.unwrap_or(21) > 22 + } + + fn make_com_module( + &self, + l: &Srcloc, + prog_args: Rc, + body: Rc + ) -> Rc { + let end_of_list = + if self.defmac_ordering() { + let mut mod_list: Vec> = self.helpers.iter().map(|h| { + h.to_sexp() + }).collect(); + mod_list.push(body); + Rc::new(enlist(l.clone(), &mod_list)) + } else { + let mut end_of_list = Rc::new(SExp::Cons( + l.clone(), + body, + Rc::new(SExp::Nil(l.clone())), + )); + + for h in self.helpers.iter() { + end_of_list = Rc::new(SExp::Cons(l.clone(), h.to_sexp(), end_of_list)); + } + + end_of_list + }; + + Rc::new(SExp::Cons( + l.clone(), + Rc::new(SExp::Atom(l.clone(), "mod".as_bytes().to_vec())), + Rc::new(SExp::Cons(l.clone(), prog_args, end_of_list)), + )) + } + #[allow(clippy::too_many_arguments)] fn invoke_primitive( &self, @@ -725,23 +764,9 @@ impl<'info> Evaluator { prog_args, )))) } else if call_name == "com".as_bytes() { - let mut end_of_list = Rc::new(SExp::Cons( - l.clone(), - arguments_to_convert[0].to_sexp(), - Rc::new(SExp::Nil(l.clone())), - )); - - for h in self.helpers.iter() { - end_of_list = Rc::new(SExp::Cons(l.clone(), h.to_sexp(), end_of_list)) - } - - let use_body = SExp::Cons( - l.clone(), - Rc::new(SExp::Atom(l.clone(), "mod".as_bytes().to_vec())), - Rc::new(SExp::Cons(l, prog_args, end_of_list)), - ); - - let compiled = self.compile_code(allocator, false, Rc::new(use_body))?; + let use_body = self.make_com_module(&l, prog_args.clone(), arguments_to_convert[0].to_sexp()); + eprintln!("use_body {use_body}"); + let compiled = self.compile_code(allocator, false, use_body)?; let compiled_borrowed: &SExp = compiled.borrow(); Ok(Rc::new(BodyForm::Quoted(compiled_borrowed.clone()))) } else { diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 0e3e37d9a..8fc6e0081 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -217,6 +217,11 @@ impl Preprocessor { self.helpers.clone(), ); eval.add_extension(Rc::new(PreprocessorExtension::new())); + eprintln!("expand macro {}", decode_string(&name)); + for (n,v) in macro_arg_env.iter() { + eprintln!("- {} = {}", decode_string(n), v.to_sexp()); + } + let res = eval.shrink_bodyform( &mut allocator, mdata.args.clone(), @@ -226,6 +231,8 @@ impl Preprocessor { None, )?; + eprintln!("expanded => {}", res.to_sexp()); + if let Ok(unquoted) = dequote(body.loc(), res) { return Ok(Some(unquoted)); } else { From 991381e496118f037757d70cae684d54c62adee6 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 29 May 2023 17:37:25 -0700 Subject: [PATCH 32/81] fmt + clippy --- src/compiler/evaluate.rs | 40 ++++++++++++-------------------- src/compiler/preprocessor/mod.rs | 2 +- 2 files changed, 16 insertions(+), 26 deletions(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 66d13bcdc..56dd40d84 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -705,32 +705,21 @@ impl<'info> Evaluator { dialect.strict || dialect.stepping.unwrap_or(21) > 22 } - fn make_com_module( - &self, - l: &Srcloc, - prog_args: Rc, - body: Rc - ) -> Rc { - let end_of_list = - if self.defmac_ordering() { - let mut mod_list: Vec> = self.helpers.iter().map(|h| { - h.to_sexp() - }).collect(); - mod_list.push(body); - Rc::new(enlist(l.clone(), &mod_list)) - } else { - let mut end_of_list = Rc::new(SExp::Cons( - l.clone(), - body, - Rc::new(SExp::Nil(l.clone())), - )); + fn make_com_module(&self, l: &Srcloc, prog_args: Rc, body: Rc) -> Rc { + let end_of_list = if self.defmac_ordering() { + let mut mod_list: Vec> = self.helpers.iter().map(|h| h.to_sexp()).collect(); + mod_list.push(body); + Rc::new(enlist(l.clone(), mod_list)) + } else { + let mut end_of_list = + Rc::new(SExp::Cons(l.clone(), body, Rc::new(SExp::Nil(l.clone())))); - for h in self.helpers.iter() { - end_of_list = Rc::new(SExp::Cons(l.clone(), h.to_sexp(), end_of_list)); - } + for h in self.helpers.iter() { + end_of_list = Rc::new(SExp::Cons(l.clone(), h.to_sexp(), end_of_list)); + } - end_of_list - }; + end_of_list + }; Rc::new(SExp::Cons( l.clone(), @@ -764,7 +753,8 @@ impl<'info> Evaluator { prog_args, )))) } else if call_name == "com".as_bytes() { - let use_body = self.make_com_module(&l, prog_args.clone(), arguments_to_convert[0].to_sexp()); + let use_body = + self.make_com_module(&l, prog_args, arguments_to_convert[0].to_sexp()); eprintln!("use_body {use_body}"); let compiled = self.compile_code(allocator, false, use_body)?; let compiled_borrowed: &SExp = compiled.borrow(); diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 8fc6e0081..a092dddaf 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -218,7 +218,7 @@ impl Preprocessor { ); eval.add_extension(Rc::new(PreprocessorExtension::new())); eprintln!("expand macro {}", decode_string(&name)); - for (n,v) in macro_arg_env.iter() { + for (n, v) in macro_arg_env.iter() { eprintln!("- {} = {}", decode_string(n), v.to_sexp()); } From 432272b6ba47647d9d2c1ea5285cb2e293ee3b84 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 29 May 2023 12:09:37 -0700 Subject: [PATCH 33/81] Fix a bug that kept macros relying on macros from evaluating in some circumstances --- src/compiler/evaluate.rs | 2 +- src/tests/compiler/preprocessor.rs | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 56dd40d84..07c8d21f4 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -1330,7 +1330,7 @@ impl<'info> Evaluator { // primitive. let updated_opts = self .opts - .set_stdenv(!in_defun) + .set_stdenv(!in_defun && !self.opts.dialect().strict) .set_in_defun(in_defun) .set_frontend_opt(false); diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 5ccfd906d..b0873b202 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -1,3 +1,11 @@ +use std::rc::Rc; +use crate::compiler::comptypes::CompilerOpts; +use crate::compiler::compiler::DefaultCompilerOpts; +use crate::compiler::dialect::AcceptedDialect; +use crate::compiler::preprocessor::preprocess; +use crate::compiler::sexp::parse_sexp; +use crate::compiler::srcloc::Srcloc; + use crate::tests::compiler::compiler::run_string; #[test] @@ -503,3 +511,25 @@ fn test_defmac_number_to_string() { let res = run_string(&prog, &"(37)".to_string()).unwrap(); assert_eq!(res.to_string(), "136"); } + +#[test] +fn test_preprocess_basic_list() { + let file = "*test*"; + let input_form_set = indoc! {"( + (include *strict-cl-21*) + (list 1 2 3) + )"}; + let parsed_forms = parse_sexp(Srcloc::start(file), input_form_set.bytes()).expect("should parse"); + let opts: Rc = + Rc::new(DefaultCompilerOpts::new(file)).set_dialect(AcceptedDialect { + stepping: Some(21), + strict: true, + }); + let mut includes = Vec::new(); + let pp = preprocess( + opts.clone(), + &mut includes, + parsed_forms[0].clone() + ).expect("should preprocess"); + assert_eq!(pp[pp.len()-1].to_string(), "(4 1 (4 2 (4 3 ())))"); +} From 4daa88e09e50ceed0c708d92cb4b77257e060b32 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 29 May 2023 13:21:28 -0700 Subject: [PATCH 34/81] Added new tests for strict mode with modern macros --- src/tests/classic/run.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index b17bcaec8..c08d615e7 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -896,16 +896,28 @@ fn test_strict_smoke_1() { assert_eq!(result, "15"); } -/* #[test] fn test_strict_list_fail() { - let result_prog = do_basic_run(&vec![ + let result = do_basic_run(&vec![ "run".to_string(), "-i".to_string(), "resources/tests/strict".to_string(), "resources/tests/strict/strict-list-fail.clsp".to_string(), ]); - eprintln!("result_prog {result_prog}"); - todo!(); + assert!(result.contains("Unbound")); + assert!(result.contains("X2")); +} + +#[test] +fn test_strict_list_pass() { + let result_prog = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + "resources/tests/strict/strict-list-pass.clsp".to_string(), + ]); + let result = do_basic_brun(&vec!["brun".to_string(), result_prog, "(13)".to_string()]) + .trim() + .to_string(); + assert_eq!(result, "(strlen 14 15)"); } -*/ From 8e04372fb715d0d363b2d33891512e56919dc3ce Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 29 May 2023 17:43:26 -0700 Subject: [PATCH 35/81] fmt + clippy --- src/compiler/evaluate.rs | 3 +- src/compiler/preprocessor/mod.rs | 45 ------------------------------ src/tests/compiler/preprocessor.rs | 16 +++++------ 3 files changed, 8 insertions(+), 56 deletions(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 07c8d21f4..7bc6ee80d 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -753,8 +753,7 @@ impl<'info> Evaluator { prog_args, )))) } else if call_name == "com".as_bytes() { - let use_body = - self.make_com_module(&l, prog_args, arguments_to_convert[0].to_sexp()); + let use_body = self.make_com_module(&l, prog_args, arguments_to_convert[0].to_sexp()); eprintln!("use_body {use_body}"); let compiled = self.compile_code(allocator, false, use_body)?; let compiled_borrowed: &SExp = compiled.borrow(); diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index a092dddaf..5b7f182d7 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -111,44 +111,6 @@ impl Preprocessor { Ok(()) } - /* - fn recurse_dependencies( - &mut self, - includes: &mut Vec, - desc: IncludeType, - ) -> Result<(), CompileErr> { - match desc { - IncludeType::Basic(desc) => { - let name_string = decode_string(&desc.name); - if KNOWN_DIALECTS.contains_key(&name_string) { - return Ok(()); - } - - let (full_name, content) = self.opts.read_new_file(self.opts.filename(), name_string)?; - includes.push(IncludeDesc { - name: full_name.as_bytes().to_vec(), - ..desc - }); - - let parsed = parse_sexp(Srcloc::start(&full_name), content.iter().copied())?; - if parsed.is_empty() { - return Ok(()); - } - - let program_form = parsed[0].clone(); - if let Some(l) = program_form.proper_list() { - for elt in l.iter() { - self.process_pp_form(includes, Rc::new(elt.clone()))?; - } - } - - Ok(()) - }, - _ => { todo!() } - } - } - */ - fn add_helper(&mut self, h: HelperForm) { for i in 0..=self.helpers.len() { if i == self.helpers.len() { @@ -217,11 +179,6 @@ impl Preprocessor { self.helpers.clone(), ); eval.add_extension(Rc::new(PreprocessorExtension::new())); - eprintln!("expand macro {}", decode_string(&name)); - for (n, v) in macro_arg_env.iter() { - eprintln!("- {} = {}", decode_string(n), v.to_sexp()); - } - let res = eval.shrink_bodyform( &mut allocator, mdata.args.clone(), @@ -231,8 +188,6 @@ impl Preprocessor { None, )?; - eprintln!("expanded => {}", res.to_sexp()); - if let Ok(unquoted) = dequote(body.loc(), res) { return Ok(Some(unquoted)); } else { diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index b0873b202..1d8ab82b9 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -1,10 +1,10 @@ -use std::rc::Rc; -use crate::compiler::comptypes::CompilerOpts; use crate::compiler::compiler::DefaultCompilerOpts; +use crate::compiler::comptypes::CompilerOpts; use crate::compiler::dialect::AcceptedDialect; use crate::compiler::preprocessor::preprocess; use crate::compiler::sexp::parse_sexp; use crate::compiler::srcloc::Srcloc; +use std::rc::Rc; use crate::tests::compiler::compiler::run_string; @@ -519,17 +519,15 @@ fn test_preprocess_basic_list() { (include *strict-cl-21*) (list 1 2 3) )"}; - let parsed_forms = parse_sexp(Srcloc::start(file), input_form_set.bytes()).expect("should parse"); + let parsed_forms = + parse_sexp(Srcloc::start(file), input_form_set.bytes()).expect("should parse"); let opts: Rc = Rc::new(DefaultCompilerOpts::new(file)).set_dialect(AcceptedDialect { stepping: Some(21), strict: true, }); let mut includes = Vec::new(); - let pp = preprocess( - opts.clone(), - &mut includes, - parsed_forms[0].clone() - ).expect("should preprocess"); - assert_eq!(pp[pp.len()-1].to_string(), "(4 1 (4 2 (4 3 ())))"); + let pp = preprocess(opts.clone(), &mut includes, parsed_forms[0].clone()) + .expect("should preprocess"); + assert_eq!(pp[pp.len() - 1].to_string(), "(4 1 (4 2 (4 3 ())))"); } From 62cfdb96920552047cfc8f23536d92611dbfc8bc Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 29 May 2023 17:45:59 -0700 Subject: [PATCH 36/81] remove spam --- src/compiler/evaluate.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 7bc6ee80d..63068ae69 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -754,7 +754,6 @@ impl<'info> Evaluator { )))) } else if call_name == "com".as_bytes() { let use_body = self.make_com_module(&l, prog_args, arguments_to_convert[0].to_sexp()); - eprintln!("use_body {use_body}"); let compiled = self.compile_code(allocator, false, use_body)?; let compiled_borrowed: &SExp = compiled.borrow(); Ok(Rc::new(BodyForm::Quoted(compiled_borrowed.clone()))) From 583d2506906c57746350f39402939fe907a35057 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 30 May 2023 08:28:46 -0700 Subject: [PATCH 37/81] Add test with deeper macro nesting --- resources/tests/strict/strict-nested-list.clsp | 4 ++++ src/tests/classic/run.rs | 14 ++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 resources/tests/strict/strict-nested-list.clsp diff --git a/resources/tests/strict/strict-nested-list.clsp b/resources/tests/strict/strict-nested-list.clsp new file mode 100644 index 000000000..6bc4bddb4 --- /dev/null +++ b/resources/tests/strict/strict-nested-list.clsp @@ -0,0 +1,4 @@ +(mod (X) + (include *strict-cl-21*) + (list X (list X) (list (list X))) + ) diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index c08d615e7..a5e696ac3 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -921,3 +921,17 @@ fn test_strict_list_pass() { .to_string(); assert_eq!(result, "(strlen 14 15)"); } + +#[test] +fn test_strict_nested_list_pass() { + let result_prog = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + "resources/tests/strict/strict-nested-list.clsp".to_string(), + ]); + let result = do_basic_brun(&vec!["brun".to_string(), result_prog, "(13)".to_string()]) + .trim() + .to_string(); + assert_eq!(result, "(strlen (strlen) ((strlen)))"); +} From 7339e26ab32154ecf15707fc5a9d298cd26739b6 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 30 May 2023 08:51:53 -0700 Subject: [PATCH 38/81] More moderately complex tests of what this kind of macro system can do --- src/tests/classic/run.rs | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index a5e696ac3..ea71d1874 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -935,3 +935,29 @@ fn test_strict_nested_list_pass() { .to_string(); assert_eq!(result, "(strlen (strlen) ((strlen)))"); } + +#[test] +fn test_double_constant_pass() { + let result_prog = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + "resources/tests/strict/double-constant-pass.clsp".to_string(), + ]); + let result = do_basic_brun(&vec!["brun".to_string(), result_prog, "()".to_string()]) + .trim() + .to_string(); + assert_eq!(result, "198"); +} + +#[test] +fn test_double_constant_fail() { + let result = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + "resources/tests/strict/double-constant-fail.clsp".to_string(), + ]); + assert!(result.contains("not a number given to only-integers")); + assert!(result.contains("\"hithere\"")); +} From a2bcc3101d6e560e212556a77101d32bca794860 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 30 May 2023 08:52:16 -0700 Subject: [PATCH 39/81] Add test files --- .../tests/strict/double-constant-fail.clsp | 24 +++++++++++++++++++ .../tests/strict/double-constant-pass.clsp | 24 +++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 resources/tests/strict/double-constant-fail.clsp create mode 100644 resources/tests/strict/double-constant-pass.clsp diff --git a/resources/tests/strict/double-constant-fail.clsp b/resources/tests/strict/double-constant-fail.clsp new file mode 100644 index 000000000..7af9d1e23 --- /dev/null +++ b/resources/tests/strict/double-constant-fail.clsp @@ -0,0 +1,24 @@ +(mod (X) + (include *strict-cl-21*) + + ;; A macro-level function to pass through only real integers. + (defun pass-through-integers (X) + (if (not (number? X)) + (x "not a number given to only-integers" X) + X + ) + ) + + ;; A macro which at preprocessing time throws if the given argument + ;; wasn't a lexical integer. + (defmac only-integers (X) (pass-through-integers X)) + + ;; Note: when macro expanding, N is the N argument to the body of + ;; the double macro, not the integer literal, so we use the function + ;; version of pass-through-integers in the macro body. + (defmac double (N) (* 2 (pass-through-integers N))) + + ;; Here the macro form of only-integers can determine whether the + ;; double macro produced an integer or some other expression. + (only-integers (double "hithere")) + ) diff --git a/resources/tests/strict/double-constant-pass.clsp b/resources/tests/strict/double-constant-pass.clsp new file mode 100644 index 000000000..7b50bd1ee --- /dev/null +++ b/resources/tests/strict/double-constant-pass.clsp @@ -0,0 +1,24 @@ +(mod (X) + (include *strict-cl-21*) + + ;; A macro-level function to pass through only real integers. + (defun pass-through-integers (X) + (if (not (number? X)) + (x "not a number given to only-integers" X) + X + ) + ) + + ;; A macro which at preprocessing time throws if the given argument + ;; wasn't a lexical integer. + (defmac only-integers (X) (pass-through-integers X)) + + ;; Note: when macro expanding, N is the N argument to the body of + ;; the double macro, not the integer literal, so we use the function + ;; version of pass-through-integers in the macro body. + (defmac double (N) (* 2 (pass-through-integers N))) + + ;; Here the macro form of only-integers can determine whether the + ;; double macro produced an integer or some other expression. + (only-integers (double 99)) + ) From 282b08f6274986ee5c2968f866b092fa91f2c6ce Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 30 May 2023 12:10:52 -0700 Subject: [PATCH 40/81] Plumb down new primitives when running with extensions. Add test using this. --- src/compiler/cldb.rs | 1 + src/compiler/clvm.rs | 24 +++++++++++-- src/compiler/codegen.rs | 2 ++ src/compiler/compiler.rs | 5 +++ src/compiler/comptypes.rs | 2 ++ src/compiler/evaluate.rs | 53 ++++++++++++++++++++++++++++- src/compiler/optimize.rs | 1 + src/compiler/preprocessor/macros.rs | 16 ++++++++- src/compiler/preprocessor/mod.rs | 5 +-- src/tests/classic/run.rs | 32 +++++++++++++++++ src/tests/classic/stage_2.rs | 3 ++ src/tests/compiler/compiler.rs | 1 + 12 files changed, 138 insertions(+), 7 deletions(-) diff --git a/src/compiler/cldb.rs b/src/compiler/cldb.rs index 95df3dcd2..11b4c7f59 100644 --- a/src/compiler/cldb.rs +++ b/src/compiler/cldb.rs @@ -150,6 +150,7 @@ impl CldbRun { self.runner.clone(), self.prim_map.clone(), &self.step, + None, ), }; diff --git a/src/compiler/clvm.rs b/src/compiler/clvm.rs index 5c2656c80..a1e535532 100644 --- a/src/compiler/clvm.rs +++ b/src/compiler/clvm.rs @@ -20,6 +20,11 @@ use crate::compiler::srcloc::Srcloc; use crate::util::{number_from_u8, u8_from_number, Number}; +/// Provide a way of intercepting and running new primitives. +pub trait PrimOverride { + fn try_handle(&self, head: Rc, context: Rc, tail: Rc) -> Result>, RunFailure>; +} + /// An object which contains the state of a running CLVM program in a compact /// form. /// @@ -161,7 +166,7 @@ fn translate_head( Some(v) => Ok(Rc::new(v.with_loc(l.clone()))), }, SExp::Cons(_l, _a, nil) => match nil.borrow() { - SExp::Nil(_l1) => run(allocator, runner, prim_map, sexp.clone(), context, None), + SExp::Nil(_l1) => run(allocator, runner, prim_map, sexp.clone(), context, None, None), _ => Err(RunFailure::RunErr( sexp.loc(), format!("Unexpected head form in clvm {sexp}"), @@ -396,6 +401,7 @@ pub fn run_step( runner: Rc, prim_map: Rc, Rc>>, step_: &RunStep, + prim_override: Option<&dyn PrimOverride>, ) -> Result { let mut step = step_.clone(); @@ -499,7 +505,7 @@ pub fn run_step( } } } - RunStep::Op(head, _context, tail, None, parent) => { + RunStep::Op(head, context, tail, None, parent) => { let aval = atom_value(head.clone())?; let apply_atom = 2_i32.to_bigint().unwrap(); let if_atom = 3_i32.to_bigint().unwrap(); @@ -517,6 +523,16 @@ pub fn run_step( -1 }; + if let Some(ovr) = prim_override { + if let Some(res) = + ovr.try_handle(head.clone(), context.clone(), tail.clone())? + { + return Ok(RunStep::OpResult( + res.loc(), res.clone(), parent.clone() + )); + } + } + let op = if aval == apply_atom { "apply".to_string() } else if aval == if_atom { @@ -632,6 +648,7 @@ pub fn run( prim_map: Rc, Rc>>, sexp_: Rc, context_: Rc, + prim_override: Option<&dyn PrimOverride>, iter_limit: Option, ) -> Result, RunFailure> { let mut step = start_step(sexp_, context_); @@ -644,7 +661,7 @@ pub fn run( } } iters += 1; - step = run_step(allocator, runner.clone(), prim_map.clone(), &step)?; + step = run_step(allocator, runner.clone(), prim_map.clone(), &step, prim_override.clone())?; if let RunStep::Done(_, x) = step { return Ok(x); } @@ -684,6 +701,7 @@ pub fn parse_and_run( prim_map, code[0].clone(), args[0].clone(), + None, step_limit, ) } diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 064512485..828f7418d 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -272,6 +272,7 @@ pub fn process_macro_call( opts.prim_map(), code, Rc::new(args_to_macro), + None, Some(MACRO_TIME_LIMIT), ) .map_err(|e| match e { @@ -994,6 +995,7 @@ fn start_codegen( opts.prim_map(), Rc::new(code), Rc::new(SExp::Nil(defc.loc.clone())), + None, Some(CONST_EVAL_LIMIT), ) .map_err(|r| { diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index ec9577096..dd599a61b 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -310,6 +310,11 @@ impl CompilerOpts for DefaultCompilerOpts { copy.start_env = start_env; Rc::new(copy) } + fn set_prim_map(&self, prims: Rc, Rc>>) -> Rc { + let mut copy = self.clone(); + copy.prim_map = prims; + Rc::new(copy) + } fn read_new_file( &self, diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 500590e74..1c86274a4 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -357,6 +357,8 @@ pub trait CompilerOpts { fn set_code_generator(&self, new_compiler: PrimaryCodegen) -> Rc; /// Set the environment shape to assume. fn set_start_env(&self, start_env: Option>) -> Rc; + /// Set the primitive map in use so we can add custom primitives. + fn set_prim_map(&self, new_map: Rc, Rc>>) -> Rc; /// Using the search paths list we have, try to read a file by name, /// Returning the expanded path to the file and its content. diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 63068ae69..d4142a0b2 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -9,7 +9,7 @@ use clvm_rs::allocator::Allocator; use crate::classic::clvm::__type_compatibility__::{bi_one, bi_zero}; use crate::classic::clvm_tools::stages::stage_0::TRunProgram; -use crate::compiler::clvm::run; +use crate::compiler::clvm::{PrimOverride, run}; use crate::compiler::codegen::codegen; use crate::compiler::compiler::is_at_capture; use crate::compiler::comptypes::{ @@ -134,6 +134,56 @@ pub struct Evaluator { ignore_exn: bool, } +fn compile_to_run_err(e: CompileErr) -> RunFailure { + match e { + CompileErr(l, e) => { + RunFailure::RunErr(l.clone(), e.clone()) + } + } +} + +impl PrimOverride for Evaluator { + fn try_handle(&self, head: Rc, _context: Rc, tail: Rc) -> Result>, RunFailure> { + let have_args: Vec> = + if let Some(args_list) = tail + .proper_list() { + args_list + .iter() + .map(|e| Rc::new(BodyForm::Quoted(e.clone()))) + .collect() + } else { + return Ok(None); + }; + + if let SExp::Atom(hl, head_atom) = head.borrow() { + let mut call_args = + vec![Rc::new(BodyForm::Value(SExp::Atom(hl.clone(), head_atom.clone())))]; + call_args.append(&mut have_args.clone()); + let call_form = Rc::new(BodyForm::Call(head.loc(), call_args)); + + for x in self.extensions.iter() { + if let Some(res) = + x.try_eval( + self, + Rc::new(SExp::Nil(head.loc())), + &HashMap::new(), + &head.loc(), + &head_atom, + &have_args, + call_form.clone(), + ).map_err(compile_to_run_err)? + { + return dequote(head.loc(), res) + .map_err(compile_to_run_err) + .map(Some); + } + } + } + + Ok(None) + } +} + fn select_helper(bindings: &[HelperForm], name: &[u8]) -> Option { for b in bindings.iter() { if b.name() == name { @@ -1305,6 +1355,7 @@ impl<'info> Evaluator { self.prims.clone(), prim, args, + Some(self), Some(PRIM_RUN_LIMIT), ) .map_err(|e| match e { diff --git a/src/compiler/optimize.rs b/src/compiler/optimize.rs index 03dda32e9..901c93ebf 100644 --- a/src/compiler/optimize.rs +++ b/src/compiler/optimize.rs @@ -89,6 +89,7 @@ pub fn optimize_expr( opts.prim_map(), code.to_sexp(), Rc::new(SExp::Nil(l)), + None, Some(CONST_FOLD_LIMIT), ) .map(|x| { diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 28abf435d..6d6375d2c 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -9,7 +9,7 @@ use num_traits::ToPrimitive; use crate::classic::clvm::__type_compatibility__::{bi_one, bi_zero}; use crate::compiler::clvm::truthy; -use crate::compiler::comptypes::{BodyForm, CompileErr}; +use crate::compiler::comptypes::{BodyForm, CompileErr, CompilerOpts}; use crate::compiler::evaluate::{EvalExtension, Evaluator}; use crate::compiler::preprocessor::dequote; use crate::compiler::sexp::{decode_string, SExp}; @@ -771,6 +771,20 @@ impl PreprocessorExtension { extfuns: HashMap::from(extfuns), } } + + /// Introduce new primitive names for the operators we use to bootstrap macros. + pub fn enrich_prims(&self, opts: Rc) -> Rc { + let old_prim_map = opts.prim_map(); + let old_prim_map_borrowed: &HashMap, Rc> = old_prim_map.borrow(); + let mut new_prim_map_cloned = old_prim_map_borrowed.clone(); + let srcloc = Srcloc::start("*defmac*"); + + for (f, _) in self.extfuns.iter() { + new_prim_map_cloned.insert(f.clone(), Rc::new(SExp::Atom(srcloc.clone(), f.clone()))); + } + + opts.set_prim_map(Rc::new(new_prim_map_cloned)) + } } impl EvalExtension for PreprocessorExtension { diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 5b7f182d7..1e935d30a 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -173,12 +173,13 @@ impl Preprocessor { mdata.args.clone(), )?; + let ppext = Rc::new(PreprocessorExtension::new()); let mut eval = Evaluator::new( - self.opts.clone(), + ppext.enrich_prims(self.opts.clone()), self.runner.clone(), self.helpers.clone(), ); - eval.add_extension(Rc::new(PreprocessorExtension::new())); + eval.add_extension(ppext); let res = eval.shrink_bodyform( &mut allocator, mdata.args.clone(), diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index ea71d1874..c735c5c53 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -961,3 +961,35 @@ fn test_double_constant_fail() { assert!(result.contains("not a number given to only-integers")); assert!(result.contains("\"hithere\"")); } + +#[test] +fn test_double_constant_pass_in_function() { + let result_prog = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + "resources/tests/strict/double-constant-pass-in-function.clsp".to_string(), + ]); + let result = do_basic_brun(&vec!["brun".to_string(), result_prog, "(13)".to_string()]) + .trim() + .to_string(); + assert_eq!(result, "211"); +} + +#[test] +fn test_check_symbol_kinds_nested_if() { + let result_prog = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + "resources/tests/strict/strict-classify-expr-if.clsp".to_string(), + ]); + let result_1 = do_basic_brun(&vec!["brun".to_string(), result_prog.clone(), "(1)".to_string()]) + .trim() + .to_string(); + assert_eq!(result_1, "2"); + let result_0 = do_basic_brun(&vec!["brun".to_string(), result_prog, "(0)".to_string()]) + .trim() + .to_string(); + assert_eq!(result_0, "(q 1 2 3 4 4)"); +} diff --git a/src/tests/classic/stage_2.rs b/src/tests/classic/stage_2.rs index a83846b9f..b92b3d921 100644 --- a/src/tests/classic/stage_2.rs +++ b/src/tests/classic/stage_2.rs @@ -374,6 +374,9 @@ impl CompilerOpts for TestCompilerOptsPresentsOwnFiles { fn set_start_env(&self, _start_env: Option>) -> Rc { Rc::new(self.clone()) } + fn set_prim_map(&self, _prims: Rc, Rc>>) -> Rc { + Rc::new(self.clone()) + } fn read_new_file( &self, inc_from: String, diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index 0eecaed3f..497b538b0 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -49,6 +49,7 @@ fn run_string_maybe_opt( Rc::new(HashMap::new()), Rc::new(x), sexp_args, + None, Some(TEST_TIMEOUT), ) .map_err(|e| match e { From 9d63e50694a91c528fc74b731e2540557903b857 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 30 May 2023 12:30:01 -0700 Subject: [PATCH 41/81] test files --- .../double-constant-pass-in-function.clsp | 26 +++++++++++++++++++ .../tests/strict/strict-classify-expr-if.clsp | 24 +++++++++++++++++ .../strict/strict-in-place-factorial.clsp | 6 +++++ 3 files changed, 56 insertions(+) create mode 100644 resources/tests/strict/double-constant-pass-in-function.clsp create mode 100644 resources/tests/strict/strict-classify-expr-if.clsp create mode 100644 resources/tests/strict/strict-in-place-factorial.clsp diff --git a/resources/tests/strict/double-constant-pass-in-function.clsp b/resources/tests/strict/double-constant-pass-in-function.clsp new file mode 100644 index 000000000..a3367039f --- /dev/null +++ b/resources/tests/strict/double-constant-pass-in-function.clsp @@ -0,0 +1,26 @@ +(mod (X) + (include *strict-cl-21*) + + ;; A macro-level function to pass through only real integers. + (defun pass-through-integers (X) + (if (not (number? X)) + (x "not a number given to only-integers" X) + X + ) + ) + + ;; A macro which at preprocessing time throws if the given argument + ;; wasn't a lexical integer. + (defmac only-integers (X) (pass-through-integers X)) + + ;; Note: when macro expanding, N is the N argument to the body of + ;; the double macro, not the integer literal, so we use the function + ;; version of pass-through-integers in the macro body. + (defmac double (N) (* 2 (pass-through-integers N))) + + ;; Here the macro form of only-integers can determine whether the + ;; double macro produced an integer or some other expression. + (defun F (N) (+ N (only-integers (double 99)))) + + (F X) + ) diff --git a/resources/tests/strict/strict-classify-expr-if.clsp b/resources/tests/strict/strict-classify-expr-if.clsp new file mode 100644 index 000000000..03288183d --- /dev/null +++ b/resources/tests/strict/strict-classify-expr-if.clsp @@ -0,0 +1,24 @@ +(mod (X) + (include *strict-cl-21*) + ;; Ensure macros can expand inside other macros when using advanced primitives. + (defmac classify-expr (G) + (if (number? G) + 1 + (if (symbol? G) + 2 + (if (string? G) + 3 + (if (l G) + 4 + 0 + ) + ) + ) + ) + ) + + (if X + (classify-expr X) + (list (classify-expr ()) (classify-expr 33) (classify-expr test) (classify-expr "foo") (classify-expr (* 3 2)) (classify-expr (list 1 2 3))) + ) + ) diff --git a/resources/tests/strict/strict-in-place-factorial.clsp b/resources/tests/strict/strict-in-place-factorial.clsp new file mode 100644 index 000000000..6b036beac --- /dev/null +++ b/resources/tests/strict/strict-in-place-factorial.clsp @@ -0,0 +1,6 @@ +(mod (X) + (include *strict-cl-21*) + (defmac factorial (N) + (if (> 2 N) 1 (qq (* (unquote N) (factorial (- N 1)))))) + (factorial 5) + ) From dc0dc627559ca7deb38999db137b2ce74dfa243e Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 30 May 2023 12:44:10 -0700 Subject: [PATCH 42/81] fmt + clippy --- src/compiler/clvm.rs | 33 ++++++++++++++++++++-------- src/compiler/evaluate.rs | 46 ++++++++++++++++++++++------------------ src/tests/classic/run.rs | 10 ++++++--- 3 files changed, 56 insertions(+), 33 deletions(-) diff --git a/src/compiler/clvm.rs b/src/compiler/clvm.rs index a1e535532..a514a67c1 100644 --- a/src/compiler/clvm.rs +++ b/src/compiler/clvm.rs @@ -22,7 +22,12 @@ use crate::util::{number_from_u8, u8_from_number, Number}; /// Provide a way of intercepting and running new primitives. pub trait PrimOverride { - fn try_handle(&self, head: Rc, context: Rc, tail: Rc) -> Result>, RunFailure>; + fn try_handle( + &self, + head: Rc, + context: Rc, + tail: Rc, + ) -> Result>, RunFailure>; } /// An object which contains the state of a running CLVM program in a compact @@ -166,7 +171,15 @@ fn translate_head( Some(v) => Ok(Rc::new(v.with_loc(l.clone()))), }, SExp::Cons(_l, _a, nil) => match nil.borrow() { - SExp::Nil(_l1) => run(allocator, runner, prim_map, sexp.clone(), context, None, None), + SExp::Nil(_l1) => run( + allocator, + runner, + prim_map, + sexp.clone(), + context, + None, + None, + ), _ => Err(RunFailure::RunErr( sexp.loc(), format!("Unexpected head form in clvm {sexp}"), @@ -524,12 +537,8 @@ pub fn run_step( }; if let Some(ovr) = prim_override { - if let Some(res) = - ovr.try_handle(head.clone(), context.clone(), tail.clone())? - { - return Ok(RunStep::OpResult( - res.loc(), res.clone(), parent.clone() - )); + if let Some(res) = ovr.try_handle(head.clone(), context.clone(), tail.clone())? { + return Ok(RunStep::OpResult(res.loc(), res.clone(), parent.clone())); } } @@ -661,7 +670,13 @@ pub fn run( } } iters += 1; - step = run_step(allocator, runner.clone(), prim_map.clone(), &step, prim_override.clone())?; + step = run_step( + allocator, + runner.clone(), + prim_map.clone(), + &step, + prim_override, + )?; if let RunStep::Done(_, x) = step { return Ok(x); } diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index d4142a0b2..033799b2b 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -9,7 +9,7 @@ use clvm_rs::allocator::Allocator; use crate::classic::clvm::__type_compatibility__::{bi_one, bi_zero}; use crate::classic::clvm_tools::stages::stage_0::TRunProgram; -use crate::compiler::clvm::{PrimOverride, run}; +use crate::compiler::clvm::{run, PrimOverride}; use crate::compiler::codegen::codegen; use crate::compiler::compiler::is_at_capture; use crate::compiler::comptypes::{ @@ -136,42 +136,46 @@ pub struct Evaluator { fn compile_to_run_err(e: CompileErr) -> RunFailure { match e { - CompileErr(l, e) => { - RunFailure::RunErr(l.clone(), e.clone()) - } + CompileErr(l, e) => RunFailure::RunErr(l, e), } } impl PrimOverride for Evaluator { - fn try_handle(&self, head: Rc, _context: Rc, tail: Rc) -> Result>, RunFailure> { - let have_args: Vec> = - if let Some(args_list) = tail - .proper_list() { - args_list - .iter() - .map(|e| Rc::new(BodyForm::Quoted(e.clone()))) - .collect() - } else { - return Ok(None); - }; + fn try_handle( + &self, + head: Rc, + _context: Rc, + tail: Rc, + ) -> Result>, RunFailure> { + let have_args: Vec> = if let Some(args_list) = tail.proper_list() { + args_list + .iter() + .map(|e| Rc::new(BodyForm::Quoted(e.clone()))) + .collect() + } else { + return Ok(None); + }; if let SExp::Atom(hl, head_atom) = head.borrow() { - let mut call_args = - vec![Rc::new(BodyForm::Value(SExp::Atom(hl.clone(), head_atom.clone())))]; + let mut call_args = vec![Rc::new(BodyForm::Value(SExp::Atom( + hl.clone(), + head_atom.clone(), + )))]; call_args.append(&mut have_args.clone()); let call_form = Rc::new(BodyForm::Call(head.loc(), call_args)); for x in self.extensions.iter() { - if let Some(res) = - x.try_eval( + if let Some(res) = x + .try_eval( self, Rc::new(SExp::Nil(head.loc())), &HashMap::new(), &head.loc(), - &head_atom, + head_atom, &have_args, call_form.clone(), - ).map_err(compile_to_run_err)? + ) + .map_err(compile_to_run_err)? { return dequote(head.loc(), res) .map_err(compile_to_run_err) diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index c735c5c53..1d48fa188 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -984,9 +984,13 @@ fn test_check_symbol_kinds_nested_if() { "resources/tests/strict".to_string(), "resources/tests/strict/strict-classify-expr-if.clsp".to_string(), ]); - let result_1 = do_basic_brun(&vec!["brun".to_string(), result_prog.clone(), "(1)".to_string()]) - .trim() - .to_string(); + let result_1 = do_basic_brun(&vec![ + "brun".to_string(), + result_prog.clone(), + "(1)".to_string(), + ]) + .trim() + .to_string(); assert_eq!(result_1, "2"); let result_0 = do_basic_brun(&vec!["brun".to_string(), result_prog, "(0)".to_string()]) .trim() From 35720bd643f61ab511d45531283bb60da591ccab Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 1 Jun 2023 19:09:40 -0700 Subject: [PATCH 43/81] Fixes bug which was clobbering the numeric names of operators when going into com --- src/compiler/preprocessor/macros.rs | 5 ++++- src/tests/compiler/preprocessor.rs | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 6d6375d2c..caac2530b 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -780,7 +780,10 @@ impl PreprocessorExtension { let srcloc = Srcloc::start("*defmac*"); for (f, _) in self.extfuns.iter() { - new_prim_map_cloned.insert(f.clone(), Rc::new(SExp::Atom(srcloc.clone(), f.clone()))); + if !new_prim_map_cloned.contains_key(f) { + new_prim_map_cloned + .insert(f.clone(), Rc::new(SExp::Atom(srcloc.clone(), f.clone()))); + } } opts.set_prim_map(Rc::new(new_prim_map_cloned)) diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 1d8ab82b9..e55d10b49 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -531,3 +531,17 @@ fn test_preprocess_basic_list() { .expect("should preprocess"); assert_eq!(pp[pp.len() - 1].to_string(), "(4 1 (4 2 (4 3 ())))"); } + +#[test] +fn test_preprocess_expansion_makes_numeric_operators() { + let prog = indoc! {" + (mod () + (include *strict-cl-21*) + (defmac G () (com (4 \"test\" ()))) + (G) + ) + "} + .to_string(); + let res = run_string(&prog, &"()".to_string()).unwrap(); + assert_eq!(res.to_string(), "(\"test\")"); +} From 7e05c1dd06f1495eb0d772e7d2fe2c23764b9c90 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 1 Jun 2023 19:32:24 -0700 Subject: [PATCH 44/81] clippy --- src/compiler/clvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/clvm.rs b/src/compiler/clvm.rs index a514a67c1..9a0261575 100644 --- a/src/compiler/clvm.rs +++ b/src/compiler/clvm.rs @@ -206,7 +206,7 @@ fn eval_args( return Ok(RunStep::Op( head, context_, - sexp.clone(), + sexp, Some(eval_list), parent, )); From 8689af2a6f393e5b2e350d48baf5c14c48e83254 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 1 Jun 2023 19:36:59 -0700 Subject: [PATCH 45/81] fmt --- src/compiler/clvm.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/compiler/clvm.rs b/src/compiler/clvm.rs index 9a0261575..f5b8bb0b9 100644 --- a/src/compiler/clvm.rs +++ b/src/compiler/clvm.rs @@ -203,13 +203,7 @@ fn eval_args( loop { match sexp.borrow() { SExp::Nil(_l) => { - return Ok(RunStep::Op( - head, - context_, - sexp, - Some(eval_list), - parent, - )); + return Ok(RunStep::Op(head, context_, sexp, Some(eval_list), parent)); } SExp::Cons(_l, a, b) => { eval_list.push(a.clone()); From 9e4979f7595b9262b03b676635241d5bd3e7261e Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 13 Jun 2023 10:45:45 -0700 Subject: [PATCH 46/81] Fix not doing full preprocessing on include file contents in strict mode --- src/compiler/preprocessor/mod.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 1e935d30a..b1b2632c2 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -23,6 +23,7 @@ struct Preprocessor { opts: Rc, runner: Rc, helpers: Vec, + strict: bool, } /* @@ -55,6 +56,7 @@ impl Preprocessor { opts: opts.clone(), runner, helpers: Vec::new(), + strict: opts.dialect().strict, } } @@ -69,7 +71,7 @@ impl Preprocessor { // Because we're also subsequently returning CompileErr later in the pipe, // this needs an explicit err map. - parse_sexp(start_of_file.clone(), content.bytes()) + let parsed: Vec> = parse_sexp(start_of_file.clone(), content.bytes()) .err_into() .and_then(|x| match x[0].proper_list() { None => Err(CompileErr( @@ -77,7 +79,20 @@ impl Preprocessor { "Includes should contain a list of forms".to_string(), )), Some(v) => Ok(v.iter().map(|x| Rc::new(x.clone())).collect()), - }) + })?; + + if self.strict { + let mut result = Vec::new(); + for p in parsed.into_iter() { + if let Some(res) = self.expand_macros(p.clone(), true)? { + result.push(res); + } + } + + Ok(result) + } else { + Ok(parsed) + } } fn recurse_dependencies( From fd9c877400983208a35ae6b215cbd9e5363b9948 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 16 Jun 2023 15:59:52 -0700 Subject: [PATCH 47/81] Fix bug in strict preprocessor and add test --- src/compiler/preprocessor/mod.rs | 2 +- src/tests/compiler/preprocessor.rs | 66 ++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index b1b2632c2..8107ea8c6 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -150,7 +150,7 @@ impl Preprocessor { let first_expanded = self.expand_macros(f.clone(), true)?; let rest_expanded = self.expand_macros(r.clone(), false)?; let new_self = match (first_expanded, rest_expanded) { - (None, None) => None, + (None, None) => Some(body.clone()), (Some(f), None) => Some(Rc::new(SExp::Cons(l.clone(), f, r.clone()))), (None, Some(r)) => Some(Rc::new(SExp::Cons(l.clone(), f.clone(), r))), (Some(f), Some(r)) => Some(Rc::new(SExp::Cons(l.clone(), f, r))), diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index e55d10b49..0546acc70 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -545,3 +545,69 @@ fn test_preprocess_expansion_makes_numeric_operators() { let res = run_string(&prog, &"()".to_string()).unwrap(); assert_eq!(res.to_string(), "(\"test\")"); } + +#[test] +fn test_preprocessor_tours_includes_properly() { + let prog = indoc! {" + ( ;; Note: preprocessing is run in the list of the body forms. + (include *standard-cl-23*) + (include condition_codes.clvm) + (include curry-and-treehash.clinc) + () + ) + "}.to_string(); + let pname = "*test*"; + let opts: Rc = Rc::new(DefaultCompilerOpts::new(pname)) + .set_search_paths(&["resources/tests".to_string()]) + .set_dialect(AcceptedDialect { + stepping: Some(23), + strict: true + }); + let parsed = parse_sexp(Srcloc::start(pname), prog.bytes()).expect("should parse"); + let mut includes = Vec::new(); + let res = preprocess(opts, &mut includes, parsed[0].clone()).expect("should preprocess"); + let expected_lines = &[ + "(defmac if (A B C) (qq (a (i (unquote A) (com (unquote B)) (com (unquote C))) @)))", + "(defun __chia__compile-list (args) (a (i args (com (c 4 (c (f args) (c (__chia__compile-list (r args)) ())))) (com ())) @))", + "(defmac list ARGS (__chia__compile-list ARGS))", + "(defun-inline / (A B) (f (divmod A B)))", + "(defun-inline c* (A B) (c A B))", + "(defun-inline a* (A B) (a A B))", + "(defun-inline coerce (X) : (Any -> Any) X)", + "(defun-inline explode (X) : (forall a ((Exec a) -> a)) X)", + "(defun-inline bless (X) : (forall a ((Pair a Unit) -> (Exec a))) (coerce X))", + "(defun-inline lift (X V) : (forall a (forall b ((Pair (Exec a) (Pair b Unit)) -> (Exec (Pair a b))))) (coerce X))", + "(defun-inline unlift (X) : (forall a (forall b ((Pair (Exec (Pair a b)) Unit) -> (Exec b)))) (coerce X))", + "(defconstant *chialisp-version* 23)", + "(defconstant AGG_SIG_UNSAFE 49)", + "(defconstant AGG_SIG_ME 50)", + "(defconstant CREATE_COIN 51)", + "(defconstant RESERVE_FEE 52)", + "(defconstant CREATE_COIN_ANNOUNCEMENT 60)", + "(defconstant ASSERT_COIN_ANNOUNCEMENT 61)", + "(defconstant CREATE_PUZZLE_ANNOUNCEMENT 62)", + "(defconstant ASSERT_PUZZLE_ANNOUNCEMENT 63)", + "(defconstant ASSERT_MY_COIN_ID 70)", + "(defconstant ASSERT_MY_PARENT_ID 71)", + "(defconstant ASSERT_MY_PUZZLEHASH 72)", + "(defconstant ASSERT_MY_AMOUNT 73)", + "(defconstant ASSERT_SECONDS_RELATIVE 80)", + "(defconstant ASSERT_SECONDS_ABSOLUTE 81)", + "(defconstant ASSERT_HEIGHT_RELATIVE 82)", + "(defconstant ASSERT_HEIGHT_ABSOLUTE 83)", + "(defconstant ONE 1)", + "(defconstant TWO 2)", + "(defconstant A_KW 2)", + "(defconstant Q_KW 1)", + "(defconstant C_KW 4)", + "(defun-inline update-hash-for-parameter-hash (parameter-hash environment-hash) (sha256 TWO (sha256 ONE C_KW) (sha256 TWO (sha256 TWO (sha256 ONE Q_KW) parameter-hash) (sha256 TWO environment-hash (sha256 ONE ())))))", + "(defun build-curry-list (reversed-curry-parameter-hashes environment-hash) (a (i reversed-curry-parameter-hashes (com (build-curry-list (r reversed-curry-parameter-hashes) (update-hash-for-parameter-hash (f reversed-curry-parameter-hashes) environment-hash))) (com environment-hash)) @))", + "(defun-inline tree-hash-of-apply (function-hash environment-hash) (sha256 TWO (sha256 ONE A_KW) (sha256 TWO (sha256 TWO (sha256 ONE Q_KW) function-hash) (sha256 TWO environment-hash (sha256 ONE ())))))", + "(defun puzzle-hash-of-curried-function (function-hash . reversed-curry-parameter-hashes) (tree-hash-of-apply function-hash (build-curry-list reversed-curry-parameter-hashes (sha256 ONE ONE))))", + "()", + ]; + for (i, r) in res.iter().enumerate() { + assert_eq!(r.to_string(), expected_lines[i]); + } + assert_eq!(res.len(), expected_lines.len()); +} From 1ec5d4cbfcde63fdf492039b70799c3860a79181 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 16 Jun 2023 16:00:16 -0700 Subject: [PATCH 48/81] fmt --- src/tests/compiler/preprocessor.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 0546acc70..07e9838ba 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -555,13 +555,14 @@ fn test_preprocessor_tours_includes_properly() { (include curry-and-treehash.clinc) () ) - "}.to_string(); + "} + .to_string(); let pname = "*test*"; let opts: Rc = Rc::new(DefaultCompilerOpts::new(pname)) .set_search_paths(&["resources/tests".to_string()]) .set_dialect(AcceptedDialect { stepping: Some(23), - strict: true + strict: true, }); let parsed = parse_sexp(Srcloc::start(pname), prog.bytes()).expect("should parse"); let mut includes = Vec::new(); From 4b7b0bfc79a882ef571116ca5db5cefb7d4c75fc Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 16 Jun 2023 20:01:08 -0700 Subject: [PATCH 49/81] Add -E preprocessing mode --- src/classic/clvm_tools/cmds.rs | 57 ++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index f8c2d7549..57c000622 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -1,5 +1,6 @@ use core::cell::RefCell; +use std::borrow::Borrow; use std::collections::{BTreeMap, HashMap}; use std::fs; use std::io; @@ -51,6 +52,7 @@ use crate::compiler::cldb::{hex_to_modern_sexp, CldbNoOverride, CldbRun, CldbRun use crate::compiler::cldb_hierarchy::{HierarchialRunner, HierarchialStepResult, RunPurpose}; use crate::compiler::clvm::start_step; use crate::compiler::compiler::{compile_file, run_optimizer, DefaultCompilerOpts}; +use crate::compiler::frontend::frontend; use crate::compiler::comptypes::{CompileErr, CompilerOpts}; use crate::compiler::debug::build_symbol_table_mut; use crate::compiler::dialect::detect_modern; @@ -773,6 +775,47 @@ fn fix_log( } } +fn perform_preprocessing(stdout: &mut Stream, opts: Rc, input_file: &str, program_text: &str) -> Result<(), CompileErr> { + let srcloc = Srcloc::start(input_file); + let parsed = parse_sexp(srcloc.clone(), program_text.bytes())?; + let stepping_form_text = + match opts.dialect().stepping { + Some(21) => Some("(include *strict-cl-21*)".to_string()), + Some(n) => Some(format!("(include *standard-cl-{n}*)")), + _ => None + }; + let frontend = frontend(opts, &parsed)?; + let fe_sexp = frontend.to_sexp(); + let with_stepping = + if let Some(s) = stepping_form_text { + let parsed_stepping_form = parse_sexp(srcloc.clone(), s.bytes())?; + if let sexp::SExp::Cons(_, a, rest) = fe_sexp.clone().borrow() { + Rc::new(sexp::SExp::Cons( + srcloc.clone(), + a.clone(), + Rc::new(sexp::SExp::Cons( + srcloc.clone(), + parsed_stepping_form[0].clone(), + rest.clone() + )) + )) + } else { + fe_sexp.clone() + } + } else { + fe_sexp.clone() + }; + + let whole_mod = sexp::SExp::Cons( + srcloc.clone(), + Rc::new(sexp::SExp::Atom(srcloc.clone(), b"mod".to_vec())), + with_stepping + ); + + stdout.write_str(&format!("{}", whole_mod)); + Ok(()) +} + pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, default_stage: u32) { let props = TArgumentParserProps { description: "Execute a clvm script.".to_string(), @@ -917,6 +960,12 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .set_action(TArgOptionAction::StoreTrue) .set_help("For modern dialects, don't treat unknown names as constants".to_string()), ); + parser.add_argument( + vec!["-E".to_string(), "--preprocess".to_string()], + Argument::new() + .set_action(TArgOptionAction::StoreTrue) + .set_help("Perform strict mode preprocessing and show the result".to_string()), + ); if tool_name == "run" { parser.add_argument( @@ -1204,6 +1253,14 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul .set_frontend_opt(stepping > 21); let mut symbol_table = HashMap::new(); + // Short circuit preprocessing display. + if let Some(_) = parsed_args.get("preprocess") { + if let Err(e) = perform_preprocessing(stdout, opts, &use_filename, &input_program) { + stdout.write_str(&format!("{}: {}", e.0, e.1)); + } + return; + } + let unopt_res = compile_file( &mut allocator, runner.clone(), From 22316dfd943a5138162cdbcf8401922f3fda956c Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 16 Jun 2023 20:03:01 -0700 Subject: [PATCH 50/81] fmt + clippy --- src/classic/clvm_tools/cmds.rs | 57 ++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index 57c000622..b1e3ef7e6 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -52,10 +52,10 @@ use crate::compiler::cldb::{hex_to_modern_sexp, CldbNoOverride, CldbRun, CldbRun use crate::compiler::cldb_hierarchy::{HierarchialRunner, HierarchialStepResult, RunPurpose}; use crate::compiler::clvm::start_step; use crate::compiler::compiler::{compile_file, run_optimizer, DefaultCompilerOpts}; -use crate::compiler::frontend::frontend; use crate::compiler::comptypes::{CompileErr, CompilerOpts}; use crate::compiler::debug::build_symbol_table_mut; use crate::compiler::dialect::detect_modern; +use crate::compiler::frontend::frontend; use crate::compiler::preprocessor::gather_dependencies; use crate::compiler::prims; use crate::compiler::runtypes::RunFailure; @@ -775,41 +775,44 @@ fn fix_log( } } -fn perform_preprocessing(stdout: &mut Stream, opts: Rc, input_file: &str, program_text: &str) -> Result<(), CompileErr> { +fn perform_preprocessing( + stdout: &mut Stream, + opts: Rc, + input_file: &str, + program_text: &str, +) -> Result<(), CompileErr> { let srcloc = Srcloc::start(input_file); let parsed = parse_sexp(srcloc.clone(), program_text.bytes())?; - let stepping_form_text = - match opts.dialect().stepping { - Some(21) => Some("(include *strict-cl-21*)".to_string()), - Some(n) => Some(format!("(include *standard-cl-{n}*)")), - _ => None - }; + let stepping_form_text = match opts.dialect().stepping { + Some(21) => Some("(include *strict-cl-21*)".to_string()), + Some(n) => Some(format!("(include *standard-cl-{n}*)")), + _ => None, + }; let frontend = frontend(opts, &parsed)?; let fe_sexp = frontend.to_sexp(); - let with_stepping = - if let Some(s) = stepping_form_text { - let parsed_stepping_form = parse_sexp(srcloc.clone(), s.bytes())?; - if let sexp::SExp::Cons(_, a, rest) = fe_sexp.clone().borrow() { + let with_stepping = if let Some(s) = stepping_form_text { + let parsed_stepping_form = parse_sexp(srcloc.clone(), s.bytes())?; + if let sexp::SExp::Cons(_, a, rest) = fe_sexp.borrow() { + Rc::new(sexp::SExp::Cons( + srcloc.clone(), + a.clone(), Rc::new(sexp::SExp::Cons( srcloc.clone(), - a.clone(), - Rc::new(sexp::SExp::Cons( - srcloc.clone(), - parsed_stepping_form[0].clone(), - rest.clone() - )) - )) - } else { - fe_sexp.clone() - } + parsed_stepping_form[0].clone(), + rest.clone(), + )), + )) } else { - fe_sexp.clone() - }; + fe_sexp + } + } else { + fe_sexp + }; let whole_mod = sexp::SExp::Cons( srcloc.clone(), - Rc::new(sexp::SExp::Atom(srcloc.clone(), b"mod".to_vec())), - with_stepping + Rc::new(sexp::SExp::Atom(srcloc, b"mod".to_vec())), + with_stepping, ); stdout.write_str(&format!("{}", whole_mod)); @@ -1254,7 +1257,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul let mut symbol_table = HashMap::new(); // Short circuit preprocessing display. - if let Some(_) = parsed_args.get("preprocess") { + if parsed_args.get("preprocess").is_some() { if let Err(e) = perform_preprocessing(stdout, opts, &use_filename, &input_program) { stdout.write_str(&format!("{}: {}", e.0, e.1)); } From 06733ded72b0964e2626d612b7a1c1870921b06d Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 16 Jun 2023 20:31:46 -0700 Subject: [PATCH 51/81] Basic smoke test of strict style preprocessing with -E --- resources/tests/strict/defmac_if_smoke.clsp | 8 +++++++ resources/tests/strict/defmac_simple_if.clib | 3 +++ src/tests/classic/run.rs | 25 ++++++++++++++++++++ 3 files changed, 36 insertions(+) create mode 100644 resources/tests/strict/defmac_if_smoke.clsp create mode 100644 resources/tests/strict/defmac_simple_if.clib diff --git a/resources/tests/strict/defmac_if_smoke.clsp b/resources/tests/strict/defmac_if_smoke.clsp new file mode 100644 index 000000000..d3baf7cca --- /dev/null +++ b/resources/tests/strict/defmac_if_smoke.clsp @@ -0,0 +1,8 @@ +(mod () + (include *strict-cl-21*) + + (include defmac_simple_if.clib) + + (if_ t1 t2 t3) + ) + \ No newline at end of file diff --git a/resources/tests/strict/defmac_simple_if.clib b/resources/tests/strict/defmac_simple_if.clib new file mode 100644 index 000000000..32655f11f --- /dev/null +++ b/resources/tests/strict/defmac_simple_if.clib @@ -0,0 +1,3 @@ +( + (defmac if_ (C T E) (qq (if (unquote C) (unquote T) (unquote E)))) +) \ No newline at end of file diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index 1d48fa188..c081baa74 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -997,3 +997,28 @@ fn test_check_symbol_kinds_nested_if() { .to_string(); assert_eq!(result_0, "(q 1 2 3 4 4)"); } + +// Note: this program is intentionally made to properly preprocess but trigger +// an error in strict compilation as a demonstration and test that the preprocessor +// is a mechanically separate step from compilation. Separating them like this +// has the advantage that you can emit preprocessed compiler input on its own +// and also that it internally untangles the stages and makes compilation simpler. +#[test] +fn test_defmac_if_smoke_preprocess() { + let result_prog = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + "-E".to_string(), + "resources/tests/strict/defmac_if_smoke.clsp".to_string(), + ]); + assert_eq!( + result_prog, + "(mod () (include *strict-cl-21*) (a (i t1 (com t2) (com t3)) @))" + ); + let result2 = do_basic_run(&vec!["run".to_string(), result_prog]); + assert!(result2.contains("Unbound use")); + // Ensure that we're identifying one of the actually misused variables, but + // do not make a claim about which one is identified first. + assert!(result2.contains("of t1") || result2.contains("of t2") || result2.contains("of t3")); +} From 93699a7a1141daad22956ef2a07eb193e36360cc Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 16 Jun 2023 21:03:43 -0700 Subject: [PATCH 52/81] Add some more defmac tests. --- resources/tests/strict/assert.clsp | 11 +++++++ resources/tests/strict/defmac_assert.clib | 10 +++++++ src/tests/classic/run.rs | 35 +++++++++++++++++++++++ 3 files changed, 56 insertions(+) create mode 100644 resources/tests/strict/assert.clsp create mode 100644 resources/tests/strict/defmac_assert.clib diff --git a/resources/tests/strict/assert.clsp b/resources/tests/strict/assert.clsp new file mode 100644 index 000000000..cf9a3801e --- /dev/null +++ b/resources/tests/strict/assert.clsp @@ -0,0 +1,11 @@ +(mod (A) + (include *strict-cl-21*) + + (include defmac_assert.clib) + + (assert + 1 + A + 13 + ) + ) diff --git a/resources/tests/strict/defmac_assert.clib b/resources/tests/strict/defmac_assert.clib new file mode 100644 index 000000000..1dccb32c8 --- /dev/null +++ b/resources/tests/strict/defmac_assert.clib @@ -0,0 +1,10 @@ +( + (defun assert_ (items) + (if (r items) + (qq (if (unquote (f items)) (unquote (assert_ (r items))) (x))) + (f items) + ) + ) + + (defmac assert items (assert_ items)) +) \ No newline at end of file diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index c081baa74..af9e68be1 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -1022,3 +1022,38 @@ fn test_defmac_if_smoke_preprocess() { // do not make a claim about which one is identified first. assert!(result2.contains("of t1") || result2.contains("of t2") || result2.contains("of t3")); } + +#[test] +fn test_defmac_assert_smoke_preprocess() { + let result_prog = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + "-E".to_string(), + "resources/tests/strict/assert.clsp".to_string(), + ]); + assert_eq!( + result_prog, + "(mod (A) (include *strict-cl-21*) (a (i 1 (com (a (i A (com 13) (com (x))) @)) (com (x))) @))" + ); + let result_after_preproc = do_basic_run(&vec!["run".to_string(), result_prog]); + let result_with_preproc = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + "resources/tests/strict/assert.clsp".to_string(), + ]); + assert_eq!(result_after_preproc, result_with_preproc); + let run_result_true = do_basic_brun(&vec![ + "brun".to_string(), + result_with_preproc.clone(), + "(15)".to_string() + ]); + assert_eq!(run_result_true.trim(), "13"); + let run_result_false = do_basic_brun(&vec![ + "brun".to_string(), + result_with_preproc.clone(), + "(0)".to_string() + ]); + assert_eq!(run_result_false.trim(), "FAIL: clvm raise ()"); +} From 6bed699ca52646f9a7b571c383f8bee4cc75c8f7 Mon Sep 17 00:00:00 2001 From: arty Date: Fri, 16 Jun 2023 21:04:11 -0700 Subject: [PATCH 53/81] fmt --- src/tests/classic/run.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index af9e68be1..f9b744781 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -1047,13 +1047,13 @@ fn test_defmac_assert_smoke_preprocess() { let run_result_true = do_basic_brun(&vec![ "brun".to_string(), result_with_preproc.clone(), - "(15)".to_string() + "(15)".to_string(), ]); assert_eq!(run_result_true.trim(), "13"); let run_result_false = do_basic_brun(&vec![ "brun".to_string(), result_with_preproc.clone(), - "(0)".to_string() + "(0)".to_string(), ]); assert_eq!(run_result_false.trim(), "FAIL: clvm raise ()"); } From 23658399acce14b91026fe4a1d475af3f7ef4257 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 26 Jun 2023 15:56:33 -0700 Subject: [PATCH 54/81] Improve macros by renaming function arguments before interning them, allowing us to find a few more problems earlier. This revealed an old bug in the renamer which is corrected. It would always have broken potentially valid programs, so won't break anything existing. --- src/compiler/preprocessor/mod.rs | 5 +++-- src/compiler/rename.rs | 11 +++++++++-- src/tests/compiler/preprocessor.rs | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 8107ea8c6..d150ff024 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -13,6 +13,7 @@ use crate::compiler::dialect::KNOWN_DIALECTS; use crate::compiler::evaluate::{create_argument_captures, dequote, ArgInputs, Evaluator}; use crate::compiler::frontend::compile_helperform; use crate::compiler::preprocessor::macros::PreprocessorExtension; +use crate::compiler::rename::rename_args_helperform; use crate::compiler::sexp::{ decode_string, enlist, parse_sexp, Atom, NodeSel, SExp, SelectNode, ThisNode, }; @@ -255,7 +256,7 @@ impl Preprocessor { )), )); if let Some(helper) = compile_helperform(self.opts.clone(), target_defun)? { - self.add_helper(helper); + self.add_helper(rename_args_helperform(&helper)); } else { return Err(CompileErr( definition.loc(), @@ -263,7 +264,7 @@ impl Preprocessor { )); } } else if let Some(helper) = compile_helperform(self.opts.clone(), definition)? { - self.add_helper(helper); + self.add_helper(rename_args_helperform(&helper)); } } } diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 73f75db1a..2673e3946 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -164,7 +164,14 @@ fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> B BodyForm::Call(l, vs) => { let new_vs = vs .iter() - .map(|x| Rc::new(rename_in_bodyform(namemap, x.clone()))) + .enumerate() + .map(|(i,x)| { + if i == 0 { + x.clone() + } else { + Rc::new(rename_in_bodyform(namemap, x.clone())) + } + }) .collect(); BodyForm::Call(l.clone(), new_vs) } @@ -289,7 +296,7 @@ fn rename_in_helperform(namemap: &HashMap, Vec>, h: &HelperForm) -> } } -fn rename_args_helperform(h: &HelperForm) -> HelperForm { +pub fn rename_args_helperform(h: &HelperForm) -> HelperForm { match h { HelperForm::Defconstant(defc) => HelperForm::Defconstant(DefconstData { loc: defc.loc.clone(), diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 07e9838ba..9d8947227 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -455,7 +455,7 @@ fn test_defmac_string_to_number_0() { let prog = indoc! {" (mod (X_7) (defmac add-n-to (X) - (let + (let* ((stringified (symbol->string X)) (slen (string-length stringified)) (number-part (substring stringified (- slen 1) slen)) From e2e4df25dd009b1422fb89080bcb020f7b4888c0 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 26 Jun 2023 15:58:20 -0700 Subject: [PATCH 55/81] fmt --- src/compiler/rename.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 2673e3946..8c7a51ec4 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -165,7 +165,7 @@ fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> B let new_vs = vs .iter() .enumerate() - .map(|(i,x)| { + .map(|(i, x)| { if i == 0 { x.clone() } else { From 4d7bea2eb562d633f123743b0a392cfea3510fa3 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 26 Jun 2023 16:11:44 -0700 Subject: [PATCH 56/81] Add pinning of wasm-pack --- .github/workflows/npm-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index d8424af7c..395a02866 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -30,7 +30,7 @@ jobs: components: rustfmt, clippy - name: install wasm-pack - run: cargo install wasm-pack + run: cargo install --version 0.11.1 wasm-pack - name: wasm-pack build and pack run: wasm-pack build --release --target=nodejs wasm && wasm-pack pack wasm From 91799907689d2de874e7b760fb98705031bec38b Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 3 Aug 2023 09:29:47 -0700 Subject: [PATCH 57/81] fmt + clippy --- src/classic/clvm/casts.rs | 2 +- src/classic/clvm/sexp.rs | 12 ++++++------ src/classic/clvm_tools/binutils.rs | 8 ++++---- src/classic/clvm_tools/cmds.rs | 11 +++++++---- src/classic/clvm_tools/stages/stage_2/compile.rs | 4 ++-- src/classic/clvm_tools/stages/stage_2/inline.rs | 2 +- src/classic/clvm_tools/stages/stage_2/module.rs | 2 +- src/classic/clvm_tools/stages/stage_2/operators.rs | 4 ++-- src/classic/clvm_tools/stages/stage_2/optimize.rs | 2 +- src/compiler/dialect.rs | 7 +++++-- src/compiler/evaluate.rs | 3 ++- src/compiler/inline.rs | 2 +- src/compiler/repl.rs | 2 +- 13 files changed, 34 insertions(+), 27 deletions(-) diff --git a/src/classic/clvm/casts.rs b/src/classic/clvm/casts.rs index 4aeeab5ec..de10c7190 100644 --- a/src/classic/clvm/casts.rs +++ b/src/classic/clvm/casts.rs @@ -13,7 +13,7 @@ pub struct TConvertOption { } pub fn int_from_bytes( - allocator: &mut Allocator, + allocator: &Allocator, b: Bytes, option: Option, ) -> Result { diff --git a/src/classic/clvm/sexp.rs b/src/classic/clvm/sexp.rs index ef730eedd..4757014b4 100644 --- a/src/classic/clvm/sexp.rs +++ b/src/classic/clvm/sexp.rs @@ -224,7 +224,7 @@ pub fn sexp_as_bin(allocator: &mut Allocator, sexp: NodePtr) -> Bytes { f.get_value() } -pub fn bool_sexp(allocator: &mut Allocator, b: bool) -> NodePtr { +pub fn bool_sexp(allocator: &Allocator, b: bool) -> NodePtr { if b { allocator.one() } else { @@ -332,7 +332,7 @@ pub fn bool_sexp(allocator: &mut Allocator, b: bool) -> NodePtr { // ; // } -pub fn non_nil(allocator: &mut Allocator, sexp: NodePtr) -> bool { +pub fn non_nil(allocator: &Allocator, sexp: NodePtr) -> bool { match allocator.sexp(sexp) { SExp::Pair(_, _) => true, // sexp is the only node in scope, was !is_empty @@ -340,28 +340,28 @@ pub fn non_nil(allocator: &mut Allocator, sexp: NodePtr) -> bool { } } -pub fn first(allocator: &mut Allocator, sexp: NodePtr) -> Result { +pub fn first(allocator: &Allocator, sexp: NodePtr) -> Result { match allocator.sexp(sexp) { SExp::Pair(f, _) => Ok(f), _ => Err(EvalErr(sexp, "first of non-cons".to_string())), } } -pub fn rest(allocator: &mut Allocator, sexp: NodePtr) -> Result { +pub fn rest(allocator: &Allocator, sexp: NodePtr) -> Result { match allocator.sexp(sexp) { SExp::Pair(_, r) => Ok(r), _ => Err(EvalErr(sexp, "rest of non-cons".to_string())), } } -pub fn atom(allocator: &mut Allocator, sexp: NodePtr) -> Result, EvalErr> { +pub fn atom(allocator: &Allocator, sexp: NodePtr) -> Result, EvalErr> { match allocator.sexp(sexp) { SExp::Atom() => Ok(allocator.atom(sexp).to_vec()), // only sexp in scope _ => Err(EvalErr(sexp, "not an atom".to_string())), } } -pub fn proper_list(allocator: &mut Allocator, sexp: NodePtr, store: bool) -> Option> { +pub fn proper_list(allocator: &Allocator, sexp: NodePtr, store: bool) -> Option> { let mut args = vec![]; let mut args_sexp = sexp; loop { diff --git a/src/classic/clvm_tools/binutils.rs b/src/classic/clvm_tools/binutils.rs index deac313cc..9fc1e4916 100644 --- a/src/classic/clvm_tools/binutils.rs +++ b/src/classic/clvm_tools/binutils.rs @@ -111,7 +111,7 @@ pub fn ir_for_atom( * (2 2 (2) (2 3 4)) => (a 2 (a) (a 3 4)) */ pub fn disassemble_to_ir_with_kw( - allocator: &mut Allocator, + allocator: &Allocator, sexp: NodePtr, keyword_from_atom: &Record, String>, mut allow_keyword: bool, @@ -136,7 +136,7 @@ pub fn disassemble_to_ir_with_kw( } pub fn disassemble_with_kw( - allocator: &mut Allocator, + allocator: &Allocator, sexp: NodePtr, keyword_from_atom: &Record, String>, ) -> String { @@ -145,8 +145,8 @@ pub fn disassemble_with_kw( write_ir(Rc::new(symbols)) } -pub fn disassemble(allocator: &mut Allocator, sexp: NodePtr) -> String { - return disassemble_with_kw(allocator, sexp, keyword_from_atom()); +pub fn disassemble(allocator: &Allocator, sexp: NodePtr) -> String { + disassemble_with_kw(allocator, sexp, keyword_from_atom()) } pub fn assemble(allocator: &mut Allocator, s: &str) -> Result { diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index b1e3ef7e6..df5d74e6a 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -1493,7 +1493,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul )); } - let mut run_output = disassemble_with_kw(&mut allocator, result, keywords); + let mut run_output = disassemble_with_kw(&allocator, result, keywords); if let Some(ArgumentValue::ArgBool(true)) = parsed_args.get("dump") { let mut f = Stream::new(None); sexp_to_stream(&mut allocator, result, &mut f); @@ -1509,7 +1509,7 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul format!( "FAIL: {} {}", ex.1, - disassemble_with_kw(&mut allocator, ex.0, keywords) + disassemble_with_kw(&allocator, ex.0, keywords) ) })); @@ -1545,7 +1545,9 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul only_exn, &log_content, symbol_table, - &disassemble, + // Clippy: disassemble no longer requires mutability, + // but this callback interface delivers it. + &|a: &mut Allocator, sexp: NodePtr| disassemble(a, sexp), ); } else { stdout.write_str("\n"); @@ -1555,7 +1557,8 @@ pub fn launch_tool(stdout: &mut Stream, args: &[String], tool_name: &str, defaul only_exn, &log_content, symbol_table, - &disassemble, + // Same as above. + &|a: &mut Allocator, sexp: NodePtr| disassemble(a, sexp), ); } } diff --git a/src/classic/clvm_tools/stages/stage_2/compile.rs b/src/classic/clvm_tools/stages/stage_2/compile.rs index bf433bdf1..2f221c8f4 100644 --- a/src/classic/clvm_tools/stages/stage_2/compile.rs +++ b/src/classic/clvm_tools/stages/stage_2/compile.rs @@ -321,7 +321,7 @@ pub fn try_expand_macro_for_atom( } fn get_macro_program( - allocator: &mut Allocator, + allocator: &Allocator, operator: &[u8], macro_lookup: NodePtr, ) -> Result, EvalErr> { @@ -448,7 +448,7 @@ enum SymbolResult { } fn find_symbol_match( - allocator: &mut Allocator, + allocator: &Allocator, opname: &[u8], r: NodePtr, symbol_table: NodePtr, diff --git a/src/classic/clvm_tools/stages/stage_2/inline.rs b/src/classic/clvm_tools/stages/stage_2/inline.rs index e47abbf73..d0db520f1 100644 --- a/src/classic/clvm_tools/stages/stage_2/inline.rs +++ b/src/classic/clvm_tools/stages/stage_2/inline.rs @@ -11,7 +11,7 @@ use std::collections::HashMap; // (@ name substructure) // then return name and substructure. pub fn is_at_capture( - allocator: &mut Allocator, + allocator: &Allocator, tree_first: NodePtr, tree_rest: NodePtr, ) -> Option<(NodePtr, NodePtr)> { diff --git a/src/classic/clvm_tools/stages/stage_2/module.rs b/src/classic/clvm_tools/stages/stage_2/module.rs index 5bc0897bb..0c1d64952 100644 --- a/src/classic/clvm_tools/stages/stage_2/module.rs +++ b/src/classic/clvm_tools/stages/stage_2/module.rs @@ -130,7 +130,7 @@ fn build_used_constants_names( new_names = HashSet::new(); for name in iterate_names { - let functions_and_macros = vec![functions.get(&name), macro_as_dict.get(&name)]; + let functions_and_macros = [functions.get(&name), macro_as_dict.get(&name)]; let matching_names_1 = functions_and_macros .iter() diff --git a/src/classic/clvm_tools/stages/stage_2/operators.rs b/src/classic/clvm_tools/stages/stage_2/operators.rs index c9913dd62..326d00f39 100644 --- a/src/classic/clvm_tools/stages/stage_2/operators.rs +++ b/src/classic/clvm_tools/stages/stage_2/operators.rs @@ -209,7 +209,7 @@ impl CompilerOperatorsInternal { } } - fn write(&self, allocator: &mut Allocator, sexp: NodePtr) -> Response { + fn write(&self, allocator: &Allocator, sexp: NodePtr) -> Response { if let SExp::Pair(filename_sexp, r) = allocator.sexp(sexp) { if let SExp::Pair(data, _) = allocator.sexp(r) { if let SExp::Atom() = allocator.sexp(filename_sexp) { @@ -280,7 +280,7 @@ impl CompilerOperatorsInternal { pub fn set_symbol_table( &self, - allocator: &mut Allocator, + allocator: &Allocator, table: NodePtr, ) -> Result { if let Some(symtable) = diff --git a/src/classic/clvm_tools/stages/stage_2/optimize.rs b/src/classic/clvm_tools/stages/stage_2/optimize.rs index 5e1e2306d..199262a6d 100644 --- a/src/classic/clvm_tools/stages/stage_2/optimize.rs +++ b/src/classic/clvm_tools/stages/stage_2/optimize.rs @@ -136,7 +136,7 @@ pub fn constant_optimizer( Ok(r) } -pub fn is_args_call(allocator: &mut Allocator, r: NodePtr) -> bool { +pub fn is_args_call(allocator: &Allocator, r: NodePtr) -> bool { if let SExp::Atom() = allocator.sexp(r) { // Only r in scope. let buf = allocator.atom(r); diff --git a/src/compiler/dialect.rs b/src/compiler/dialect.rs index 19f5ad92a..953bc553d 100644 --- a/src/compiler/dialect.rs +++ b/src/compiler/dialect.rs @@ -85,10 +85,13 @@ lazy_static! { }; } -fn include_dialect(allocator: &mut Allocator, e: &[NodePtr]) -> Option { +fn include_dialect(allocator: &Allocator, e: &[NodePtr]) -> Option { let include_keyword_sexp = e[0]; let name_sexp = e[1]; - if let (SExp::Atom(), SExp::Atom()) = (allocator.sexp(include_keyword_sexp), allocator.sexp(name_sexp)) { + if let (SExp::Atom(), SExp::Atom()) = ( + allocator.sexp(include_keyword_sexp), + allocator.sexp(name_sexp), + ) { if allocator.atom(include_keyword_sexp) == "include".as_bytes().to_vec() { if let Some(dialect) = KNOWN_DIALECTS.get(&decode_string(allocator.atom(name_sexp))) { return Some(dialect.accepted.clone()); diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index a81245538..f4901dfd5 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -814,7 +814,8 @@ impl<'info> Evaluator { prog_args, )))) } else if call.name == "com".as_bytes() { - let use_body = self.make_com_module(&call.loc, prog_args, arguments_to_convert[0].to_sexp()); + let use_body = + self.make_com_module(&call.loc, prog_args, arguments_to_convert[0].to_sexp()); let compiled = self.compile_code(allocator, false, use_body)?; let compiled_borrowed: &SExp = compiled.borrow(); Ok(Rc::new(BodyForm::Quoted(compiled_borrowed.clone()))) diff --git a/src/compiler/inline.rs b/src/compiler/inline.rs index 0d27e8875..604014e2b 100644 --- a/src/compiler/inline.rs +++ b/src/compiler/inline.rs @@ -262,7 +262,7 @@ fn get_inline_callable( /// expressions given in the ultimate original call. This allows inline functions /// to seem to call each other as long as there's no cycle. fn make_args_for_call_from_inline( - visited_inlines: &mut HashSet>, + visited_inlines: &HashSet>, runner: Rc, opts: Rc, compiler: &PrimaryCodegen, diff --git a/src/compiler/repl.rs b/src/compiler/repl.rs index ce8101f2c..e8c228311 100644 --- a/src/compiler/repl.rs +++ b/src/compiler/repl.rs @@ -92,7 +92,7 @@ impl Repl { let loc = Srcloc::start(&opts.filename()); let mut toplevel_forms = HashSet::new(); - for w in vec!["defun", "defun-inline", "defconstant", "defmacro"].iter() { + for w in ["defun", "defun-inline", "defconstant", "defmacro"].iter() { toplevel_forms.insert(w.to_string()); } From df41c1be2019093a32f61ef37e6939d27e7d2b7a Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 7 Aug 2023 11:46:59 -0700 Subject: [PATCH 58/81] transport comment from base --- src/compiler/dialect.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/compiler/dialect.rs b/src/compiler/dialect.rs index 953bc553d..ba2641489 100644 --- a/src/compiler/dialect.rs +++ b/src/compiler/dialect.rs @@ -102,6 +102,13 @@ fn include_dialect(allocator: &Allocator, e: &[NodePtr]) -> Option AcceptedDialect { let mut result = Default::default(); From 351c4f81ba7f9f636e22a95507f15c4530f12891 Mon Sep 17 00:00:00 2001 From: arty Date: Mon, 7 Aug 2023 18:04:37 -0700 Subject: [PATCH 59/81] Add a comment --- src/classic/clvm_tools/cmds.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/classic/clvm_tools/cmds.rs b/src/classic/clvm_tools/cmds.rs index 2039ff116..fbfd5fc4c 100644 --- a/src/classic/clvm_tools/cmds.rs +++ b/src/classic/clvm_tools/cmds.rs @@ -808,6 +808,14 @@ fn fix_log( } } +// A function which performs preprocessing on a whole program and renders the +// output to the user. +// +// This is used in the same way as cc -E in a C compiler; to see what +// preprocessing did to the source so you can debug and improve your macros. +// +// Without this, it's difficult for some to visualize how macro are functioning +// and what forms they output. fn perform_preprocessing( stdout: &mut Stream, opts: Rc, @@ -815,7 +823,19 @@ fn perform_preprocessing( program_text: &str, ) -> Result<(), CompileErr> { let srcloc = Srcloc::start(input_file); + // Parse the source file. let parsed = parse_sexp(srcloc.clone(), program_text.bytes())?; + // Get the detected dialect and compose a sigil that matches. + // Classic preprocessing (also shared by standard sigil 21 and 21) does macro + // expansion during the compile process, making all macros available to all + // code regardless of its lexical order and therefore isn't rendered in a + // unified way (for example, 'com' and 'mod' forms invoke macros when + // encountered and expanded. By contrast strict mode reads the macros and + // evaluates them in that order (as in C). + // + // The result is fully rendered before the next stage of compilation so that + // it can be inspected and so that the execution environment for macros is + // fully and cleanly separated from compile time. let stepping_form_text = match opts.dialect().stepping { Some(21) => Some("(include *strict-cl-21*)".to_string()), Some(n) => Some(format!("(include *standard-cl-{n}*)")), From 6725447787faa945f3ac0f400b1f7dc0ab0ffcd9 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 8 Aug 2023 08:49:28 -0700 Subject: [PATCH 60/81] Add more commentary --- src/compiler/codegen.rs | 14 ++++++++++++++ src/compiler/rename.rs | 10 ++++++++++ 2 files changed, 24 insertions(+) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index e9666abde..1dc4ecf09 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -563,6 +563,16 @@ pub fn generate_expr_code( create_name_lookup(compiler, l.clone(), atom) .map(|f| Ok(CompiledCode(l.clone(), f))) .unwrap_or_else(|_| { + // Finally enable strictness for variable names. + // This is possible because the modern macro system + // takes great care to preserve as much information + // from the source code as possible. + // + // When we come here in strict mode, we have + // a string, integer or atom depending on the + // user's desire and the explicitly generated + // result from the macro, therefore we can return + // an error if this atom didn't have a binding. if opts.dialect().strict { return Err(CompileErr( l.clone(), @@ -587,6 +597,10 @@ pub fn generate_expr_code( } } SExp::Integer(l, i) => { + // This code can assume that an integer is an integer because + // strict mode closes the necessary loophole below. Values + // intended as variable names are never crushed into integer + // like values from modern macros. if opts.dialect().strict { return generate_expr_code( allocator, diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 319b0e819..c8219175a 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -183,6 +183,8 @@ fn rename_in_bodyform(namemap: &HashMap, Vec>, b: Rc) -> B } } +/// Given a set of sequential bindings, create a stack of let forms that have +/// the same meaning. This is used to propogate renaming. pub fn desugar_sequential_let_bindings( bindings: &[Rc], body: &BodyForm, @@ -373,6 +375,8 @@ fn rename_in_compileform(namemap: &HashMap, Vec>, c: Rc } } +/// For all the HelperForms in a CompileForm, do renaming in them so that all +/// unique variable bindings in the program have unique names. pub fn rename_children_compileform(c: &CompileForm) -> CompileForm { let local_renamed_helpers = c.helpers.iter().map(rename_args_helperform).collect(); let local_renamed_body = rename_args_bodyform(c.exp.borrow()); @@ -385,6 +389,12 @@ pub fn rename_children_compileform(c: &CompileForm) -> CompileForm { } } +/// Given a compileform, perform renaming in descendants so that every variable +/// name that lives in a different scope has a unique name. This allows +/// compilation to treat identical forms as equivalent and ensures that forms +/// that look the same but refer to different variables are different. It also +/// ensures that future tricky variable name uses decide on one binding from their +/// lexical scope. pub fn rename_args_compileform(c: &CompileForm) -> CompileForm { let new_names = invent_new_names_sexp(c.args.clone()); let mut local_namemap = HashMap::new(); From f183d5b78a367c020ae68763cb97ecb7206c6713 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 9 Aug 2023 15:07:12 -0700 Subject: [PATCH 61/81] Compiled macros rather than interpreted in the evaluator --- src/compiler/compiler.rs | 26 +- src/compiler/evaluate.rs | 59 +--- src/compiler/frontend.rs | 8 +- src/compiler/preprocessor/macros.rs | 530 ++++++---------------------- src/compiler/preprocessor/mod.rs | 68 ++-- src/compiler/runtypes.rs | 10 + src/compiler/sexp.rs | 2 +- src/tests/compiler/preprocessor.rs | 2 +- 8 files changed, 201 insertions(+), 504 deletions(-) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index fc196fae5..00c06c47c 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -158,16 +158,13 @@ fn fe_opt( }) } -pub fn compile_pre_forms( +pub fn compile_from_compileform( allocator: &mut Allocator, runner: Rc, opts: Rc, - pre_forms: &[Rc], + p0: CompileForm, symbol_table: &mut HashMap, ) -> Result { - // Resolve includes, convert program source to lexemes - let p0 = frontend(opts.clone(), pre_forms)?; - let p1 = if opts.frontend_opt() { // Front end optimization fe_opt(allocator, runner.clone(), opts.clone(), p0)? @@ -197,6 +194,25 @@ pub fn compile_pre_forms( codegen(allocator, runner, opts, &p2, symbol_table) } +pub fn compile_pre_forms( + allocator: &mut Allocator, + runner: Rc, + opts: Rc, + pre_forms: &[Rc], + symbol_table: &mut HashMap, +) -> Result { + // Resolve includes, convert program source to lexemes + let p0 = frontend(opts.clone(), pre_forms)?; + + compile_from_compileform( + allocator, + runner, + opts, + p0, + symbol_table + ) +} + pub fn compile_file( allocator: &mut Allocator, runner: Rc, diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index f4901dfd5..e0b4f7da6 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -9,7 +9,7 @@ use clvm_rs::allocator::Allocator; use crate::classic::clvm::__type_compatibility__::{bi_one, bi_zero}; use crate::classic::clvm_tools::stages::stage_0::TRunProgram; -use crate::compiler::clvm::{run, PrimOverride}; +use crate::compiler::clvm::run; use crate::compiler::codegen::codegen; use crate::compiler::compiler::is_at_capture; use crate::compiler::comptypes::{ @@ -135,61 +135,6 @@ pub struct Evaluator { ignore_exn: bool, } -fn compile_to_run_err(e: CompileErr) -> RunFailure { - match e { - CompileErr(l, e) => RunFailure::RunErr(l, e), - } -} - -impl PrimOverride for Evaluator { - fn try_handle( - &self, - head: Rc, - _context: Rc, - tail: Rc, - ) -> Result>, RunFailure> { - let have_args: Vec> = if let Some(args_list) = tail.proper_list() { - args_list - .iter() - .map(|e| Rc::new(BodyForm::Quoted(e.clone()))) - .collect() - } else { - return Ok(None); - }; - - if let SExp::Atom(hl, head_atom) = head.borrow() { - let mut call_args = vec![Rc::new(BodyForm::Value(SExp::Atom( - hl.clone(), - head_atom.clone(), - )))]; - call_args.append(&mut have_args.clone()); - // Primitives can't have tails. - let call_form = Rc::new(BodyForm::Call(head.loc(), call_args, None)); - - for x in self.extensions.iter() { - if let Some(res) = x - .try_eval( - self, - Rc::new(SExp::Nil(head.loc())), - &HashMap::new(), - &head.loc(), - head_atom, - &have_args, - call_form.clone(), - ) - .map_err(compile_to_run_err)? - { - return dequote(head.loc(), res) - .map_err(compile_to_run_err) - .map(Some); - } - } - } - - Ok(None) - } -} - fn select_helper(bindings: &[HelperForm], name: &[u8]) -> Option { for b in bindings.iter() { if b.name() == name { @@ -1383,7 +1328,7 @@ impl<'info> Evaluator { self.prims.clone(), prim, args, - Some(self), + None, Some(PRIM_RUN_LIMIT), ) .map_err(|e| match e { diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index 7e2b38348..d936a8d75 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -251,8 +251,9 @@ fn make_let_bindings( ) -> Result>, CompileErr> { let err = Err(CompileErr( body.loc(), - "Bad binding tail ".to_string() + &body.to_string(), + format!("Bad binding tail {body:?}") )); + eprintln!("make_let_bindings {body}"); match body.borrow() { SExp::Nil(_) => Ok(vec![]), SExp::Cons(_, head, tl) => head @@ -271,7 +272,10 @@ fn make_let_bindings( result.append(&mut rest_bindings); Ok(result) } - _ => err.clone(), + _ => { + eprintln!("crap {body:?}"); + err.clone() + } }) .unwrap_or_else(|| err.clone()), _ => err, diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 9eae22249..859d8c79b 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -2,31 +2,24 @@ use std::borrow::Borrow; use std::collections::HashMap; use std::rc::Rc; -use clvmr::allocator::Allocator; use num_bigint::ToBigInt; use num_traits::ToPrimitive; use crate::classic::clvm::__type_compatibility__::{bi_one, bi_zero}; -use crate::compiler::clvm::truthy; -use crate::compiler::comptypes::{BodyForm, CompileErr, CompilerOpts}; -use crate::compiler::evaluate::{EvalExtension, Evaluator}; -use crate::compiler::preprocessor::dequote; -use crate::compiler::sexp::{decode_string, SExp}; +use crate::compiler::clvm::PrimOverride; +use crate::compiler::comptypes::{CompileErr, CompilerOpts}; +use crate::compiler::runtypes::RunFailure; +use crate::compiler::sexp::{decode_string, printable, SExp}; use crate::compiler::srcloc::Srcloc; use crate::util::{number_from_u8, Number}; // If the bodyform represents a constant, only match a quoted string. -fn match_quoted_string(body: Rc) -> Result)>, CompileErr> { +fn match_quoted_string(body: Rc) -> Result)>, CompileErr> { let is_string = match body.borrow() { - BodyForm::Quoted(SExp::QuotedString(_, b'x', _)) => None, - BodyForm::Quoted(SExp::QuotedString(al, _, an)) => Some((al.clone(), an.clone())), - BodyForm::Value(SExp::QuotedString(_, b'x', _)) => None, - BodyForm::Value(SExp::QuotedString(al, _, an)) => Some((al.clone(), an.clone())), - BodyForm::Quoted(_) => None, - _ => { - return Ok(None); - } + SExp::QuotedString(_, b'x', _) => None, + SExp::QuotedString(al, _, an) => Some((al.clone(), an.clone())), + _ => None }; if let Some((loc, s)) = is_string { @@ -36,13 +29,11 @@ fn match_quoted_string(body: Rc) -> Result)>, } } -fn match_atom(body: Rc) -> Result)>, CompileErr> { - if let BodyForm::Quoted(SExp::Atom(al, an)) = body.borrow() { +fn match_atom(body: Rc) -> Result)>, CompileErr> { + if let SExp::Atom(al, an) = body.borrow() { Ok(Some((al.clone(), an.clone()))) - } else if let BodyForm::Quoted(_) = body.borrow() { - Err(CompileErr(body.loc(), "atom required".to_string())) } else { - Ok(None) + Err(CompileErr(body.loc(), "atom required".to_string())) } } @@ -51,23 +42,31 @@ enum MatchedNumber { MatchedHex(Srcloc, Vec), } -fn match_number(body: Rc) -> Result, CompileErr> { +fn match_number(body: Rc) -> Result, CompileErr> { match body.borrow() { - BodyForm::Quoted(SExp::Integer(il, n)) => { - Ok(Some(MatchedNumber::MatchedInt(il.clone(), n.clone()))) + SExp::Integer(il, n) => { + return Ok(Some(MatchedNumber::MatchedInt(il.clone(), n.clone()))); } - BodyForm::Quoted(SExp::QuotedString(ql, b'x', b)) => { - Ok(Some(MatchedNumber::MatchedHex(ql.clone(), b.clone()))) + SExp::QuotedString(ql, b'x', b) => { + return Ok(Some(MatchedNumber::MatchedHex(ql.clone(), b.clone()))); } - BodyForm::Quoted(SExp::Nil(il)) => { - Ok(Some(MatchedNumber::MatchedInt(il.clone(), bi_zero()))) + SExp::Atom(al, b) => { + // An atom with unprintable characters is rendered as an integer. + if !printable(&b) { + let to_integer = number_from_u8(&b); + return Ok(Some(MatchedNumber::MatchedInt(al.clone(), to_integer))); + } } - BodyForm::Quoted(_) => Err(CompileErr(body.loc(), "number required".to_string())), - _ => Ok(None), + SExp::Nil(il) => { + return Ok(Some(MatchedNumber::MatchedInt(il.clone(), bi_zero()))); + } + _ => { } } + + Err(CompileErr(body.loc(), "number required".to_string())) } -fn numeric_value(body: Rc) -> Result { +fn numeric_value(body: Rc) -> Result { match match_number(body.clone())? { Some(MatchedNumber::MatchedInt(_, n)) => Ok(n), Some(MatchedNumber::MatchedHex(_, h)) => Ok(number_from_u8(&h)), @@ -75,7 +74,7 @@ fn numeric_value(body: Rc) -> Result { } } -fn usize_value(body: Rc) -> Result { +fn usize_value(body: Rc) -> Result { let n = numeric_value(body.clone())?; if let Some(res) = n.to_usize() { Ok(res) @@ -84,28 +83,6 @@ fn usize_value(body: Rc) -> Result { } } -fn reify_args( - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - args: &[Rc], -) -> Result>, CompileErr> { - let mut allocator = Allocator::new(); - let mut converted_args = Vec::new(); - for a in args.iter() { - let shrunk = evaluator.shrink_bodyform( - &mut allocator, - prog_args.clone(), - env, - a.clone(), - false, - None, - )?; - converted_args.push(shrunk); - } - Ok(converted_args) -} - /// A container for a function to evaluate in advanced preprocessor macros. /// We use this trait (which is very similar to the extension trait in Evaluator) /// as a definite handler for a specific named form, so optional returns aren't @@ -119,14 +96,9 @@ pub trait ExtensionFunction { #[allow(clippy::too_many_arguments)] fn try_eval( &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, loc: &Srcloc, - name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr>; + args: &[Rc], + ) -> Result, CompileErr>; } struct StringQ {} @@ -144,23 +116,15 @@ impl ExtensionFunction for StringQ { fn try_eval( &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { + args: &[Rc], + ) -> Result, CompileErr> { let res = match match_quoted_string(args[0].clone()) { Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), - Ok(None) => { - return Ok(body); - } - Err(_) => SExp::Nil(loc.clone()), + _ => SExp::Nil(loc.clone()), }; - Ok(Rc::new(BodyForm::Quoted(res))) + Ok(Rc::new(res)) } } @@ -179,23 +143,15 @@ impl ExtensionFunction for NumberQ { fn try_eval( &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { + args: &[Rc], + ) -> Result, CompileErr> { let res = match match_number(args[0].clone()) { Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), - Ok(None) => { - return Ok(body); - } - Err(_) => SExp::Nil(loc.clone()), + _ => SExp::Nil(loc.clone()), }; - Ok(Rc::new(BodyForm::Quoted(res))) + Ok(Rc::new(res)) } } @@ -214,23 +170,15 @@ impl ExtensionFunction for SymbolQ { fn try_eval( &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { + args: &[Rc], + ) -> Result, CompileErr> { let res = match match_atom(args[0].clone()) { Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), - Ok(None) => { - return Ok(body); - } - Err(_) => SExp::Nil(loc.clone()), + _ => SExp::Nil(loc.clone()), }; - Ok(Rc::new(BodyForm::Quoted(res))) + Ok(Rc::new(res)) } } @@ -249,20 +197,15 @@ impl ExtensionFunction for SymbolToString { fn try_eval( &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, _loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { + args: &[Rc], + ) -> Result, CompileErr> { if let Some((loc, value)) = match_atom(args[0].clone())? { - Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( + Ok(Rc::new(SExp::QuotedString( loc, b'\"', value, - )))) + ))) } else { - Ok(body) + Err(CompileErr(args[0].loc(), "Not a symbol".to_string())) } } } @@ -282,18 +225,13 @@ impl ExtensionFunction for StringToSymbol { fn try_eval( &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, _loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { + args: &[Rc], + ) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { - Ok(Rc::new(BodyForm::Quoted(SExp::Atom(loc, value)))) + Ok(Rc::new(SExp::Atom(loc, value))) } else { - Ok(body) + Err(CompileErr(args[0].loc(), "Not a string".to_string())) } } } @@ -313,14 +251,9 @@ impl ExtensionFunction for StringAppend { fn try_eval( &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, - _loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { + loc: &Srcloc, + args: &[Rc], + ) -> Result, CompileErr> { let mut out_vec = Vec::new(); let mut out_loc = None; for a in args.iter() { @@ -330,14 +263,14 @@ impl ExtensionFunction for StringAppend { } out_vec.append(&mut value); } else { - return Ok(body); + return Err(CompileErr(a.loc(), "not a quoted string".to_string())); } } - Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( - out_loc.unwrap_or_else(|| body.loc()), + Ok(Rc::new(SExp::QuotedString( + out_loc.unwrap_or_else(|| loc.clone()), b'\"', out_vec, - )))) + ))) } } @@ -356,27 +289,22 @@ impl ExtensionFunction for NumberToString { fn try_eval( &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, _loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { + args: &[Rc], + ) -> Result, CompileErr> { let match_res = match_number(args[0].clone())?; let (use_loc, int_val) = match &match_res { Some(MatchedNumber::MatchedInt(l, i)) => (l.clone(), i.clone()), Some(MatchedNumber::MatchedHex(l, h)) => (l.clone(), number_from_u8(h)), _ => { - return Ok(body); + return Err(CompileErr(args[0].loc(), "Not a number".to_string())); } }; - Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( + Ok(Rc::new(SExp::QuotedString( use_loc, b'\"', int_val.to_string().as_bytes().to_vec(), - )))) + ))) } } @@ -395,17 +323,12 @@ impl ExtensionFunction for StringToNumber { fn try_eval( &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, loc: &Srcloc, - _name: &[u8], - args: &[Rc], - _body: Rc, - ) -> Result, CompileErr> { + args: &[Rc], + ) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { if let Ok(cvt_bi) = decode_string(&value).parse::() { - Ok(Rc::new(BodyForm::Quoted(SExp::Integer(loc, cvt_bi)))) + Ok(Rc::new(SExp::Integer(loc, cvt_bi))) } else { Err(CompileErr(loc, "bad number".to_string())) } @@ -433,23 +356,16 @@ impl ExtensionFunction for StringLength { fn try_eval( &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, _loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { + args: &[Rc], + ) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { if let Some(len_bi) = value.len().to_bigint() { - Ok(Rc::new(BodyForm::Quoted(SExp::Integer(loc, len_bi)))) - } else { - Err(CompileErr(loc, "Error getting string length".to_string())) + return Ok(Rc::new(SExp::Integer(loc, len_bi))); } - } else { - Ok(body) } + + Err(CompileErr(args[0].loc(), "Error getting string length".to_string())) } } @@ -468,19 +384,14 @@ impl ExtensionFunction for Substring { fn try_eval( &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, _loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { + args: &[Rc], + ) -> Result, CompileErr> { let start_element = usize_value(args[1].clone())?; let end_element = usize_value(args[2].clone())?; match args[0].borrow() { - BodyForm::Quoted(SExp::QuotedString(l, ch, s)) => { + SExp::QuotedString(l, ch, s) => { if start_element > end_element || start_element > s.len() || end_element > s.len() { return Err(CompileErr( l.clone(), @@ -493,221 +404,13 @@ impl ExtensionFunction for Substring { .skip(start_element) .copied() .collect(); - Ok(Rc::new(BodyForm::Quoted(SExp::QuotedString( + Ok(Rc::new(SExp::QuotedString( l.clone(), *ch, result_value, - )))) - } - BodyForm::Quoted(_) => Err(CompileErr(body.loc(), "Not a string".to_string())), - _ => Ok(body), - } - } -} - -struct List {} - -impl List { - fn create() -> Rc { - Rc::new(List {}) - } -} - -impl ExtensionFunction for List { - fn required_args(&self) -> Option { - None - } - - fn try_eval( - &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - _name: &[u8], - args: &[Rc], - _body: Rc, - ) -> Result, CompileErr> { - let mut allocator = Allocator::new(); - let mut res = Rc::new(BodyForm::Quoted(SExp::Nil(loc.clone()))); - for a in args.iter().rev() { - res = Rc::new(BodyForm::Call( - loc.clone(), - vec![ - Rc::new(BodyForm::Value(SExp::Atom(loc.clone(), b"c".to_vec()))), - a.clone(), - res, - ], - // Calls primitive 'c' so no tail. - None, - )); - } - evaluator.shrink_bodyform(&mut allocator, prog_args, env, res, false, None) - } -} - -struct Cons {} - -impl Cons { - fn create() -> Rc { - Rc::new(Cons {}) - } -} - -impl ExtensionFunction for Cons { - fn required_args(&self) -> Option { - Some(2) - } - - fn try_eval( - &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, - loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { - if let (BodyForm::Quoted(a), BodyForm::Quoted(b)) = (args[0].borrow(), args[1].borrow()) { - Ok(Rc::new(BodyForm::Quoted(SExp::Cons( - loc.clone(), - Rc::new(a.clone()), - Rc::new(b.clone()), - )))) - } else { - Ok(body) - } - } -} - -struct First {} - -impl First { - fn create() -> Rc { - Rc::new(First {}) - } -} - -impl ExtensionFunction for First { - fn required_args(&self) -> Option { - Some(1) - } - - fn try_eval( - &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, - loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { - if let BodyForm::Quoted(SExp::Cons(_, a, _)) = args[0].borrow() { - let a_borrowed: &SExp = a.borrow(); - Ok(Rc::new(BodyForm::Quoted(a_borrowed.clone()))) - } else if let BodyForm::Quoted(_) = args[0].borrow() { - Err(CompileErr(loc.clone(), "bad cons in first".to_string())) - } else { - Ok(body) - } - } -} - -struct Rest {} - -impl Rest { - fn create() -> Rc { - Rc::new(Rest {}) - } -} - -impl ExtensionFunction for Rest { - fn required_args(&self) -> Option { - Some(1) - } - - fn try_eval( - &self, - _evaluator: &Evaluator, - _prog_args: Rc, - _env: &HashMap, Rc>, - loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { - if let BodyForm::Quoted(SExp::Cons(_, _, b)) = args[0].borrow() { - let a_borrowed: &SExp = b.borrow(); - Ok(Rc::new(BodyForm::Quoted(a_borrowed.clone()))) - } else if let BodyForm::Quoted(_) = args[0].borrow() { - Err(CompileErr(loc.clone(), "bad cons in rest".to_string())) - } else { - Ok(body) - } - } -} - -struct If {} - -impl If { - fn create() -> Rc { - Rc::new(If {}) - } -} - -impl ExtensionFunction for If { - fn want_interp(&self) -> bool { - false - } - - fn required_args(&self) -> Option { - Some(3) - } - - fn try_eval( - &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - _loc: &Srcloc, - _name: &[u8], - args: &[Rc], - body: Rc, - ) -> Result, CompileErr> { - let mut allocator = Allocator::new(); - let cond_result = evaluator.shrink_bodyform( - &mut allocator, - prog_args.clone(), - env, - args[0].clone(), - false, - None, - )?; - - if let Ok(unquoted) = dequote(body.loc(), cond_result) { - if truthy(unquoted) { - evaluator.shrink_bodyform( - &mut allocator, - prog_args, - env, - args[1].clone(), - false, - None, - ) - } else { - evaluator.shrink_bodyform( - &mut allocator, - prog_args, - env, - args[2].clone(), - false, - None, - ) + ))) } - } else { - Ok(body.clone()) + _ => Err(CompileErr(args[0].loc(), "Not a string".to_string())), } } } @@ -750,14 +453,44 @@ pub struct PreprocessorExtension { extfuns: HashMap, Rc>, } +fn compile_to_run_err(e: CompileErr) -> RunFailure { + match e { + CompileErr(l, e) => RunFailure::RunErr(l, e), + } +} + +impl PrimOverride for PreprocessorExtension { + fn try_handle( + &self, + head: Rc, + _context: Rc, + tail: Rc, + ) -> Result>, RunFailure> { + eprintln!("running {head} {tail}"); + if let SExp::Atom(hl, head_atom) = head.borrow() { + let have_args: Vec> = + if let Some(args_list) = tail.proper_list() { + args_list.into_iter().map(Rc::new).collect() + } else { + return Ok(None); + }; + + if let Some(extension) = self.extfuns.get(head_atom) { + let res = extension.try_eval(&hl, &have_args) + .map_err(compile_to_run_err)?; + + eprintln!("res = {res}"); + return Ok(Some(res)); + } + } + + Ok(None) + } +} + impl PreprocessorExtension { pub fn new() -> Self { let extfuns = [ - (b"if".to_vec(), If::create()), - (b"list".to_vec(), List::create()), - (b"c".to_vec(), Cons::create()), - (b"f".to_vec(), First::create()), - (b"r".to_vec(), Rest::create()), (b"string?".to_vec(), StringQ::create()), (b"number?".to_vec(), NumberQ::create()), (b"symbol?".to_vec(), SymbolQ::create()), @@ -791,38 +524,3 @@ impl PreprocessorExtension { opts.set_prim_map(Rc::new(new_prim_map_cloned)) } } - -impl EvalExtension for PreprocessorExtension { - fn try_eval( - &self, - evaluator: &Evaluator, - prog_args: Rc, - env: &HashMap, Rc>, - loc: &Srcloc, - name: &[u8], - raw_args: &[Rc], - body: Rc, - ) -> Result>, CompileErr> { - if let Some(extfun) = self.extfuns.get(name) { - if let Some(n) = extfun.required_args() { - if raw_args.len() != n { - return Err(CompileErr( - loc.clone(), - format!("{} requires {} args", decode_string(name), n), - )); - } - } - - let args = if extfun.want_interp() { - reify_args(evaluator, prog_args.clone(), env, raw_args)? - } else { - raw_args.to_vec() - }; - Ok(Some(extfun.try_eval( - evaluator, prog_args, env, loc, name, &args, body, - )?)) - } else { - Ok(None) - } - } -} diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 0dc9042f6..6cff08790 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -10,12 +10,14 @@ use crate::classic::clvm_tools::clvmc::compile_clvm_text_maybe_opt; use crate::classic::clvm_tools::stages::stage_0::{DefaultProgramRunner, TRunProgram}; use crate::compiler::cldb::hex_to_modern_sexp; -use crate::compiler::clvm::convert_from_clvm_rs; +use crate::compiler::clvm; +use crate::compiler::clvm::{convert_from_clvm_rs, truthy}; +use crate::compiler::compiler::compile_from_compileform; use crate::compiler::comptypes::{ - BodyForm, CompileErr, CompilerOpts, HelperForm, IncludeDesc, IncludeProcessType, + BodyForm, CompileErr, CompileForm, CompilerOpts, HelperForm, IncludeDesc, IncludeProcessType, }; use crate::compiler::dialect::KNOWN_DIALECTS; -use crate::compiler::evaluate::{create_argument_captures, dequote, ArgInputs, Evaluator}; +use crate::compiler::evaluate::{create_argument_captures, ArgInputs}; use crate::compiler::frontend::compile_helperform; use crate::compiler::preprocessor::macros::PreprocessorExtension; use crate::compiler::rename::rename_args_helperform; @@ -69,6 +71,22 @@ fn make_defmac_name(name: &[u8]) -> Vec { res } +fn nilize(v: Rc) -> Rc { + if let SExp::Cons(l,a,b) = v.borrow() { + let a_conv = nilize(a.clone()); + let b_conv = nilize(b.clone()); + if Rc::as_ptr(&a_conv) == Rc::as_ptr(&a) && Rc::as_ptr(&b_conv) == Rc::as_ptr(&b) { + v.clone() + } else { + Rc::new(SExp::Cons(l.clone(), a_conv, b_conv)) + } + } else if !truthy(v.clone()) { + Rc::new(SExp::Nil(v.loc())) + } else { + v + } +} + impl Preprocessor { pub fn new(opts: Rc) -> Self { let runner = Rc::new(DefaultProgramRunner::new()); @@ -278,29 +296,35 @@ impl Preprocessor { )?; let ppext = Rc::new(PreprocessorExtension::new()); - let mut eval = Evaluator::new( - ppext.enrich_prims(self.opts.clone()), + let extension: &PreprocessorExtension = ppext.borrow(); + let opts_prims = extension.enrich_prims(self.opts.clone()); + let new_program = CompileForm { + loc: body.loc(), + args: mdata.args.clone(), + include_forms: vec![], + helpers: self.helpers.clone(), + exp: mdata.body.clone(), + }; + let mut symbol_table = HashMap::new(); + let compiled_program = compile_from_compileform( + &mut allocator, self.runner.clone(), - self.helpers.clone(), - ); - eval.add_extension(ppext); - let res = eval.shrink_bodyform( + opts_prims.clone(), + new_program, + &mut symbol_table, + )?; + let res = clvm::run( &mut allocator, - mdata.args.clone(), - ¯o_arg_env, - mdata.body.clone(), - false, + self.runner.clone(), + opts_prims.prim_map(), + Rc::new(compiled_program), + args.clone(), + Some(extension), None, - )?; + ).map(nilize).map_err(|e| CompileErr::from(e))?; - if let Ok(unquoted) = dequote(body.loc(), res) { - return Ok(Some(unquoted)); - } else { - return Err(CompileErr( - body.loc(), - "Failed to fully evaluate macro".to_string(), - )); - } + eprintln!("macro {} {args} => {res}", decode_string(&name)); + return Ok(Some(res)); } } } diff --git a/src/compiler/runtypes.rs b/src/compiler/runtypes.rs index 720822337..c19d5cc45 100644 --- a/src/compiler/runtypes.rs +++ b/src/compiler/runtypes.rs @@ -1,6 +1,7 @@ use std::fmt::Display; use std::rc::Rc; +use crate::compiler::comptypes::CompileErr; use crate::compiler::sexp::SExp; use crate::compiler::srcloc::Srcloc; @@ -27,3 +28,12 @@ impl Display for RunFailure { Ok(()) } } + +impl From for CompileErr { + fn from(item: RunFailure) -> Self { + match item { + RunFailure::RunExn(l, s) => CompileErr(l.clone(), format!("Runtime exception: {s}")), + RunFailure::RunErr(l, s) => CompileErr(l.clone(), format!("Runtime error: {s}")), + } + } +} diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 8135428e1..93ef4c848 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -361,7 +361,7 @@ pub fn decode_string(v: &[u8]) -> String { return String::from_utf8_lossy(v).as_ref().to_string(); } -fn printable(a: &[u8]) -> bool { +pub fn printable(a: &[u8]) -> bool { for ch in a.iter() { if (*ch as char).is_control() || !(*ch as char).is_ascii() { return false; diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 9d8947227..4ada850c1 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -325,7 +325,7 @@ fn test_defmac_create_match_form() { (match X ((16 x y) (c 1 (+ x y))) ((3 () b c) c) - ((3 (q . 1) b c) b) + ((3 1 b c) b) (x x) ) ) From a5e0c13943db1829bb4e290bb5c464d8b7daa395 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 12 Sep 2023 10:04:57 -0700 Subject: [PATCH 62/81] Add continued if --- src/compiler/compiler.rs | 11 ++++++++++- src/tests/classic/run.rs | 30 ++++++++++++++++++++++++++++++ src/tests/compiler/preprocessor.rs | 4 +++- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index fc196fae5..00f5c7bc5 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -45,10 +45,19 @@ lazy_static! { }; pub static ref ADVANCED_MACROS: String = { indoc! {"( - (defmac if (A B C) + (defmac __chia__primitive__if (A B C) (qq (a (i (unquote A) (com (unquote B)) (com (unquote C))) @)) ) + (defun __chia__if (ARGS) + (__chia__primitive__if (r (r (r ARGS))) + (qq (i (unquote (f ARGS)) (com (unquote (f (r ARGS)))) (unquote (__chia__if (r (r ARGS)))))) + (qq (i (unquote (f ARGS)) (com (unquote (f (r ARGS)))) (com (unquote (f (r (r ARGS))))))) + ) + ) + + (defmac if ARGS (qq (a (unquote (__chia__if ARGS)) @))) + (defun __chia__compile-list (args) (if args (c 4 (c (f args) (c (__chia__compile-list (r args)) ()))) diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index 24681f4e7..be2b99733 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -1430,3 +1430,33 @@ fn test_classic_obeys_operator_choice_at_compile_time_version_0() { .to_string(); assert_eq!(compiled, "FAIL: unimplemented operator 48"); } + +#[test] +fn test_continued_if() { + let prog = indoc! {" +(mod X + (include *strict-cl-21*) + + (defun bigatom (Xs Ys) + (if + Xs (concat (f Xs) (bigatom (r Xs) Ys)) + Ys (concat (f Ys) (bigatom (r Ys) ())) + () + ) + ) + + (bigatom (q . (3 4 5)) X) + )"} + .to_string(); + let compiled = do_basic_run(&vec!["run".to_string(), prog]) + .trim() + .to_string(); + let res = do_basic_brun(&vec![ + "brun".to_string(), + compiled, + "(13 99 144)".to_string(), + ]) + .trim() + .to_string(); + assert_eq!(res.to_string(), "0x0304050d630090"); +} diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index 9d8947227..d6ede7f50 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -568,7 +568,9 @@ fn test_preprocessor_tours_includes_properly() { let mut includes = Vec::new(); let res = preprocess(opts, &mut includes, parsed[0].clone()).expect("should preprocess"); let expected_lines = &[ - "(defmac if (A B C) (qq (a (i (unquote A) (com (unquote B)) (com (unquote C))) @)))", + "(defmac __chia__primitive__if (A B C) (qq (a (i (unquote A) (com (unquote B)) (com (unquote C))) @)))", + "(defun __chia__if (ARGS) (a (i (r (r (r ARGS))) (com (qq (i (unquote (f ARGS)) (com (unquote (f (r ARGS)))) (unquote (__chia__if (r (r ARGS))))))) (com (qq (i (unquote (f ARGS)) (com (unquote (f (r ARGS)))) (com (unquote (f (r (r ARGS))))))))) @))", + "(defmac if ARGS (qq (a (unquote (__chia__if ARGS)) @)))", "(defun __chia__compile-list (args) (a (i args (com (c 4 (c (f args) (c (__chia__compile-list (r args)) ())))) (com ())) @))", "(defmac list ARGS (__chia__compile-list ARGS))", "(defun-inline / (A B) (f (divmod A B)))", From aeecc7506618d45346e49fc6f2b303cddf46e063 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 13 Sep 2023 14:20:58 -0700 Subject: [PATCH 63/81] fmt + clippy --- src/compiler/frontend.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index 040dbb74e..346f2fe88 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -253,12 +253,11 @@ fn make_let_bindings( body.loc(), "Bad binding tail ".to_string() + &body.to_string(), )); - let do_atomize = - if opts.dialect().strict { - |a: &SExp| -> SExp { a.atomize() } - } else { - |a: &SExp| -> SExp { a.clone() } - }; + let do_atomize = if opts.dialect().strict { + |a: &SExp| -> SExp { a.atomize() } + } else { + |a: &SExp| -> SExp { a.clone() } + }; match body.borrow() { SExp::Nil(_) => Ok(vec![]), SExp::Cons(_, head, tl) => head From df48f917d33f736cb3113bd6fa087fba893f4d48 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 26 Sep 2023 10:20:42 -0700 Subject: [PATCH 64/81] Remove spam --- src/compiler/rename.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index 09c3e85d5..7e997bae4 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -215,7 +215,6 @@ fn rename_in_bodyform( ..*letdata.clone() }), ); - eprintln!("rename {}", res.to_sexp()); Ok(res) } From 67abadd746097ad62a4ba1472322bd46a23d6c6d Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 26 Sep 2023 12:59:02 -0700 Subject: [PATCH 65/81] fmt + clippy --- src/compiler/evaluate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 77f2ed9d9..195d28b47 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -9,7 +9,7 @@ use clvm_rs::allocator::Allocator; use crate::classic::clvm::__type_compatibility__::{bi_one, bi_zero}; use crate::classic::clvm_tools::stages::stage_0::TRunProgram; -use crate::compiler::clvm::{run, PrimOverride, truthy}; +use crate::compiler::clvm::{run, truthy, PrimOverride}; use crate::compiler::codegen::{codegen, hoist_assign_form}; use crate::compiler::compiler::is_at_capture; use crate::compiler::comptypes::{ From 609c0a086d1fd6ba99f65dbaff65cbf9edb6f5b6 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 27 Sep 2023 12:01:57 -0700 Subject: [PATCH 66/81] fmt --- src/compiler/compiler.rs | 12 +--- src/compiler/frontend.rs | 5 +- src/compiler/preprocessor/macros.rs | 99 ++++++++--------------------- src/compiler/preprocessor/mod.rs | 6 +- 4 files changed, 33 insertions(+), 89 deletions(-) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 7f458d078..e831fcfca 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -175,9 +175,7 @@ pub fn compile_from_compileform( p0: CompileForm, symbol_table: &mut HashMap, ) -> Result { - let mut wrapper = CompileContextWrapper::new( - allocator, runner, symbol_table - ); + let mut wrapper = CompileContextWrapper::new(allocator, runner, symbol_table); let p1 = if opts.frontend_opt() { // Front end optimization fe_opt(&mut wrapper.context, opts.clone(), p0)? @@ -217,13 +215,7 @@ pub fn compile_pre_forms( // Resolve includes, convert program source to lexemes let p0 = frontend(opts.clone(), pre_forms)?; - compile_from_compileform( - allocator, - runner, - opts, - p0, - symbol_table - ) + compile_from_compileform(allocator, runner, opts, p0, symbol_table) } pub fn compile_file( diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index bcd91247e..16ba840bd 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -255,10 +255,7 @@ fn make_let_bindings( opts: Rc, body: Rc, ) -> Result>, CompileErr> { - let err = Err(CompileErr( - body.loc(), - format!("Bad binding tail {body:?}") - )); + let err = Err(CompileErr(body.loc(), format!("Bad binding tail {body:?}"))); let do_atomize = if opts.dialect().strict { |a: &SExp| -> SExp { a.atomize() } } else { diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 859d8c79b..e485181de 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -19,7 +19,7 @@ fn match_quoted_string(body: Rc) -> Result)>, Comp let is_string = match body.borrow() { SExp::QuotedString(_, b'x', _) => None, SExp::QuotedString(al, _, an) => Some((al.clone(), an.clone())), - _ => None + _ => None, }; if let Some((loc, s)) = is_string { @@ -60,7 +60,7 @@ fn match_number(body: Rc) -> Result, CompileErr> { SExp::Nil(il) => { return Ok(Some(MatchedNumber::MatchedInt(il.clone(), bi_zero()))); } - _ => { } + _ => {} } Err(CompileErr(body.loc(), "number required".to_string())) @@ -94,11 +94,7 @@ pub trait ExtensionFunction { } fn required_args(&self) -> Option; #[allow(clippy::too_many_arguments)] - fn try_eval( - &self, - loc: &Srcloc, - args: &[Rc], - ) -> Result, CompileErr>; + fn try_eval(&self, loc: &Srcloc, args: &[Rc]) -> Result, CompileErr>; } struct StringQ {} @@ -114,11 +110,7 @@ impl ExtensionFunction for StringQ { Some(1) } - fn try_eval( - &self, - loc: &Srcloc, - args: &[Rc], - ) -> Result, CompileErr> { + fn try_eval(&self, loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { let res = match match_quoted_string(args[0].clone()) { Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), _ => SExp::Nil(loc.clone()), @@ -141,11 +133,7 @@ impl ExtensionFunction for NumberQ { Some(1) } - fn try_eval( - &self, - loc: &Srcloc, - args: &[Rc], - ) -> Result, CompileErr> { + fn try_eval(&self, loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { let res = match match_number(args[0].clone()) { Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), _ => SExp::Nil(loc.clone()), @@ -168,11 +156,7 @@ impl ExtensionFunction for SymbolQ { Some(1) } - fn try_eval( - &self, - loc: &Srcloc, - args: &[Rc], - ) -> Result, CompileErr> { + fn try_eval(&self, loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { let res = match match_atom(args[0].clone()) { Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), _ => SExp::Nil(loc.clone()), @@ -195,15 +179,9 @@ impl ExtensionFunction for SymbolToString { Some(1) } - fn try_eval( - &self, - _loc: &Srcloc, - args: &[Rc], - ) -> Result, CompileErr> { + fn try_eval(&self, _loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { if let Some((loc, value)) = match_atom(args[0].clone())? { - Ok(Rc::new(SExp::QuotedString( - loc, b'\"', value, - ))) + Ok(Rc::new(SExp::QuotedString(loc, b'\"', value))) } else { Err(CompileErr(args[0].loc(), "Not a symbol".to_string())) } @@ -223,11 +201,7 @@ impl ExtensionFunction for StringToSymbol { Some(1) } - fn try_eval( - &self, - _loc: &Srcloc, - args: &[Rc], - ) -> Result, CompileErr> { + fn try_eval(&self, _loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { Ok(Rc::new(SExp::Atom(loc, value))) } else { @@ -249,11 +223,7 @@ impl ExtensionFunction for StringAppend { None } - fn try_eval( - &self, - loc: &Srcloc, - args: &[Rc], - ) -> Result, CompileErr> { + fn try_eval(&self, loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { let mut out_vec = Vec::new(); let mut out_loc = None; for a in args.iter() { @@ -287,11 +257,7 @@ impl ExtensionFunction for NumberToString { Some(1) } - fn try_eval( - &self, - _loc: &Srcloc, - args: &[Rc], - ) -> Result, CompileErr> { + fn try_eval(&self, _loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { let match_res = match_number(args[0].clone())?; let (use_loc, int_val) = match &match_res { Some(MatchedNumber::MatchedInt(l, i)) => (l.clone(), i.clone()), @@ -321,11 +287,7 @@ impl ExtensionFunction for StringToNumber { Some(1) } - fn try_eval( - &self, - loc: &Srcloc, - args: &[Rc], - ) -> Result, CompileErr> { + fn try_eval(&self, loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { if let Ok(cvt_bi) = decode_string(&value).parse::() { Ok(Rc::new(SExp::Integer(loc, cvt_bi))) @@ -354,18 +316,17 @@ impl ExtensionFunction for StringLength { Some(1) } - fn try_eval( - &self, - _loc: &Srcloc, - args: &[Rc], - ) -> Result, CompileErr> { + fn try_eval(&self, _loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { if let Some((loc, value)) = match_quoted_string(args[0].clone())? { if let Some(len_bi) = value.len().to_bigint() { return Ok(Rc::new(SExp::Integer(loc, len_bi))); } } - Err(CompileErr(args[0].loc(), "Error getting string length".to_string())) + Err(CompileErr( + args[0].loc(), + "Error getting string length".to_string(), + )) } } @@ -382,11 +343,7 @@ impl ExtensionFunction for Substring { Some(3) } - fn try_eval( - &self, - _loc: &Srcloc, - args: &[Rc], - ) -> Result, CompileErr> { + fn try_eval(&self, _loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { let start_element = usize_value(args[1].clone())?; let end_element = usize_value(args[2].clone())?; @@ -404,11 +361,7 @@ impl ExtensionFunction for Substring { .skip(start_element) .copied() .collect(); - Ok(Rc::new(SExp::QuotedString( - l.clone(), - *ch, - result_value, - ))) + Ok(Rc::new(SExp::QuotedString(l.clone(), *ch, result_value))) } _ => Err(CompileErr(args[0].loc(), "Not a string".to_string())), } @@ -468,15 +421,15 @@ impl PrimOverride for PreprocessorExtension { ) -> Result>, RunFailure> { eprintln!("running {head} {tail}"); if let SExp::Atom(hl, head_atom) = head.borrow() { - let have_args: Vec> = - if let Some(args_list) = tail.proper_list() { - args_list.into_iter().map(Rc::new).collect() - } else { - return Ok(None); - }; + let have_args: Vec> = if let Some(args_list) = tail.proper_list() { + args_list.into_iter().map(Rc::new).collect() + } else { + return Ok(None); + }; if let Some(extension) = self.extfuns.get(head_atom) { - let res = extension.try_eval(&hl, &have_args) + let res = extension + .try_eval(&hl, &have_args) .map_err(compile_to_run_err)?; eprintln!("res = {res}"); diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index bc94064a6..2474f5492 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -72,7 +72,7 @@ fn make_defmac_name(name: &[u8]) -> Vec { } fn nilize(v: Rc) -> Rc { - if let SExp::Cons(l,a,b) = v.borrow() { + if let SExp::Cons(l, a, b) = v.borrow() { let a_conv = nilize(a.clone()); let b_conv = nilize(b.clone()); if Rc::as_ptr(&a_conv) == Rc::as_ptr(&a) && Rc::as_ptr(&b_conv) == Rc::as_ptr(&b) { @@ -321,7 +321,9 @@ impl Preprocessor { args.clone(), Some(extension), None, - ).map(nilize).map_err(|e| CompileErr::from(e))?; + ) + .map(nilize) + .map_err(|e| CompileErr::from(e))?; eprintln!("macro {} {args} => {res}", decode_string(&name)); return Ok(Some(res)); From e8aee718b324a25f42fc58a67f0bcce23ff93ed6 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 27 Sep 2023 12:06:06 -0700 Subject: [PATCH 67/81] clippy --- src/compiler/preprocessor/macros.rs | 6 +++--- src/compiler/preprocessor/mod.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index e485181de..4bf31b064 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -52,8 +52,8 @@ fn match_number(body: Rc) -> Result, CompileErr> { } SExp::Atom(al, b) => { // An atom with unprintable characters is rendered as an integer. - if !printable(&b) { - let to_integer = number_from_u8(&b); + if !printable(b) { + let to_integer = number_from_u8(b); return Ok(Some(MatchedNumber::MatchedInt(al.clone(), to_integer))); } } @@ -429,7 +429,7 @@ impl PrimOverride for PreprocessorExtension { if let Some(extension) = self.extfuns.get(head_atom) { let res = extension - .try_eval(&hl, &have_args) + .try_eval(hl, &have_args) .map_err(compile_to_run_err)?; eprintln!("res = {res}"); diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 2474f5492..558165fae 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -75,7 +75,7 @@ fn nilize(v: Rc) -> Rc { if let SExp::Cons(l, a, b) = v.borrow() { let a_conv = nilize(a.clone()); let b_conv = nilize(b.clone()); - if Rc::as_ptr(&a_conv) == Rc::as_ptr(&a) && Rc::as_ptr(&b_conv) == Rc::as_ptr(&b) { + if Rc::as_ptr(&a_conv) == Rc::as_ptr(a) && Rc::as_ptr(&b_conv) == Rc::as_ptr(b) { v.clone() } else { Rc::new(SExp::Cons(l.clone(), a_conv, b_conv)) @@ -323,7 +323,7 @@ impl Preprocessor { None, ) .map(nilize) - .map_err(|e| CompileErr::from(e))?; + .map_err(CompileErr::from)?; eprintln!("macro {} {args} => {res}", decode_string(&name)); return Ok(Some(res)); From 694c516dfb208a9d24ebb2ab2ea4c586741b4723 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 17 Oct 2023 21:48:53 -0700 Subject: [PATCH 68/81] Pull in some changes from staging --- src/compiler/preprocessor/macros.rs | 2 - src/compiler/preprocessor/mod.rs | 116 +++++++++++++++++----------- src/tests/classic/run.rs | 14 ++++ 3 files changed, 83 insertions(+), 49 deletions(-) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 4bf31b064..2267018e4 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -419,7 +419,6 @@ impl PrimOverride for PreprocessorExtension { _context: Rc, tail: Rc, ) -> Result>, RunFailure> { - eprintln!("running {head} {tail}"); if let SExp::Atom(hl, head_atom) = head.borrow() { let have_args: Vec> = if let Some(args_list) = tail.proper_list() { args_list.into_iter().map(Rc::new).collect() @@ -432,7 +431,6 @@ impl PrimOverride for PreprocessorExtension { .try_eval(hl, &have_args) .map_err(compile_to_run_err)?; - eprintln!("res = {res}"); return Ok(Some(res)); } } diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 558165fae..cbe95d9ef 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -6,6 +6,7 @@ use std::rc::Rc; use clvmr::allocator::Allocator; +use crate::classic::clvm_tools::binutils::assemble; use crate::classic::clvm_tools::clvmc::compile_clvm_text_maybe_opt; use crate::classic::clvm_tools::stages::stage_0::{DefaultProgramRunner, TRunProgram}; @@ -16,9 +17,9 @@ use crate::compiler::compiler::compile_from_compileform; use crate::compiler::comptypes::{ BodyForm, CompileErr, CompileForm, CompilerOpts, HelperForm, IncludeDesc, IncludeProcessType, }; -use crate::compiler::dialect::KNOWN_DIALECTS; +use crate::compiler::dialect::{detect_modern, KNOWN_DIALECTS}; use crate::compiler::evaluate::{create_argument_captures, ArgInputs}; -use crate::compiler::frontend::compile_helperform; +use crate::compiler::frontend::{compile_helperform, frontend}; use crate::compiler::preprocessor::macros::PreprocessorExtension; use crate::compiler::rename::rename_args_helperform; use crate::compiler::runtypes::RunFailure; @@ -45,9 +46,11 @@ enum IncludeType { struct Preprocessor { opts: Rc, + ppext: Rc, runner: Rc, helpers: Vec, strict: bool, + stored_macros: HashMap, Rc>, } fn compose_defconst(loc: Srcloc, name: &[u8], sexp: Rc) -> Rc { @@ -90,17 +93,25 @@ fn nilize(v: Rc) -> Rc { impl Preprocessor { pub fn new(opts: Rc) -> Self { let runner = Rc::new(DefaultProgramRunner::new()); + let ppext = Rc::new(PreprocessorExtension::new()); + let opts_prims = ppext.enrich_prims(opts.clone()); Preprocessor { - opts: opts.clone(), + opts: opts_prims, + ppext, runner, helpers: Vec::new(), strict: opts.dialect().strict, + stored_macros: HashMap::default(), } } /// Given a specification of an include file, load up the forms inside it and /// return them (or an error if the file couldn't be read or wasn't a list). - pub fn process_include(&mut self, include: &IncludeDesc) -> Result>, CompileErr> { + pub fn process_include( + &mut self, + includes: &mut Vec, + include: &IncludeDesc, + ) -> Result>, CompileErr> { let filename_and_content = self .opts .read_new_file(self.opts.filename(), decode_string(&include.name))?; @@ -122,9 +133,8 @@ impl Preprocessor { if self.strict { let mut result = Vec::new(); for p in parsed.into_iter() { - if let Some(res) = self.expand_macros(p.clone(), true)? { - result.push(res); - } + let mut new_forms = self.process_pp_form(includes, p.clone())?; + result.append(&mut new_forms); } Ok(result) @@ -281,9 +291,6 @@ impl Preprocessor { continue; } - // as inline defuns because they're closest to that - // semantically. - let mut allocator = Allocator::new(); // The name matched, try calling it. // Form argument env. @@ -295,37 +302,46 @@ impl Preprocessor { mdata.args.clone(), )?; - let ppext = Rc::new(PreprocessorExtension::new()); - let extension: &PreprocessorExtension = ppext.borrow(); - let opts_prims = extension.enrich_prims(self.opts.clone()); - let new_program = CompileForm { - loc: body.loc(), - args: mdata.args.clone(), - include_forms: vec![], - helpers: self.helpers.clone(), - exp: mdata.body.clone(), - }; - let mut symbol_table = HashMap::new(); - let compiled_program = compile_from_compileform( - &mut allocator, - self.runner.clone(), - opts_prims.clone(), - new_program, - &mut symbol_table, - )?; + let mut allocator = Allocator::new(); + let compiled_program = + if let Some(compiled_program) = self.stored_macros.get(&mdata.name) { + compiled_program.clone() + } else { + // as inline defuns because they're closest to that + // semantically. + let mut symbol_table = HashMap::new(); + let new_program = CompileForm { + loc: body.loc(), + args: mdata.args.clone(), + include_forms: vec![], + helpers: self.helpers.clone(), + exp: mdata.body.clone(), + }; + let compiled_program = compile_from_compileform( + &mut allocator, + self.runner.clone(), + self.opts.clone(), + new_program, + &mut symbol_table, + )?; + self.stored_macros + .insert(mdata.name.clone(), Rc::new(compiled_program.clone())); + Rc::new(compiled_program) + }; + + let ppext: &PreprocessorExtension = self.ppext.borrow(); let res = clvm::run( &mut allocator, self.runner.clone(), - opts_prims.prim_map(), - Rc::new(compiled_program), + self.opts.prim_map(), + compiled_program, args.clone(), - Some(extension), + Some(ppext), None, ) .map(nilize) .map_err(CompileErr::from)?; - eprintln!("macro {} {args} => {res}", decode_string(&name)); return Ok(Some(res)); } } @@ -515,7 +531,7 @@ impl Preprocessor { Ok(vec![]) } else if let Some(IncludeType::Basic(i)) = &included { self.recurse_dependencies(includes, IncludeProcessType::Compiled, i.clone())?; - self.process_include(i) + self.process_include(includes, i) } else if let Some(IncludeType::Processed(f, kind, name)) = &included { self.recurse_dependencies(includes, kind.clone(), f.clone())?; self.process_embed(body.loc(), &decode_string(&f.name), kind, name) @@ -588,24 +604,30 @@ pub fn preprocess( /// form that causes compilation to include another file. The file names are path /// expanded based on the include path they were found in (from opts). pub fn gather_dependencies( - opts: Rc, + mut opts: Rc, real_input_path: &str, file_content: &str, ) -> Result, CompileErr> { - let mut includes = Vec::new(); - let no_stdenv_opts = opts.set_stdenv(false); - let mut p = Preprocessor::new(no_stdenv_opts); - let loc = Srcloc::start(real_input_path); - - let parsed = parse_sexp(loc, file_content.bytes())?; - - if parsed.is_empty() { - return Ok(vec![]); + let mut allocator = Allocator::new(); + + let assembled_input = assemble(&mut allocator, &file_content).map_err(|e| { + CompileErr(Srcloc::start(real_input_path), e.1) + })?; + let dialect = detect_modern(&mut allocator, assembled_input); + opts = opts.set_stdenv(dialect.strict).set_dialect(dialect.clone()); + if let Some(stepping) = dialect.stepping { + opts = opts + .set_optimize(stepping > 22) + .set_frontend_opt(stepping > 21); } - for elt in parsed.iter() { - p.run(&mut includes, elt.clone())?; - } + let parsed = parse_sexp(Srcloc::start(real_input_path), file_content.bytes())?; + let program = frontend(opts, &parsed)?; - Ok(includes) + let filtered_results: Vec = program + .include_forms + .into_iter() + .filter(|f| !f.name.starts_with(b"*")) + .collect(); + Ok(filtered_results) } diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index 94b93010a..318a59912 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -1561,3 +1561,17 @@ fn test_continued_if() { .to_string(); assert_eq!(res.to_string(), "0x0304050d630090"); } + +#[test] +fn test_preprocess_can_recurse() { + let prog = "resources/tests/strict/test-inner-include.clsp".to_string(); + let res = do_basic_run(&vec![ + "run".to_string(), + "-i".to_string(), + "resources/tests/strict".to_string(), + prog.clone(), + ]) + .trim() + .to_string(); + assert_eq!(res, "(2 (1 2 (3 5 (1 2 (1 18 5 (1 . 2)) 1) (1 2 (1 16 5 (1 . 1)) 1)) 1) (4 (1) 1))"); +} From 8988aeeed8257d8d9a8fd1feecc3b20fcf871e6b Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 17 Oct 2023 21:49:27 -0700 Subject: [PATCH 69/81] Add test files --- resources/tests/strict/test-inner-include.clinc | 3 +++ resources/tests/strict/test-inner-include.clsp | 7 +++++++ 2 files changed, 10 insertions(+) create mode 100644 resources/tests/strict/test-inner-include.clinc create mode 100644 resources/tests/strict/test-inner-include.clsp diff --git a/resources/tests/strict/test-inner-include.clinc b/resources/tests/strict/test-inner-include.clinc new file mode 100644 index 000000000..704dfcca7 --- /dev/null +++ b/resources/tests/strict/test-inner-include.clinc @@ -0,0 +1,3 @@ +( + (include defmac_simple_if.clib) +) diff --git a/resources/tests/strict/test-inner-include.clsp b/resources/tests/strict/test-inner-include.clsp new file mode 100644 index 000000000..6fc0dabf8 --- /dev/null +++ b/resources/tests/strict/test-inner-include.clsp @@ -0,0 +1,7 @@ +(mod (X) + (include *strict-cl-21*) + + (include test-inner-include.clinc) + + (if_ X (* X 2) (+ X 1)) + ) From 555e266b6bac8bf25614be708469368e63f54950 Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 17 Oct 2023 21:49:49 -0700 Subject: [PATCH 70/81] fmt --- src/compiler/preprocessor/mod.rs | 5 ++--- src/tests/classic/run.rs | 5 ++++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index cbe95d9ef..8c3978f9e 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -610,9 +610,8 @@ pub fn gather_dependencies( ) -> Result, CompileErr> { let mut allocator = Allocator::new(); - let assembled_input = assemble(&mut allocator, &file_content).map_err(|e| { - CompileErr(Srcloc::start(real_input_path), e.1) - })?; + let assembled_input = assemble(&mut allocator, &file_content) + .map_err(|e| CompileErr(Srcloc::start(real_input_path), e.1))?; let dialect = detect_modern(&mut allocator, assembled_input); opts = opts.set_stdenv(dialect.strict).set_dialect(dialect.clone()); if let Some(stepping) = dialect.stepping { diff --git a/src/tests/classic/run.rs b/src/tests/classic/run.rs index 318a59912..8b48f99a8 100644 --- a/src/tests/classic/run.rs +++ b/src/tests/classic/run.rs @@ -1573,5 +1573,8 @@ fn test_preprocess_can_recurse() { ]) .trim() .to_string(); - assert_eq!(res, "(2 (1 2 (3 5 (1 2 (1 18 5 (1 . 2)) 1) (1 2 (1 16 5 (1 . 1)) 1)) 1) (4 (1) 1))"); + assert_eq!( + res, + "(2 (1 2 (3 5 (1 2 (1 18 5 (1 . 2)) 1) (1 2 (1 16 5 (1 . 1)) 1)) 1) (4 (1) 1))" + ); } From 2a69fd4e435568b7a08712bb1f7e7c50c4f5b46b Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 17 Oct 2023 21:50:38 -0700 Subject: [PATCH 71/81] clippy --- src/compiler/preprocessor/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 8c3978f9e..605dd9b08 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -610,7 +610,7 @@ pub fn gather_dependencies( ) -> Result, CompileErr> { let mut allocator = Allocator::new(); - let assembled_input = assemble(&mut allocator, &file_content) + let assembled_input = assemble(&mut allocator, file_content) .map_err(|e| CompileErr(Srcloc::start(real_input_path), e.1))?; let dialect = detect_modern(&mut allocator, assembled_input); opts = opts.set_stdenv(dialect.strict).set_dialect(dialect.clone()); From d17d2a35799b2f300fe0f8b2f6686ed772a375b5 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 18 Oct 2023 10:03:43 -0700 Subject: [PATCH 72/81] fmt --- src/compiler/compiler.rs | 4 +- src/compiler/evaluate.rs | 1 - src/compiler/frontend.rs | 4 +- src/compiler/preprocessor/mod.rs | 67 +++++++++++++++----------------- src/tests/compiler/compiler.rs | 11 ++++-- 5 files changed, 42 insertions(+), 45 deletions(-) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 50f286536..6f2f26330 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -12,9 +12,7 @@ use crate::classic::clvm_tools::stages::stage_0::TRunProgram; use crate::compiler::clvm::sha256tree; use crate::compiler::codegen::{codegen, hoist_body_let_binding, process_helper_let_bindings}; -use crate::compiler::comptypes::{ - CompileErr, CompileForm, CompilerOpts, PrimaryCodegen, -}; +use crate::compiler::comptypes::{CompileErr, CompileForm, CompilerOpts, PrimaryCodegen}; use crate::compiler::dialect::{AcceptedDialect, KNOWN_DIALECTS}; use crate::compiler::frontend::frontend; use crate::compiler::optimize::get_optimizer; diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index 7f4aa21b4..fab04f1e1 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -80,7 +80,6 @@ pub enum ArgInputs { Pair(Rc, Rc), } - /// Evaluator is an object that simplifies expressions, given the helpers /// (helpers are forms that are reusable parts of programs, such as defconst, /// defun or defmacro) from a program. In the simplest form, it can be used to diff --git a/src/compiler/frontend.rs b/src/compiler/frontend.rs index 7c9dc2fd0..a2c1727bd 100644 --- a/src/compiler/frontend.rs +++ b/src/compiler/frontend.rs @@ -280,9 +280,7 @@ fn make_let_bindings( result.append(&mut rest_bindings); Ok(result) } - _ => { - err.clone() - } + _ => err.clone(), }) .unwrap_or_else(|| err.clone()), _ => err, diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 3e0f6b649..862bb4d44 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -302,39 +302,39 @@ impl Preprocessor { )?; let mut allocator = Allocator::new(); - let compiled_program = - if let Some(compiled_program) = self.stored_macros.get(&mdata.name) { - compiled_program.clone() - } else { - // as inline defuns because they're closest to that - // semantically. - let mut symbol_table = HashMap::new(); - let new_program = CompileForm { - loc: body.loc(), - args: mdata.args.clone(), - include_forms: vec![], - helpers: self.helpers.clone(), - exp: mdata.body.clone(), - }; - - let program_sexp = - Rc::new(SExp::Cons( - body.loc(), - Rc::new(SExp::Atom(body.loc(), b"mod".to_vec())), - new_program.to_sexp() - )); - - let compiled_program = self.opts.set_stdenv(false).compile_program( - &mut allocator, - self.runner.clone(), - program_sexp, - &mut symbol_table, - )?; - self.stored_macros - .insert(mdata.name.clone(), Rc::new(compiled_program.clone())); - Rc::new(compiled_program) + let compiled_program = if let Some(compiled_program) = + self.stored_macros.get(&mdata.name) + { + compiled_program.clone() + } else { + // as inline defuns because they're closest to that + // semantically. + let mut symbol_table = HashMap::new(); + let new_program = CompileForm { + loc: body.loc(), + args: mdata.args.clone(), + include_forms: vec![], + helpers: self.helpers.clone(), + exp: mdata.body.clone(), }; + let program_sexp = Rc::new(SExp::Cons( + body.loc(), + Rc::new(SExp::Atom(body.loc(), b"mod".to_vec())), + new_program.to_sexp(), + )); + + let compiled_program = self.opts.set_stdenv(false).compile_program( + &mut allocator, + self.runner.clone(), + program_sexp, + &mut symbol_table, + )?; + self.stored_macros + .insert(mdata.name.clone(), Rc::new(compiled_program.clone())); + Rc::new(compiled_program) + }; + let ppext: &PreprocessorExtension = self.ppext.borrow(); let res = clvm::run( &mut allocator, @@ -348,10 +348,7 @@ impl Preprocessor { .map(nilize) .map_err(CompileErr::from)?; - if let Some(final_result) = self.expand_macros( - res.clone(), - true, - )? { + if let Some(final_result) = self.expand_macros(res.clone(), true)? { return Ok(Some(final_result)); } else { return Ok(Some(res)); diff --git a/src/tests/compiler/compiler.rs b/src/tests/compiler/compiler.rs index e6ca97428..b43211a8d 100644 --- a/src/tests/compiler/compiler.rs +++ b/src/tests/compiler/compiler.rs @@ -6,8 +6,8 @@ use clvm_rs::allocator::Allocator; use crate::classic::clvm_tools::stages::stage_0::DefaultProgramRunner; use crate::compiler::clvm::run; use crate::compiler::compiler::{compile_file, DefaultCompilerOpts}; -use crate::compiler::dialect::AcceptedDialect; use crate::compiler::comptypes::{CompileErr, CompilerOpts}; +use crate::compiler::dialect::AcceptedDialect; use crate::compiler::frontend::{collect_used_names_sexp, frontend}; use crate::compiler::rename::rename_in_cons; use crate::compiler::runtypes::RunFailure; @@ -264,8 +264,13 @@ fn run_test_4_opt() { } fn run_test_5_maybe_opt(opt: bool) { - let result = - run_string_maybe_opt(&"(mod (a) (list 1 2))".to_string(), &"()".to_string(), opt, false).unwrap(); + let result = run_string_maybe_opt( + &"(mod (a) (list 1 2))".to_string(), + &"()".to_string(), + opt, + false, + ) + .unwrap(); assert_eq!(result.to_string(), "(1 2)".to_string()); } From 78ec5a5e1b2d87cde50083158fd10080ee70c279 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 18 Oct 2023 10:24:25 -0700 Subject: [PATCH 73/81] Back out some stuff that's from the future --- src/compiler/comptypes.rs | 6 ----- src/compiler/preprocessor/mod.rs | 44 +++----------------------------- 2 files changed, 3 insertions(+), 47 deletions(-) diff --git a/src/compiler/comptypes.rs b/src/compiler/comptypes.rs index 45014cbb2..8f5b2bb62 100644 --- a/src/compiler/comptypes.rs +++ b/src/compiler/comptypes.rs @@ -305,12 +305,6 @@ pub enum IncludeProcessType { Hex, /// Read clvm in s-expression form as a clvm value. SExpression, - /// Pointing to a chialisp program, Compiled specifies that we want - /// the compiled form of the program as a clvm value. It's possible - /// because chialisp programs self-identify how they're compiled and - /// the form they take is promised to be stable for released versions - /// of the language. - Compiled, } /// A description of an include form. Here, records the locations of the various diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index 862bb4d44..cc4f0b253 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -7,12 +7,11 @@ use std::rc::Rc; use clvmr::allocator::Allocator; use crate::classic::clvm_tools::binutils::assemble; -use crate::classic::clvm_tools::clvmc::compile_clvm_text_maybe_opt; use crate::classic::clvm_tools::stages::stage_0::{DefaultProgramRunner, TRunProgram}; use crate::compiler::cldb::hex_to_modern_sexp; use crate::compiler::clvm; -use crate::compiler::clvm::{convert_from_clvm_rs, truthy}; +use crate::compiler::clvm::truthy; use crate::compiler::comptypes::{ BodyForm, CompileErr, CompileForm, CompilerOpts, HelperForm, IncludeDesc, IncludeProcessType, }; @@ -182,23 +181,6 @@ impl Preprocessor { parsed[0].clone() } - IncludeProcessType::Compiled => { - let decoded_content = decode_string(&content); - let mut symtab = HashMap::new(); - let newly_compiled = compile_clvm_text_maybe_opt( - &mut allocator, - self.opts.optimize(), - self.opts.clone(), - &mut symtab, - &decoded_content, - &full_name, - true, - ) - .map_err(|e| CompileErr(loc.clone(), format!("Subcompile failed: {}", e.1)))?; - - convert_from_clvm_rs(&mut allocator, loc.clone(), newly_compiled) - .map_err(run_to_compile_err)? - } }; Ok(vec![compose_defconst(loc, constant_name, content)]) @@ -208,7 +190,6 @@ impl Preprocessor { fn recurse_dependencies( &mut self, includes: &mut Vec, - kind: IncludeProcessType, desc: IncludeDesc, ) -> Result<(), CompileErr> { let name_string = decode_string(&desc.name); @@ -222,10 +203,6 @@ impl Preprocessor { ..desc }); - if !matches!(kind, IncludeProcessType::Compiled) { - return Ok(()); - } - let parsed = parse_sexp(Srcloc::start(&full_name), content.iter().copied()) .map_err(|e| CompileErr(e.0, e.1))?; if parsed.is_empty() { @@ -452,21 +429,6 @@ impl Preprocessor { } } - [SExp::Atom(kl, compile_file), SExp::Atom(_, name), SExp::Atom(nl, fname)] => { - if compile_file == b"compile-file" { - return Ok(Some(IncludeType::Processed( - IncludeDesc { - kw: kl.clone(), - nl: nl.clone(), - kind: Some(IncludeProcessType::Compiled), - name: fname.clone(), - }, - IncludeProcessType::Compiled, - name.clone() - ))); - } - } - [SExp::Atom(kl, embed_file), SExp::Atom(_, name), SExp::Atom(_, kind), SExp::Atom(nl, fname)] => { if embed_file == b"embed-file" { if kind == b"hex" { @@ -540,10 +502,10 @@ impl Preprocessor { if let Some(()) = self.decode_macro(body.clone())? { Ok(vec![]) } else if let Some(IncludeType::Basic(i)) = &included { - self.recurse_dependencies(includes, IncludeProcessType::Compiled, i.clone())?; + self.recurse_dependencies(includes, i.clone())?; self.process_include(includes, i) } else if let Some(IncludeType::Processed(f, kind, name)) = &included { - self.recurse_dependencies(includes, kind.clone(), f.clone())?; + self.recurse_dependencies(includes, f.clone())?; self.process_embed(body.loc(), &decode_string(&f.name), kind, name) } else { Ok(vec![body]) From 53cbedceb860120bfc8cff913be677bd87b1e4c9 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 18 Oct 2023 10:27:58 -0700 Subject: [PATCH 74/81] Back out this change until it can be re-evaluated --- src/compiler/compiler.rs | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 6f2f26330..04703f69c 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -127,12 +127,15 @@ pub fn desugar_pre_forms( do_desugar(&p1) } -pub fn compile_from_compileform( +pub fn compile_pre_forms( context: &mut BasicCompileContext, opts: Rc, - p0: CompileForm, + pre_forms: &[Rc], ) -> Result { - let p3 = context.post_desugar_optimization(opts.clone(), p0)?; + // Resolve includes, convert program source to lexemes + let p2 = desugar_pre_forms(context, opts.clone(), pre_forms)?; + + let p3 = context.post_desugar_optimization(opts.clone(), p2)?; // generate code from AST, optionally with optimization let generated = codegen(context, opts.clone(), &p3)?; @@ -142,17 +145,6 @@ pub fn compile_from_compileform( Ok(g2) } -pub fn compile_pre_forms( - context: &mut BasicCompileContext, - opts: Rc, - pre_forms: &[Rc], -) -> Result { - // Resolve includes, convert program source to lexemes - let p0 = desugar_pre_forms(context, opts.clone(), pre_forms)?; - - compile_from_compileform(context, opts, p0) -} - pub fn compile_file( allocator: &mut Allocator, runner: Rc, From 89ddca5272809b33e0b41e971ed992f24af14caa Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 18 Oct 2023 10:30:42 -0700 Subject: [PATCH 75/81] Back out artifacts from original evaluate based implementation --- src/compiler/evaluate.rs | 55 +++++++++++++++------------------------- 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/src/compiler/evaluate.rs b/src/compiler/evaluate.rs index fab04f1e1..940b5fe92 100644 --- a/src/compiler/evaluate.rs +++ b/src/compiler/evaluate.rs @@ -19,7 +19,7 @@ use crate::compiler::comptypes::{ use crate::compiler::frontend::frontend; use crate::compiler::optimize::get_optimizer; use crate::compiler::runtypes::RunFailure; -use crate::compiler::sexp::{enlist, SExp}; +use crate::compiler::sexp::SExp; use crate::compiler::srcloc::Srcloc; use crate::compiler::stackvisit::{HasDepthLimit, VisitedMarker}; use crate::compiler::CompileContextWrapper; @@ -73,7 +73,6 @@ pub struct LambdaApply { // Frontend evaluator based on my fuzzer representation and direct interpreter of // that. - #[derive(Debug)] pub enum ArgInputs { Whole(Rc), @@ -347,7 +346,7 @@ fn decons_args(formed_tail: Rc) -> ArgInputs { } } -pub fn build_argument_captures( +fn build_argument_captures( l: &Srcloc, arguments_to_convert: &[Rc], tail: Option>, @@ -777,34 +776,6 @@ impl<'info> Evaluator { } } - fn defmac_ordering(&self) -> bool { - let dialect = self.opts.dialect(); - dialect.strict || dialect.stepping.unwrap_or(21) > 22 - } - - fn make_com_module(&self, l: &Srcloc, prog_args: Rc, body: Rc) -> Rc { - let end_of_list = if self.defmac_ordering() { - let mut mod_list: Vec> = self.helpers.iter().map(|h| h.to_sexp()).collect(); - mod_list.push(body); - Rc::new(enlist(l.clone(), &mod_list)) - } else { - let mut end_of_list = - Rc::new(SExp::Cons(l.clone(), body, Rc::new(SExp::Nil(l.clone())))); - - for h in self.helpers.iter() { - end_of_list = Rc::new(SExp::Cons(l.clone(), h.to_sexp(), end_of_list)); - } - - end_of_list - }; - - Rc::new(SExp::Cons( - l.clone(), - Rc::new(SExp::Atom(l.clone(), "mod".as_bytes().to_vec())), - Rc::new(SExp::Cons(l.clone(), prog_args, end_of_list)), - )) - } - fn is_lambda_apply( &self, allocator: &mut Allocator, @@ -915,9 +886,23 @@ impl<'info> Evaluator { prog_args, )))) } else if call.name == "com".as_bytes() { - let use_body = - self.make_com_module(&call.loc, prog_args, arguments_to_convert[0].to_sexp()); - let compiled = self.compile_code(allocator, false, use_body)?; + let mut end_of_list = Rc::new(SExp::Cons( + call.loc.clone(), + arguments_to_convert[0].to_sexp(), + Rc::new(SExp::Nil(call.loc.clone())), + )); + + for h in self.helpers.iter() { + end_of_list = Rc::new(SExp::Cons(call.loc.clone(), h.to_sexp(), end_of_list)) + } + + let use_body = SExp::Cons( + call.loc.clone(), + Rc::new(SExp::Atom(call.loc.clone(), "mod".as_bytes().to_vec())), + Rc::new(SExp::Cons(call.loc.clone(), prog_args, end_of_list)), + ); + + let compiled = self.compile_code(allocator, false, Rc::new(use_body))?; let compiled_borrowed: &SExp = compiled.borrow(); Ok(Rc::new(BodyForm::Quoted(compiled_borrowed.clone()))) } else { @@ -1657,7 +1642,7 @@ impl<'info> Evaluator { // primitive. let updated_opts = self .opts - .set_stdenv(!in_defun && !self.opts.dialect().strict) + .set_stdenv(!in_defun) .set_in_defun(in_defun) .set_frontend_opt(false); From 9181d184d935dd4d6efdabdecd98a805fd125401 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 18 Oct 2023 10:36:54 -0700 Subject: [PATCH 76/81] Back out further small changes that aren't needed --- src/classic/clvm_tools/clvmc.rs | 5 +++-- src/compiler/rename.rs | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/classic/clvm_tools/clvmc.rs b/src/classic/clvm_tools/clvmc.rs index 87f60e476..1ef267e4a 100644 --- a/src/classic/clvm_tools/clvmc.rs +++ b/src/classic/clvm_tools/clvmc.rs @@ -50,6 +50,7 @@ pub fn compile_clvm_text_maybe_opt( ) -> Result { let ir_src = read_ir(text).map_err(|s| EvalErr(allocator.null(), s.to_string()))?; let assembled_sexp = assemble_from_ir(allocator, Rc::new(ir_src))?; + let dialect = detect_modern(allocator, assembled_sexp); // Now the stepping is optional (None for classic) but we may communicate // other information in dialect as well. @@ -60,8 +61,8 @@ pub fn compile_clvm_text_maybe_opt( let runner = Rc::new(DefaultProgramRunner::new()); let opts = opts .set_dialect(dialect) - .set_optimize(do_optimize) - .set_frontend_opt(stepping > 21); + .set_optimize(do_optimize || stepping > 22) // Would apply to cl23 + .set_frontend_opt(stepping == 22); let unopt_res = compile_file(allocator, runner.clone(), opts.clone(), text, symbol_table); let res = unopt_res.and_then(|x| { diff --git a/src/compiler/rename.rs b/src/compiler/rename.rs index a43501b81..b88cbd638 100644 --- a/src/compiler/rename.rs +++ b/src/compiler/rename.rs @@ -207,15 +207,14 @@ fn rename_in_bodyform( &letdata.bindings, )?; let new_body = rename_in_bodyform(namemap, letdata.body.clone())?; - let res = BodyForm::Let( + Ok(BodyForm::Let( kind.clone(), Box::new(LetData { bindings: new_bindings, body: Rc::new(new_body), ..*letdata.clone() }), - ); - Ok(res) + )) } BodyForm::Quoted(atom) => match atom { From 482a5a971314762489d696b6f028555a2ff6c057 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 18 Oct 2023 14:19:11 -0700 Subject: [PATCH 77/81] Proper definitions of continued if --- src/compiler/compiler.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 04703f69c..bc5cde0cd 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -48,12 +48,12 @@ lazy_static! { (defun __chia__if (ARGS) (__chia__primitive__if (r (r (r ARGS))) - (qq (i (unquote (f ARGS)) (com (unquote (f (r ARGS)))) (unquote (__chia__if (r (r ARGS)))))) - (qq (i (unquote (f ARGS)) (com (unquote (f (r ARGS)))) (com (unquote (f (r (r ARGS))))))) + (qq (a (i (unquote (f ARGS)) (com (unquote (f (r ARGS)))) (com (unquote (__chia__if (r (r ARGS)))))) @)) + (qq (a (i (unquote (f ARGS)) (com (unquote (f (r ARGS)))) (com (unquote (f (r (r ARGS)))))) @)) ) ) - (defmac if ARGS (qq (a (unquote (__chia__if ARGS)) @))) + (defmac if ARGS (__chia__if ARGS)) (defun __chia__compile-list (args) (if args From c794430613d43100feac38757ac9d508f5592c97 Mon Sep 17 00:00:00 2001 From: arty Date: Wed, 18 Oct 2023 14:28:57 -0700 Subject: [PATCH 78/81] Test update --- src/tests/compiler/preprocessor.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tests/compiler/preprocessor.rs b/src/tests/compiler/preprocessor.rs index cbd0d8821..e2a40e515 100644 --- a/src/tests/compiler/preprocessor.rs +++ b/src/tests/compiler/preprocessor.rs @@ -570,8 +570,8 @@ fn test_preprocessor_tours_includes_properly() { let res = preprocess(opts, &mut includes, parsed[0].clone()).expect("should preprocess"); let expected_lines = &[ "(defmac __chia__primitive__if (A B C) (qq (a (i (unquote A) (com (unquote B)) (com (unquote C))) @)))", - "(defun __chia__if (ARGS) (a (i (r (r (r ARGS))) (com (qq (i (unquote (f ARGS)) (com (unquote (f (r ARGS)))) (unquote (__chia__if (r (r ARGS))))))) (com (qq (i (unquote (f ARGS)) (com (unquote (f (r ARGS)))) (com (unquote (f (r (r ARGS))))))))) @))", - "(defmac if ARGS (qq (a (unquote (__chia__if ARGS)) @)))", + "(defun __chia__if (ARGS) (a (i (r (r (r ARGS))) (com (qq (a (i (unquote (f ARGS)) (com (unquote (f (r ARGS)))) (com (unquote (__chia__if (r (r ARGS)))))) @))) (com (qq (a (i (unquote (f ARGS)) (com (unquote (f (r ARGS)))) (com (unquote (f (r (r ARGS)))))) @)))) @))", + "(defmac if ARGS (__chia__if ARGS))", "(defun __chia__compile-list (args) (a (i args (com (c 4 (c (f args) (c (__chia__compile-list (r args)) ())))) (com ())) @))", "(defmac list ARGS (__chia__compile-list ARGS))", "(defun-inline / (A B) (f (divmod A B)))", From e996ec957dc0951e199cc551117499db63e5290e Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 19 Oct 2023 08:51:28 -0700 Subject: [PATCH 79/81] Simplify value kind testing functions, thanks to amine. --- src/compiler/preprocessor/macros.rs | 107 +++++++++++----------------- 1 file changed, 41 insertions(+), 66 deletions(-) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 2267018e4..f5ccb9f31 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -15,26 +15,22 @@ use crate::compiler::srcloc::Srcloc; use crate::util::{number_from_u8, Number}; // If the bodyform represents a constant, only match a quoted string. -fn match_quoted_string(body: Rc) -> Result)>, CompileErr> { - let is_string = match body.borrow() { - SExp::QuotedString(_, b'x', _) => None, - SExp::QuotedString(al, _, an) => Some((al.clone(), an.clone())), - _ => None, - }; - - if let Some((loc, s)) = is_string { - Ok(Some((loc, s))) - } else { - Err(CompileErr(body.loc(), "string required".to_string())) +fn match_quoted_string(body: Rc) -> Result<(Srcloc, Vec), CompileErr> { + match body.borrow() { + SExp::QuotedString(_, b'x', _) => {} + SExp::QuotedString(al, _, an) => return Ok((al.clone(), an.clone())), + _ => {} } + + Err(CompileErr(body.loc(), "string required".to_string())) } -fn match_atom(body: Rc) -> Result)>, CompileErr> { - if let SExp::Atom(al, an) = body.borrow() { - Ok(Some((al.clone(), an.clone()))) - } else { - Err(CompileErr(body.loc(), "atom required".to_string())) +fn match_atom(body: Rc) -> Result<(Srcloc, Vec), CompileErr> { + match body.borrow() { + SExp::Atom(al, an) => return Ok((al.clone(), an.clone())), + _ => {} } + Err(CompileErr(body.loc(), "atom required".to_string())) } enum MatchedNumber { @@ -42,35 +38,34 @@ enum MatchedNumber { MatchedHex(Srcloc, Vec), } -fn match_number(body: Rc) -> Result, CompileErr> { +fn match_number(body: Rc) -> Result { match body.borrow() { SExp::Integer(il, n) => { - return Ok(Some(MatchedNumber::MatchedInt(il.clone(), n.clone()))); + return Ok(MatchedNumber::MatchedInt(il.clone(), n.clone())); } SExp::QuotedString(ql, b'x', b) => { - return Ok(Some(MatchedNumber::MatchedHex(ql.clone(), b.clone()))); + return Ok(MatchedNumber::MatchedHex(ql.clone(), b.clone())); } SExp::Atom(al, b) => { // An atom with unprintable characters is rendered as an integer. if !printable(b) { let to_integer = number_from_u8(b); - return Ok(Some(MatchedNumber::MatchedInt(al.clone(), to_integer))); + return Ok(MatchedNumber::MatchedInt(al.clone(), to_integer)); } } SExp::Nil(il) => { - return Ok(Some(MatchedNumber::MatchedInt(il.clone(), bi_zero()))); + return Ok(MatchedNumber::MatchedInt(il.clone(), bi_zero())); } _ => {} } - Err(CompileErr(body.loc(), "number required".to_string())) + Err(CompileErr(body.loc(), "Not a number".to_string())) } fn numeric_value(body: Rc) -> Result { match match_number(body.clone())? { - Some(MatchedNumber::MatchedInt(_, n)) => Ok(n), - Some(MatchedNumber::MatchedHex(_, h)) => Ok(number_from_u8(&h)), - _ => Err(CompileErr(body.loc(), "Not a number".to_string())), + MatchedNumber::MatchedInt(_, n) => Ok(n), + MatchedNumber::MatchedHex(_, h) => Ok(number_from_u8(&h)), } } @@ -112,7 +107,7 @@ impl ExtensionFunction for StringQ { fn try_eval(&self, loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { let res = match match_quoted_string(args[0].clone()) { - Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), + Ok(_) => SExp::Integer(loc.clone(), bi_one()), _ => SExp::Nil(loc.clone()), }; @@ -135,7 +130,7 @@ impl ExtensionFunction for NumberQ { fn try_eval(&self, loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { let res = match match_number(args[0].clone()) { - Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), + Ok(_) => SExp::Integer(loc.clone(), bi_one()), _ => SExp::Nil(loc.clone()), }; @@ -158,7 +153,7 @@ impl ExtensionFunction for SymbolQ { fn try_eval(&self, loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { let res = match match_atom(args[0].clone()) { - Ok(Some(_)) => SExp::Integer(loc.clone(), bi_one()), + Ok(_) => SExp::Integer(loc.clone(), bi_one()), _ => SExp::Nil(loc.clone()), }; @@ -180,11 +175,8 @@ impl ExtensionFunction for SymbolToString { } fn try_eval(&self, _loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { - if let Some((loc, value)) = match_atom(args[0].clone())? { - Ok(Rc::new(SExp::QuotedString(loc, b'\"', value))) - } else { - Err(CompileErr(args[0].loc(), "Not a symbol".to_string())) - } + let (loc, value) = match_atom(args[0].clone())?; + Ok(Rc::new(SExp::QuotedString(loc, b'\"', value))) } } @@ -202,11 +194,8 @@ impl ExtensionFunction for StringToSymbol { } fn try_eval(&self, _loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { - if let Some((loc, value)) = match_quoted_string(args[0].clone())? { - Ok(Rc::new(SExp::Atom(loc, value))) - } else { - Err(CompileErr(args[0].loc(), "Not a string".to_string())) - } + let (loc, value) = match_quoted_string(args[0].clone())?; + Ok(Rc::new(SExp::Atom(loc, value))) } } @@ -227,14 +216,11 @@ impl ExtensionFunction for StringAppend { let mut out_vec = Vec::new(); let mut out_loc = None; for a in args.iter() { - if let Some((loc, mut value)) = match_quoted_string(a.clone())? { - if out_loc.is_none() { - out_loc = Some(loc); - } - out_vec.append(&mut value); - } else { - return Err(CompileErr(a.loc(), "not a quoted string".to_string())); + let (loc, mut value) = match_quoted_string(a.clone())?; + if out_loc.is_none() { + out_loc = Some(loc); } + out_vec.append(&mut value); } Ok(Rc::new(SExp::QuotedString( out_loc.unwrap_or_else(|| loc.clone()), @@ -260,11 +246,8 @@ impl ExtensionFunction for NumberToString { fn try_eval(&self, _loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { let match_res = match_number(args[0].clone())?; let (use_loc, int_val) = match &match_res { - Some(MatchedNumber::MatchedInt(l, i)) => (l.clone(), i.clone()), - Some(MatchedNumber::MatchedHex(l, h)) => (l.clone(), number_from_u8(h)), - _ => { - return Err(CompileErr(args[0].loc(), "Not a number".to_string())); - } + MatchedNumber::MatchedInt(l, i) => (l.clone(), i.clone()), + MatchedNumber::MatchedHex(l, h) => (l.clone(), number_from_u8(h)), }; Ok(Rc::new(SExp::QuotedString( use_loc, @@ -287,18 +270,12 @@ impl ExtensionFunction for StringToNumber { Some(1) } - fn try_eval(&self, loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { - if let Some((loc, value)) = match_quoted_string(args[0].clone())? { - if let Ok(cvt_bi) = decode_string(&value).parse::() { - Ok(Rc::new(SExp::Integer(loc, cvt_bi))) - } else { - Err(CompileErr(loc, "bad number".to_string())) - } + fn try_eval(&self, _loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { + let (loc, value) = match_quoted_string(args[0].clone())?; + if let Ok(cvt_bi) = decode_string(&value).parse::() { + Ok(Rc::new(SExp::Integer(loc, cvt_bi))) } else { - Err(CompileErr( - loc.clone(), - "should be given a string".to_string(), - )) + Err(CompileErr(loc, "bad number".to_string())) } } } @@ -317,12 +294,10 @@ impl ExtensionFunction for StringLength { } fn try_eval(&self, _loc: &Srcloc, args: &[Rc]) -> Result, CompileErr> { - if let Some((loc, value)) = match_quoted_string(args[0].clone())? { - if let Some(len_bi) = value.len().to_bigint() { - return Ok(Rc::new(SExp::Integer(loc, len_bi))); - } + let (loc, value) = match_quoted_string(args[0].clone())?; + if let Some(len_bi) = value.len().to_bigint() { + return Ok(Rc::new(SExp::Integer(loc, len_bi))); } - Err(CompileErr( args[0].loc(), "Error getting string length".to_string(), From 5efb7a984ad3da04a7cc2e2ce8fe58cf75768449 Mon Sep 17 00:00:00 2001 From: arty Date: Thu, 19 Oct 2023 09:14:23 -0700 Subject: [PATCH 80/81] fmt + clippy --- src/compiler/preprocessor/macros.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index f5ccb9f31..2064371df 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -26,9 +26,8 @@ fn match_quoted_string(body: Rc) -> Result<(Srcloc, Vec), CompileErr> } fn match_atom(body: Rc) -> Result<(Srcloc, Vec), CompileErr> { - match body.borrow() { - SExp::Atom(al, an) => return Ok((al.clone(), an.clone())), - _ => {} + if let SExp::Atom(al, an) = body.borrow() { + return Ok((al.clone(), an.clone())); } Err(CompileErr(body.loc(), "atom required".to_string())) } From ba0035ad4f00f11465ff9d294cf60140a5495cbe Mon Sep 17 00:00:00 2001 From: arty Date: Tue, 24 Oct 2023 06:14:21 -0700 Subject: [PATCH 81/81] Take some code shortenings from amine --- src/compiler/codegen.rs | 30 ++++++----------- src/compiler/preprocessor/macros.rs | 52 ++++++++++++----------------- src/compiler/preprocessor/mod.rs | 10 ++---- src/compiler/runtypes.rs | 8 +++++ src/compiler/sexp.rs | 4 +-- 5 files changed, 44 insertions(+), 60 deletions(-) diff --git a/src/compiler/codegen.rs b/src/compiler/codegen.rs index 589cecaf1..8267a4964 100644 --- a/src/compiler/codegen.rs +++ b/src/compiler/codegen.rs @@ -700,28 +700,20 @@ pub fn generate_expr_code( // strict mode closes the necessary loophole below. Values // intended as variable names are never crushed into integer // like values from modern macros. - if opts.dialect().strict { - return generate_expr_code( - context, - opts, - compiler, - Rc::new(BodyForm::Quoted(SExp::Integer(l.clone(), i.clone()))), - ); - } - - // Since macros are in this language and the runtime has - // a very narrow data representation, we'll need to - // accomodate bare numbers coming back in place of identifiers, - // but only in legacy non-strict mode. - generate_expr_code( - context, - opts, - compiler, + let ambiguous_int_value = if opts.dialect().strict { + Rc::new(BodyForm::Quoted(SExp::Integer(l.clone(), i.clone()))) + } else { + // Since macros are in this language and the runtime has + // a very narrow data representation, we'll need to + // accomodate bare numbers coming back in place of identifiers, + // but only in legacy non-strict mode. Rc::new(BodyForm::Value(SExp::Atom( l.clone(), u8_from_number(i.clone()), - ))), - ) + ))) + }; + + generate_expr_code(context, opts, compiler, ambiguous_int_value) } _ => Ok(CompiledCode( v.loc(), diff --git a/src/compiler/preprocessor/macros.rs b/src/compiler/preprocessor/macros.rs index 2064371df..68a0c0b0d 100644 --- a/src/compiler/preprocessor/macros.rs +++ b/src/compiler/preprocessor/macros.rs @@ -87,15 +87,14 @@ pub trait ExtensionFunction { true } fn required_args(&self) -> Option; - #[allow(clippy::too_many_arguments)] fn try_eval(&self, loc: &Srcloc, args: &[Rc]) -> Result, CompileErr>; } -struct StringQ {} +struct StringQ; impl StringQ { fn create() -> Rc { - Rc::new(StringQ {}) + Rc::new(StringQ) } } @@ -114,11 +113,11 @@ impl ExtensionFunction for StringQ { } } -struct NumberQ {} +struct NumberQ; impl NumberQ { fn create() -> Rc { - Rc::new(NumberQ {}) + Rc::new(NumberQ) } } @@ -137,11 +136,11 @@ impl ExtensionFunction for NumberQ { } } -struct SymbolQ {} +struct SymbolQ; impl SymbolQ { fn create() -> Rc { - Rc::new(SymbolQ {}) + Rc::new(SymbolQ) } } @@ -160,11 +159,11 @@ impl ExtensionFunction for SymbolQ { } } -struct SymbolToString {} +struct SymbolToString; impl SymbolToString { fn create() -> Rc { - Rc::new(SymbolToString {}) + Rc::new(SymbolToString) } } @@ -179,11 +178,11 @@ impl ExtensionFunction for SymbolToString { } } -struct StringToSymbol {} +struct StringToSymbol; impl StringToSymbol { fn create() -> Rc { - Rc::new(StringToSymbol {}) + Rc::new(StringToSymbol) } } @@ -198,11 +197,11 @@ impl ExtensionFunction for StringToSymbol { } } -struct StringAppend {} +struct StringAppend; impl StringAppend { fn create() -> Rc { - Rc::new(StringAppend {}) + Rc::new(StringAppend) } } @@ -229,11 +228,11 @@ impl ExtensionFunction for StringAppend { } } -struct NumberToString {} +struct NumberToString; impl NumberToString { fn create() -> Rc { - Rc::new(NumberToString {}) + Rc::new(NumberToString) } } @@ -256,11 +255,11 @@ impl ExtensionFunction for NumberToString { } } -struct StringToNumber {} +struct StringToNumber; impl StringToNumber { fn create() -> Rc { - Rc::new(StringToNumber {}) + Rc::new(StringToNumber) } } @@ -279,11 +278,11 @@ impl ExtensionFunction for StringToNumber { } } -struct StringLength {} +struct StringLength; impl StringLength { fn create() -> Rc { - Rc::new(StringLength {}) + Rc::new(StringLength) } } @@ -304,11 +303,11 @@ impl ExtensionFunction for StringLength { } } -struct Substring {} +struct Substring; impl Substring { fn create() -> Rc { - Rc::new(Substring {}) + Rc::new(Substring) } } @@ -380,12 +379,6 @@ pub struct PreprocessorExtension { extfuns: HashMap, Rc>, } -fn compile_to_run_err(e: CompileErr) -> RunFailure { - match e { - CompileErr(l, e) => RunFailure::RunErr(l, e), - } -} - impl PrimOverride for PreprocessorExtension { fn try_handle( &self, @@ -401,10 +394,7 @@ impl PrimOverride for PreprocessorExtension { }; if let Some(extension) = self.extfuns.get(head_atom) { - let res = extension - .try_eval(hl, &have_args) - .map_err(compile_to_run_err)?; - + let res = extension.try_eval(hl, &have_args)?; return Ok(Some(res)); } } diff --git a/src/compiler/preprocessor/mod.rs b/src/compiler/preprocessor/mod.rs index cc4f0b253..69fadd9dc 100644 --- a/src/compiler/preprocessor/mod.rs +++ b/src/compiler/preprocessor/mod.rs @@ -253,9 +253,8 @@ impl Preprocessor { return Ok(new_self); } - if let Ok(NodeSel::Cons((_, name), args)) = - NodeSel::Cons(Atom::Here(()), ThisNode::Here) - .select_nodes(new_self.clone().unwrap_or_else(|| body.clone())) + if let Ok(NodeSel::Cons((_, name), args)) = NodeSel::Cons(Atom::Here(()), ThisNode) + .select_nodes(new_self.clone().unwrap_or_else(|| body.clone())) { let defmac_name = make_defmac_name(&name); @@ -347,10 +346,7 @@ impl Preprocessor { NodeSel::Cons((nl, name), NodeSel::Cons(args, body)), )) = NodeSel::Cons( Atom::Here(()), - NodeSel::Cons( - Atom::Here(()), - NodeSel::Cons(ThisNode::Here, ThisNode::Here), - ), + NodeSel::Cons(Atom::Here(()), NodeSel::Cons(ThisNode, ThisNode)), ) .select_nodes(definition.clone()) { diff --git a/src/compiler/runtypes.rs b/src/compiler/runtypes.rs index c19d5cc45..526f9afef 100644 --- a/src/compiler/runtypes.rs +++ b/src/compiler/runtypes.rs @@ -37,3 +37,11 @@ impl From for CompileErr { } } } + +impl From for RunFailure { + fn from(e: CompileErr) -> Self { + match e { + CompileErr(l, e) => RunFailure::RunErr(l, e), + } + } +} diff --git a/src/compiler/sexp.rs b/src/compiler/sexp.rs index 605bf48d0..30becac44 100644 --- a/src/compiler/sexp.rs +++ b/src/compiler/sexp.rs @@ -1007,9 +1007,7 @@ pub enum Rest { } #[derive(Debug, Clone)] -pub enum ThisNode { - Here, -} +pub struct ThisNode; pub enum Atom { Here(T),