From 86c0fc8b3006a9a744270c03f5ca0057e4757a53 Mon Sep 17 00:00:00 2001 From: John-John Tedro Date: Sun, 14 Apr 2024 19:33:28 +0200 Subject: [PATCH] Add support for captured arguments from macros --- book/src/hot_reloading.md | 2 +- book/src/multithreading.md | 15 +-- crates/rune/src/compile/context.rs | 1 - crates/rune/src/lib.rs | 17 +-- crates/rune/src/macros/format_args.rs | 154 ++++++++++++++---------- crates/rune/src/macros/token_stream.rs | 2 +- crates/rune/src/modules.rs | 1 - crates/rune/src/modules/io.rs | 20 ++- crates/rune/src/tests.rs | 1 + crates/rune/src/tests/builtin_macros.rs | 73 +++++++++++ 10 files changed, 199 insertions(+), 87 deletions(-) create mode 100644 crates/rune/src/tests/builtin_macros.rs diff --git a/book/src/hot_reloading.md b/book/src/hot_reloading.md index 0a586eeab..879892ac8 100644 --- a/book/src/hot_reloading.md +++ b/book/src/hot_reloading.md @@ -11,7 +11,7 @@ A typical way to accomplish this is to watch a scripts directory using the to the directory are detected. See the [`hot_reloading` example] and in particular the [`PathReloader`] type. -``` +```rust {{#include ../../examples/examples/hot_reloading.rs}} ``` diff --git a/book/src/multithreading.md b/book/src/multithreading.md index 967450690..86751c998 100644 --- a/book/src/multithreading.md +++ b/book/src/multithreading.md @@ -14,11 +14,6 @@ used by multiple threads simultaneously through `Arc` and `Arc`. Constructing a `Vm` with these through `Vm::new` is a very cheap operation. -> `Vm`'s do allocate a stack, to avoid this you'd have to employ even more -> advanced techniques, such as storing the virtual machine in a thread local and -> using it by swapping out the `Unit` and `RuntimeContext` associated with it -> through [`Vm::unit_mut`] and [`Vm::context_mut`] respectively. - ```rust let unit: Arc = /* todo */; let context: Arc = /* todo */; @@ -30,18 +25,24 @@ std::thread::spawn(move || { }); ``` +> Virtual machines do allocate memory. To this overhead too you'd have to employ +> more advanced techniques, such as storing virtual machines in a pool or thread +> locals. Once a machine has been acquired the `Unit` and `RuntimeContext` +> associated with it can be swapped out to the ones you need using +> [`Vm::unit_mut`] and [`Vm::context_mut`] respectively. + Using [`Vm::send_execute`] is a way to assert that a given execution is thread safe. And allows you to use Rune in asynchronous multithreaded environments, such as Tokio. This is achieved by ensuring that all captured arguments are [`ConstValue`]'s, which in contrast to [`Value`]'s are guaranteed to be thread-safe: -``` +```rust {{#include ../../examples/examples/tokio_spawn.rs}} ``` Finally [`Function::into_sync`] exists to coerce a function into a -[`SyncFunction], which is a thread-safe variant of a regular [`Function`]. This +[`SyncFunction`], which is a thread-safe variant of a regular [`Function`]. This is a fallible operation since all values which are captured in the function-type in case its a closure has to be coerced to [`ConstValue`]. If this is not the case, the conversion will fail. diff --git a/crates/rune/src/compile/context.rs b/crates/rune/src/compile/context.rs index 6effa0157..86e0e7e80 100644 --- a/crates/rune/src/compile/context.rs +++ b/crates/rune/src/compile/context.rs @@ -152,7 +152,6 @@ impl Context { this.install(crate::modules::fmt::module()?)?; this.install(crate::modules::future::module()?)?; this.install(crate::modules::i64::module()?)?; - #[cfg(feature = "std")] this.install(crate::modules::io::module(stdio)?)?; this.install(crate::modules::iter::module()?)?; this.install(crate::modules::macros::module()?)?; diff --git a/crates/rune/src/lib.rs b/crates/rune/src/lib.rs index 06807fe78..99379ad78 100644 --- a/crates/rune/src/lib.rs +++ b/crates/rune/src/lib.rs @@ -29,18 +29,17 @@ //! * Runs a compact representation of the language on top of an efficient //! [stack-based virtual machine][support-virtual-machine]. //! * Clean [Rust integration 💻][support-rust-integration]. +//! * [Multithreaded 📖][support-multithreading] execution. +//! * [Hot reloading 📖][support-hot-reloading]. //! * Memory safe through [reference counting 📖][support-reference-counted]. -//! * [Awesome macros 📖][support-macros]. -//! * [Template literals 📖][support-templates]. -//! * [Try operators 📖][support-try]. -//! * [Pattern matching 📖][support-patterns]. +//! * [Awesome macros 📖][support-macros] and [Template literals 📖][support-templates]. +//! * [Try operators 📖][support-try] and [Pattern matching 📖][support-patterns]. //! * [Structs and enums 📖][support-structs] with associated data and //! functions. -//! * Dynamic [vectors 📖][support-dynamic-vectors], [objects -//! 📖][support-anon-objects], and [tuples 📖][support-anon-tuples] with +//! * Dynamic containers like [vectors 📖][support-dynamic-vectors], [objects +//! 📖][support-anon-objects], and [tuples 📖][support-anon-tuples] all with //! out-of-the-box [serde support 💻][support-serde]. -//! * First-class [async support 📖][support-async]. -//! * [Generators 📖][support-generators]. +//! * First-class [async support 📖][support-async] with [Generators 📖][support-generators]. //! * Dynamic [instance functions 📖][support-instance-functions]. //! * [Stack isolation 📖][support-stack-isolation] between function calls. //! @@ -114,8 +113,10 @@ //! [support-async]: https://rune-rs.github.io/book/async.html //! [support-dynamic-vectors]: https://rune-rs.github.io/book/vectors.html //! [support-generators]: https://rune-rs.github.io/book/generators.html +//! [support-hot-reloading]: https://rune-rs.github.io/book/hot_reloading.html //! [support-instance-functions]: https://rune-rs.github.io/book/instance_functions.html //! [support-macros]: https://rune-rs.github.io/book/macros.html +//! [support-multithreading]: https://rune-rs.github.io/book/multithreading.html //! [support-patterns]: https://rune-rs.github.io/book/pattern_matching.html //! [support-reference-counted]: https://rune-rs.github.io/book/variables.html //! [support-rust-integration]: https://github.com/rune-rs/rune/tree/main/crates/rune-modules diff --git a/crates/rune/src/macros/format_args.rs b/crates/rune/src/macros/format_args.rs index d5fad43a1..989feb564 100644 --- a/crates/rune/src/macros/format_args.rs +++ b/crates/rune/src/macros/format_args.rs @@ -5,7 +5,7 @@ use crate::alloc::prelude::*; use crate::alloc::{self, BTreeMap, BTreeSet, Box, HashMap, String, Vec}; use crate::ast::{self, Span, Spanned}; use crate::compile::{self, WithSpan}; -use crate::macros::{quote, MacroContext, Quote}; +use crate::macros::{quote, MacroContext, Quote, ToTokens, TokenStream}; use crate::parse::{Parse, Parser, Peek, Peeker}; use crate::runtime::format; use crate::runtime::ValueKind; @@ -49,14 +49,11 @@ impl FormatArgs { } } - let format = match format.take_kind().with_span(&self.format)? { - ValueKind::String(string) => string, - _ => { - return Err(compile::Error::msg( - &self.format, - "format argument must be a string", - )); - } + let ValueKind::String(format) = format.take_kind().with_span(&self.format)? else { + return Err(compile::Error::msg( + &self.format, + "format argument must be a string", + )); }; let mut unused_pos = (0..pos.len()).try_collect::>()?; @@ -65,7 +62,7 @@ impl FormatArgs { .map(|(key, n)| Ok::<_, alloc::Error>((key.try_clone()?, n.span()))) .try_collect::>>()??; - let expanded = match expand_format_spec( + let result = expand_format_spec( cx, self.format.span(), &format, @@ -73,11 +70,11 @@ impl FormatArgs { &mut unused_pos, &named, &mut unused_named, - ) { + ); + + let expanded = match result { Ok(expanded) => expanded, - Err(message) => { - return Err(compile::Error::msg(self.format.span(), message)); - } + Err(message) => return Err(compile::Error::msg(self.format.span(), message)), }; if let Some(expr) = unused_pos.into_iter().flat_map(|n| pos.get(n)).next() { @@ -174,18 +171,27 @@ fn expand_format_spec<'a>( let mut name = String::new(); let mut width = String::new(); let mut precision = String::new(); - let mut buf = String::new(); + let mut buf = String::new(); let mut components = Vec::new(); let mut count = 0; + let mut start = Some(0); - while let Some(value) = iter.next() { - match value { + while let Some((at, a, b)) = iter.next() { + match (a, b) { ('}', '}') => { + if let Some(start) = start.take() { + buf.try_push_str(&input[start..at])?; + } + buf.try_push('}')?; iter.next(); } ('{', '{') => { + if let Some(start) = start.take() { + buf.try_push_str(&input[start..at])?; + } + buf.try_push('{')?; iter.next(); } @@ -196,8 +202,12 @@ fn expand_format_spec<'a>( )); } ('{', _) => { + if let Some(start) = start.take() { + buf.try_push_str(&input[start..at])?; + } + if !buf.is_empty() { - components.try_push(C::Literal(buf.try_clone()?.try_into_boxed_str()?))?; + components.try_push(C::Literal(Box::try_from(&buf[..])?))?; buf.clear(); } @@ -215,14 +225,20 @@ fn expand_format_spec<'a>( unused_named, )?)?; } - (a, _) => { - buf.try_push(a)?; + _ => { + if start.is_none() { + start = Some(at); + } } } } + if let Some(start) = start.take() { + buf.try_push_str(&input[start..])?; + } + if !buf.is_empty() { - components.try_push(C::Literal(buf.try_clone()?.try_into_boxed_str()?))?; + components.try_push(C::Literal(Box::try_from(&buf[..])?))?; buf.clear(); } @@ -235,7 +251,7 @@ fn expand_format_spec<'a>( for c in components { match c { C::Literal(literal) => { - let lit = cx.lit(&*literal)?; + let lit = cx.lit(literal.as_ref())?; args.try_push(quote!(#lit))?; } C::Format { @@ -314,10 +330,28 @@ fn expand_format_spec<'a>( #[builtin] template!(#(args),*) }); + enum ExprOrIdent<'a> { + Expr(&'a ast::Expr), + Ident(ast::Ident), + } + + impl ToTokens for ExprOrIdent<'_> { + fn to_tokens( + &self, + cx: &mut MacroContext<'_, '_, '_>, + stream: &mut TokenStream, + ) -> alloc::Result<()> { + match self { + Self::Expr(expr) => expr.to_tokens(cx, stream), + Self::Ident(ident) => ident.to_tokens(cx, stream), + } + } + } + enum C<'a> { Literal(Box), Format { - expr: &'a ast::Expr, + expr: ExprOrIdent<'a>, fill: Option, align: Option, width: Option, @@ -384,11 +418,8 @@ fn expand_format_spec<'a>( let mut mode = Mode::Start; loop { - let (a, b) = match iter.current() { - Some(item) => item, - _ => { - return Err(compile::Error::msg(span, "unexpected end of format string")); - } + let Some((_, a, b)) = iter.current() else { + return Err(compile::Error::msg(span, "unexpected end of format string")); }; match mode { @@ -577,7 +608,20 @@ fn expand_format_spec<'a>( None }; - let expr = if !name.is_empty() { + let expr = 'expr: { + if name.is_empty() { + let Some(expr) = pos.get(*count) else { + return Err(compile::Error::msg( + span, + format!("missing positional argument #{count}"), + )); + }; + + unused_pos.remove(count); + *count += 1; + break 'expr ExprOrIdent::Expr(expr); + }; + if let Ok(n) = str::parse::(name) { let expr = match pos.get(n) { Some(expr) => *expr, @@ -590,35 +634,17 @@ fn expand_format_spec<'a>( }; unused_pos.remove(&n); - expr - } else { - let expr = match named.get(name.as_str()) { - Some(n) => &n.expr, - None => { - return Err(compile::Error::msg( - span, - format!("missing named argument `{}`", name), - )); - } - }; + break 'expr ExprOrIdent::Expr(expr); + } + if let Some(n) = named.get(name.as_str()) { unused_named.remove(name.as_str()); - expr + break 'expr ExprOrIdent::Expr(&n.expr); } - } else { - let expr = match pos.get(*count) { - Some(expr) => *expr, - None => { - return Err(compile::Error::msg( - span, - format!("missing positional argument #{}", count), - )); - } - }; - unused_pos.remove(count); - *count += 1; - expr + let mut ident = cx.ident(name.as_str())?; + ident.span = span; + ExprOrIdent::Ident(ident) }; let width = if !width.is_empty() { @@ -648,30 +674,28 @@ fn expand_format_spec<'a>( } struct Iter<'a> { - iter: str::Chars<'a>, - a: Option, - b: Option, + iter: str::CharIndices<'a>, + a: Option<(usize, char)>, + b: Option<(usize, char)>, } impl<'a> Iter<'a> { fn new(input: &'a str) -> Self { - let mut iter = input.chars(); - + let mut iter = input.char_indices(); let a = iter.next(); let b = iter.next(); - Self { iter, a, b } } - fn current(&self) -> Option<(char, char)> { - let a = self.a?; - let b = self.b.unwrap_or_default(); - Some((a, b)) + fn current(&self) -> Option<(usize, char, char)> { + let (pos, a) = self.a?; + let (_, b) = self.b.unwrap_or_default(); + Some((pos, a, b)) } } impl Iterator for Iter<'_> { - type Item = (char, char); + type Item = (usize, char, char); fn next(&mut self) -> Option { let value = self.current()?; diff --git a/crates/rune/src/macros/token_stream.rs b/crates/rune/src/macros/token_stream.rs index 0a941a243..436fd8caf 100644 --- a/crates/rune/src/macros/token_stream.rs +++ b/crates/rune/src/macros/token_stream.rs @@ -125,7 +125,7 @@ pub trait ToTokens { /// Turn the current item into tokens. fn to_tokens( &self, - context: &mut MacroContext<'_, '_, '_>, + cx: &mut MacroContext<'_, '_, '_>, stream: &mut TokenStream, ) -> alloc::Result<()>; } diff --git a/crates/rune/src/modules.rs b/crates/rune/src/modules.rs index 86737af4a..ff3fdc5df 100644 --- a/crates/rune/src/modules.rs +++ b/crates/rune/src/modules.rs @@ -21,7 +21,6 @@ pub mod future; pub mod generator; pub mod hash; pub mod i64; -#[cfg(feature = "std")] pub mod io; pub mod iter; pub mod macros; diff --git a/crates/rune/src/modules/io.rs b/crates/rune/src/modules/io.rs index 280a80494..a124e57dd 100644 --- a/crates/rune/src/modules/io.rs +++ b/crates/rune/src/modules/io.rs @@ -1,18 +1,25 @@ //! I/O functions. +#[cfg(feature = "std")] use std::io::{self, Write as _}; use crate as rune; +#[cfg(feature = "std")] use crate::alloc::fmt::TryWrite; use crate::compile; use crate::macros::{quote, FormatArgs, MacroContext, TokenStream}; use crate::parse::Parser; -use crate::runtime::{Formatter, Panic, Stack, Value, VmResult}; +#[cfg(feature = "std")] +use crate::runtime::{Formatter, VmResult}; +#[cfg(feature = "std")] +use crate::runtime::{Panic, Stack, Value}; use crate::{ContextError, Module}; /// I/O functions. #[rune::module(::std::io)] -pub fn module(stdio: bool) -> Result { +pub fn module( + #[cfg_attr(not(feature = "std"), allow(unused))] stdio: bool, +) -> Result { let mut module = Module::from_meta(self::module_meta)?.with_unique("std::io"); module.item_mut().docs([ @@ -31,9 +38,12 @@ pub fn module(stdio: bool) -> Result { "to be hooked up to whatever system you want.", ])?; + #[cfg(feature = "std")] module.ty::()?; + #[cfg(feature = "std")] module.function_meta(io_error_string_display)?; + #[cfg(feature = "std")] if stdio { module.function_meta(print_impl)?; module.function_meta(println_impl)?; @@ -68,11 +78,13 @@ pub fn module(stdio: bool) -> Result { } #[rune::function(instance, protocol = STRING_DISPLAY)] +#[cfg(feature = "std")] fn io_error_string_display(error: &io::Error, f: &mut Formatter) -> VmResult<()> { - vm_write!(f, "{}", error); + vm_write!(f, "{error}"); VmResult::Ok(()) } +#[cfg(feature = "std")] fn dbg_impl(stack: &mut Stack, args: usize) -> VmResult<()> { let stdout = io::stdout(); let mut stdout = stdout.lock(); @@ -147,6 +159,7 @@ pub(crate) fn print_macro( /// print("Hi!"); /// ``` #[rune::function(path = print)] +#[cfg(feature = "std")] fn print_impl(m: &str) -> VmResult<()> { let stdout = io::stdout(); let mut stdout = stdout.lock(); @@ -192,6 +205,7 @@ pub(crate) fn println_macro( /// println("Hi!"); /// ``` #[rune::function(path = println)] +#[cfg(feature = "std")] fn println_impl(message: &str) -> VmResult<()> { let stdout = io::stdout(); let mut stdout = stdout.lock(); diff --git a/crates/rune/src/tests.rs b/crates/rune/src/tests.rs index 729d37661..022766521 100644 --- a/crates/rune/src/tests.rs +++ b/crates/rune/src/tests.rs @@ -409,6 +409,7 @@ mod bug_422; mod bug_428; mod bug_454; mod bugfixes; +mod builtin_macros; mod capture; mod char; mod collections; diff --git a/crates/rune/src/tests/builtin_macros.rs b/crates/rune/src/tests/builtin_macros.rs new file mode 100644 index 000000000..c472f1ab0 --- /dev/null +++ b/crates/rune/src/tests/builtin_macros.rs @@ -0,0 +1,73 @@ +prelude!(); + +use crate::termcolor::{ColorChoice, StandardStream}; + +macro_rules! capture { + ($($tt:tt)*) => {{ + let capture = crate::modules::capture_io::CaptureIo::new(); + let module = crate::modules::capture_io::module(&capture).context("building capture module")?; + + let mut context = Context::with_config(false).context("building context")?; + context.install(module).context("installing module")?; + + let source = Source::memory(concat!("pub fn main() { ", stringify!($($tt)*), " }")).context("building source")?; + + let mut sources = Sources::new(); + sources.insert(source).context("inserting source")?; + + let mut diagnostics = Diagnostics::new(); + + let unit = prepare(&mut sources).with_context(&context).with_diagnostics(&mut diagnostics).build(); + + if !diagnostics.is_empty() { + let mut writer = StandardStream::stderr(ColorChoice::Always); + diagnostics.emit(&mut writer, &sources)?; + } + + let unit = Arc::new(unit.context("building unit")?); + + let context = context.runtime().context("constructing runtime context")?; + let context = Arc::new(context); + + let mut vm = Vm::new(context, unit); + vm.call(["main"], ()).context("calling main")?; + capture.drain_utf8().context("draining utf-8 capture")? + }}; +} + +macro_rules! test_case { + ($expected:expr, {$($prefix:tt)*}, $($format:tt)*) => {{ + let string = capture!($($prefix)* println!($($format)*)); + assert_eq!(string, concat!($expected, "\n"), "Expecting println!"); + + let string = capture!($($prefix)* print!($($format)*)); + assert_eq!(string, $expected, "Expecting print!"); + + let string: String = rune!(pub fn main() { $($prefix)* format!($($format)*) }); + assert_eq!(string, $expected, "Expecting format!"); + }} +} + +#[test] +fn format_macros() -> Result<()> { + test_case!("Hello World!", {}, "Hello World!"); + test_case!("Hello World!", {}, "Hello {}!", "World"); + test_case!( + "Hello World!", + { + let pos = "Hello"; + }, + "{pos} {}!", + "World" + ); + test_case!( + "Hello World!", + { + let pos = "Not Hello"; + }, + "{pos} {}!", + "World", + pos = "Hello" + ); + Ok(()) +}