From 1a395b786b09274667bf1495abcca09d6ae0c57b Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Sat, 23 Sep 2023 16:32:57 +0800 Subject: [PATCH 01/28] Initial draft of API-wide MeTTa Environment object, to replace the non-UI parts of the REPL environment --- lib/Cargo.toml | 3 +- lib/src/metta/environment.rs | 148 ++++++++++++++++++ .../src => lib/src/metta}/init.default.metta | 0 lib/src/metta/mod.rs | 2 + repl/Cargo.toml | 1 - repl/src/config_params.rs | 99 +++--------- repl/src/main.rs | 47 ++++-- repl/src/metta_shim.rs | 24 ++- 8 files changed, 219 insertions(+), 105 deletions(-) create mode 100644 lib/src/metta/environment.rs rename {repl/src => lib/src/metta}/init.default.metta (100%) diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 0b40e6864..2e173b68b 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -2,13 +2,14 @@ name = "hyperon" version = "0.1.6" authors = ["Vitaly Bogdanov "] -edition = "2018" +edition = "2021" [dependencies] mopa = "0.2.2" regex = "1.5.4" log = "0.4.0" env_logger = "0.8.4" +directories = "5.0.1" # For Environment to find platform-specific config location ctor = "0.2.0" smallvec = "1.10.0" diff --git a/lib/src/metta/environment.rs b/lib/src/metta/environment.rs new file mode 100644 index 000000000..9839b00f7 --- /dev/null +++ b/lib/src/metta/environment.rs @@ -0,0 +1,148 @@ + +use std::path::{Path, PathBuf}; +use std::io::Write; +use std::fs; +use std::borrow::Borrow; + +use directories::ProjectDirs; + +/// Contains state and platform interfaces shared by all MeTTa runners. This includes config settings +/// and logger +/// +/// Generally there will be only one environment object needed, and it can be accessed by calling the [platform_env] method +#[derive(Debug)] +pub struct Environment { + config_dir: Option, + init_metta_path: Option, + working_dir: Option, + extra_include_paths: Vec, +} + +const DEFAULT_INIT_METTA: &[u8] = include_bytes!("init.default.metta"); + +static PLATFORM_ENV: std::sync::OnceLock = std::sync::OnceLock::new(); + +impl Environment { + + /// Returns a reference to the shared "platform" Environment + pub fn platform_env() -> &'static Self { + PLATFORM_ENV.get_or_init(|| Self::new_with_defaults(None)) + } + + /// Initializes the shared "platform" Environment with with the OS-Specific platform configuration + /// + /// Config directory locations will be: + /// Linux: ~/.config/metta/ + /// Windows: ~\AppData\Roaming\TrueAGI\metta\config\ + /// Mac: ~/Library/Application Support/io.TrueAGI.metta/ + /// + /// TODO: Repeat this documentation somewhere more prominent, like the top-level README + /// + /// NOTE: This method will panic if the platform Environment has already been initialized + pub fn init(working_dir: Option<&Path>) { + PLATFORM_ENV.set(Self::new_with_defaults(working_dir)).expect("Fatal Error: Platform Environment already initialized"); + } + + /// Initializes the shared "platform" Environment with with the configuration stored in the `config_dir`. Will create + /// `config_dir` and its contents if it does not exist + /// + /// NOTE: This method will panic if the platform Environment has already been initialized + pub fn init_with_cfg_dir(working_dir: Option<&Path>, config_dir: &Path) { + PLATFORM_ENV.set(Self::new_with_config_dir(working_dir, config_dir)).expect("Fatal Error: Platform Environment already initialized"); + } + + /// Returns the Path to the config dir, in an OS-specific location + pub fn config_dir(&self) -> Option<&Path> { + self.config_dir.as_deref() + } + + /// Returns the path to the init.metta file, that is run to initialize a MeTTa runner and customize the MeTTa environment + pub fn initialization_metta_file_path(&self) -> Option<&Path> { + self.init_metta_path.as_deref() + } + + /// Returns the search paths to look in for MeTTa modules, in search priority order + /// + /// The working_dir is always returned first + pub fn modules_search_paths<'a>(&'a self) -> impl Iterator + 'a { + [&self.working_dir].into_iter().filter_map(|opt| opt.as_deref()) + .chain(self.extra_include_paths.iter().map(|path| path.borrow())) + } + + /// Returns a newly created Environment with the OS-specific platform defaults + /// + /// NOTE: Creating owned Environments is usually not necessary. It is usually sufficient to use the [platform_env]. + pub fn new_with_defaults(working_dir: Option<&Path>) -> Self { + match ProjectDirs::from("io", "TrueAGI", "metta") { + Some(proj_dirs) => { + Self::new_with_config_dir(working_dir, proj_dirs.config_dir()) + }, + None => { + eprint!("Failed to initialize config!"); + Self::new_without_config_dir(working_dir) + } + } + } + + /// Returns a newly created Environment with the configuration stored in the `config_dir`. Will create + /// `config_dir` and its contents if it does not exist + /// + /// NOTE: Creating owned Environments is usually not necessary. It is usually sufficient to use the [platform_env]. + pub fn new_with_config_dir(working_dir: Option<&Path>, config_dir: &Path) -> Self { + + //Create the modules dir inside the config dir, if it doesn't already exist. + // This will create the cfg_dir iteslf in the process + let modules_dir = config_dir.join("modules"); + std::fs::create_dir_all(&modules_dir).unwrap(); + + //Create the default init.metta file if they don't already exist + let init_metta_path = config_dir.join("init.metta"); + if !init_metta_path.exists() { + let mut file = fs::OpenOptions::new() + .create(true) + .write(true) + .open(&init_metta_path) + .expect(&format!("Error creating default init file at {init_metta_path:?}")); + file.write_all(&DEFAULT_INIT_METTA).unwrap(); + } + + //TODO_NOW, come back here and rethink what we do with include_paths. ie. where they are stored + // //Push the "modules" dir, as the last place to search after the paths specified on the cmd line + // //TODO: the config.metta file will be able to append / modify the search paths, and can choose not to + // // include the "modules" dir in the future. + // let mut include_paths = include_paths; + // include_paths.push(modules_dir); + + Self { + config_dir: Some(config_dir.into()), + init_metta_path: Some(init_metta_path), + working_dir: working_dir.map(|dir| dir.into()), + extra_include_paths: vec![], //TODO_NOW, need a way to pass in extra include paths. This is the straw that pushes me over to a builder API + } + } + + /// Returns a newly created Environment with no specialized configuration. This method will not touch any files. + /// + /// NOTE: Creating owned Environments is usually not necessary. It is usually sufficient to use the [platform_env]. + pub fn new_without_config_dir(working_dir: Option<&Path>) -> Self { + Self { + config_dir: None, + init_metta_path: None, + working_dir: working_dir.map(|dir| dir.into()), + extra_include_paths: vec![], + } + } +} + +//TODO_NOW: Transition to a builder API +// pub struct EnvBuilder { +// env: Environment +// } + +// impl EnvBuilder { + +// /// Returns a new EnvBuilder, to set the parameters for the MeTTa Environment +// pub fn new() -> Self { + +// } +// } \ No newline at end of file diff --git a/repl/src/init.default.metta b/lib/src/metta/init.default.metta similarity index 100% rename from repl/src/init.default.metta rename to lib/src/metta/init.default.metta diff --git a/lib/src/metta/mod.rs b/lib/src/metta/mod.rs index 94a82e634..c2aafdd77 100644 --- a/lib/src/metta/mod.rs +++ b/lib/src/metta/mod.rs @@ -6,6 +6,8 @@ pub mod interpreter; pub mod interpreter2; pub mod types; pub mod runner; +mod environment; +pub use environment::Environment; use text::{SExprParser, Tokenizer}; use regex::Regex; diff --git a/repl/Cargo.toml b/repl/Cargo.toml index 157439a6c..b205ec93e 100644 --- a/repl/Cargo.toml +++ b/repl/Cargo.toml @@ -12,7 +12,6 @@ hyperon = { path = "../lib/" } # TODO: Yay, our fix landed in main. Still needs to publish however. One step closer rustyline = {git = "https://github.com/kkawakam/rustyline", version = "12.0.0", features = ["derive"] } clap = { version = "4.4.0", features = ["derive"] } -directories = "5.0.1" signal-hook = "0.3.17" pyo3 = { version = "0.19.2", features = ["auto-initialize"], optional = true } pep440_rs = { version = "0.3.11", optional = true } diff --git a/repl/src/config_params.rs b/repl/src/config_params.rs index ae4414607..dbc4ddc75 100644 --- a/repl/src/config_params.rs +++ b/repl/src/config_params.rs @@ -1,9 +1,10 @@ -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use std::io::Write; use std::fs; -const DEFAULT_INIT_METTA: &[u8] = include_bytes!("init.default.metta"); +use hyperon::metta::Environment; + const DEFAULT_REPL_METTA: &[u8] = include_bytes!("repl.default.metta"); pub const CFG_PROMPT: &str = "&ReplPrompt"; @@ -20,89 +21,41 @@ pub const CFG_HISTORY_MAX_LEN: &str = "&ReplHistoryMaxLen"; #[derive(Default, Debug)] pub struct ReplParams { - /// Path to the config dir for the whole repl, in an OS-specific location - pub config_dir: PathBuf, - - /// A path to the init.metta file that's run to customize the MeTTa environment - pub init_metta_path: PathBuf, - /// A path to the repl.metta file that's run to configure the repl environment - pub repl_config_metta_path: PathBuf, - - /// Path to the dir containing the script being run, or the cwd the repl was invoked from in interactive mode - pub metta_working_dir: PathBuf, - - /// Other include paths, specified either through command-line args or config settings - include_paths: Vec, + pub repl_config_metta_path: Option, /// A file for previous statements in the interactive repl pub history_file: Option, } impl ReplParams { - pub fn new(config_dir: &Path, include_paths: Vec, metta_file: Option<&PathBuf>) -> Self { - - //If we have a metta_file, then the working dir is the parent of that file - //If we are running in interactive mode, it's the working dir at the time the repl is invoked - let metta_working_dir: PathBuf = match metta_file { - Some(metta_file) => { - metta_file.parent().unwrap().into() - }, - None => { - match std::env::current_dir() { - Ok(cwd) => cwd, - Err(_) => PathBuf::from("./").canonicalize().unwrap(), - } + pub fn new() -> Self { + + if let Some(config_dir) = Environment::platform_env().config_dir() { + + //Create the default repl.meta file, if it doesn't already exist + let repl_config_metta_path = config_dir.join("repl.metta"); + if !repl_config_metta_path.exists() { + let mut file = fs::OpenOptions::new() + .create(true) + .write(true) + .open(&repl_config_metta_path) + .expect(&format!("Error creating default repl config file at {repl_config_metta_path:?}")); + file.write_all(&DEFAULT_REPL_METTA).unwrap(); } - }; - - //Create the modules dir inside the config dir, if it doesn't already exist - let modules_dir = config_dir.join("modules"); - std::fs::create_dir_all(&modules_dir).unwrap(); - //Create the default init.metta file and repl.meta file, if they don't already exist - let init_metta_path = config_dir.join("init.metta"); - if !init_metta_path.exists() { - let mut file = fs::OpenOptions::new() - .create(true) - .write(true) - .open(&init_metta_path) - .expect(&format!("Error creating default init file at {init_metta_path:?}")); - file.write_all(&DEFAULT_INIT_METTA).unwrap(); - } - let repl_config_metta_path = config_dir.join("repl.metta"); - if !repl_config_metta_path.exists() { - let mut file = fs::OpenOptions::new() - .create(true) - .write(true) - .open(&repl_config_metta_path) - .expect(&format!("Error creating default repl config file at {repl_config_metta_path:?}")); - file.write_all(&DEFAULT_REPL_METTA).unwrap(); - } - - //Push the "modules" dir, as the last place to search after the paths specified on the cmd line - //TODO: the config.metta file will be able to append / modify the search paths, and can choose not to - // include the "modules" dir in the future. - let mut include_paths = include_paths; - include_paths.push(modules_dir); - - Self { - config_dir: config_dir.into(), - init_metta_path, - repl_config_metta_path, - metta_working_dir, - include_paths, - history_file: Some(config_dir.join("history.txt")), + Self { + repl_config_metta_path: Some(repl_config_metta_path), + history_file: Some(config_dir.join("history.txt")), + } + } else { + Self { + repl_config_metta_path: None, + history_file: None, + } } } - /// Returns the search paths, in order to search - /// - /// The metta_working_dir is always returned first - pub fn modules_search_paths<'a>(&'a self) -> impl Iterator + 'a { - [self.metta_working_dir.clone()].into_iter().chain( - self.include_paths.iter().cloned()) - } } /// Returns the MeTTa code to init the Repl's MeTTa params and set them to default values diff --git a/repl/src/main.rs b/repl/src/main.rs index 1c52c1faa..e567ca22d 100644 --- a/repl/src/main.rs +++ b/repl/src/main.rs @@ -9,10 +9,9 @@ use rustyline::{Cmd, CompletionType, Config, EditMode, Editor, KeyEvent, KeyCode use anyhow::Result; use clap::Parser; -use directories::ProjectDirs; use signal_hook::{consts::SIGINT, iterator::Signals}; -use hyperon::common::shared::Shared; +use hyperon::metta::Environment; mod metta_shim; use metta_shim::*; @@ -46,21 +45,34 @@ fn main() -> Result<()> { (None, &[] as &[PathBuf]) }; - //Config directory will be here: TODO: Document this in README. - // Linux: ~/.config/metta/ - // Windows: ~\AppData\Roaming\TrueAGI\metta\config\ - // Mac: ~/Library/Application Support/io.TrueAGI.metta/ - let repl_params = match ProjectDirs::from("io", "TrueAGI", "metta") { - Some(proj_dirs) => ReplParams::new(proj_dirs.config_dir(), cli_args.include_paths, primary_metta_file), + //If we have a metta_file, then the working dir is the parent of that file + //If we are running in interactive mode, it's the working dir at the time the repl is invoked + let metta_working_dir: PathBuf = match primary_metta_file { + Some(metta_file) => { + metta_file.parent().unwrap().into() + }, None => { - eprint!("Failed to initialize config!"); - ReplParams::default() + match std::env::current_dir() { + Ok(cwd) => cwd, + Err(_) => PathBuf::from("./").canonicalize().unwrap(), + } } }; - let repl_params = Shared::new(repl_params); + + //TODO_NOW, Pass the extra include paths into the environment creation, when we have + // the environment builder API. + // //Push the "modules" dir, as the last place to search after the paths specified on the cmd line + // //TODO: the config.metta file will be able to append / modify the search paths, and can choose not to + // // include the "modules" dir in the future. + // let mut include_paths = cli_args.include_paths; + // include_paths.push(modules_dir); + + //Init our runtime environment + Environment::init(Some(&metta_working_dir)); + let repl_params = ReplParams::new(); //Create our MeTTa runtime environment - let mut metta = MettaShim::new(repl_params.clone()); + let mut metta = MettaShim::new(); //Spawn a signal handler background thread, to deal with passing interrupts to the execution loop let mut signals = Signals::new(&[SIGINT])?; @@ -109,13 +121,16 @@ fn main() -> Result<()> { // To debug rustyline: // RUST_LOG=rustyline=debug cargo run --example example 2> debug.log -fn start_interactive_mode(repl_params: Shared, mut metta: MettaShim) -> rustyline::Result<()> { +fn start_interactive_mode(repl_params: ReplParams, mut metta: MettaShim) -> rustyline::Result<()> { //Run the built-in repl-init code metta.exec(&builtin_init_metta_code()); //Run the repl init file - metta.load_metta_module(repl_params.borrow().repl_config_metta_path.clone()); + if let Some(repl_config_metta_path) = &repl_params.repl_config_metta_path { + metta.load_metta_module(repl_config_metta_path.clone()); + } + let max_len = metta.get_config_int(CFG_HISTORY_MAX_LEN).unwrap_or_else(|| 500); //Init RustyLine @@ -141,7 +156,7 @@ fn start_interactive_mode(repl_params: Shared, mut metta: MettaShim) rl.bind_sequence(KeyEvent::ctrl('j'), EventHandler::Conditional(Box::new(EnterKeyHandler::new(rl.helper().unwrap().force_submit.clone())))); rl.bind_sequence(KeyEvent::alt('n'), Cmd::HistorySearchForward); rl.bind_sequence(KeyEvent::alt('p'), Cmd::HistorySearchBackward); - if let Some(history_path) = &repl_params.borrow().history_file { + if let Some(history_path) = &repl_params.history_file { if rl.load_history(history_path).is_err() { println!("No previous history found."); } @@ -184,7 +199,7 @@ fn start_interactive_mode(repl_params: Shared, mut metta: MettaShim) } } - if let Some(history_path) = &repl_params.borrow().history_file { + if let Some(history_path) = &repl_params.history_file { rl.append_history(history_path)? } diff --git a/repl/src/metta_shim.rs b/repl/src/metta_shim.rs index d14c89c0f..c6a78a07d 100644 --- a/repl/src/metta_shim.rs +++ b/repl/src/metta_shim.rs @@ -5,6 +5,7 @@ use hyperon::ExpressionAtom; use hyperon::Atom; use hyperon::space::*; use hyperon::space::grounding::GroundingSpace; +use hyperon::metta::Environment; use hyperon::metta::runner::{Metta, atom_is_error}; #[cfg(not(feature = "minimal"))] use hyperon::metta::runner::stdlib::register_rust_tokens; @@ -14,7 +15,6 @@ use hyperon::metta::text::Tokenizer; use hyperon::metta::text::SExprParser; use hyperon::common::shared::Shared; -use crate::ReplParams; use crate::SIGINT_RECEIVED_COUNT; /// MettaShim is responsible for **ALL** calls between the repl and MeTTa, and is in charge of keeping @@ -26,7 +26,6 @@ use crate::SIGINT_RECEIVED_COUNT; pub struct MettaShim { pub metta: Metta, pub result: Vec>, - _repl_params: Shared, //TODO: We'll likely want this back soon, but so I'm not un-plumbing it just yet } #[macro_export] @@ -59,15 +58,14 @@ impl Drop for MettaShim { impl MettaShim { - pub fn new(repl_params: Shared) -> Self { + pub fn new() -> Self { //Init the MeTTa interpreter let space = DynSpace::new(GroundingSpace::new()); let tokenizer = Shared::new(Tokenizer::new()); let mut new_shim = Self { - metta: Metta::from_space(space, tokenizer, repl_params.borrow().modules_search_paths().collect()), + metta: Metta::from_space(space, tokenizer, Environment::platform_env().modules_search_paths().map(|path| path.into()).collect()), result: vec![], - _repl_params: repl_params.clone(), }; //Init HyperonPy if the repl includes Python support @@ -92,7 +90,7 @@ impl MettaShim { //Add the extend-py! token, if we have Python support #[cfg(feature = "python")] { - let extendpy_atom = Atom::gnd(py_mod_loading::ImportPyOp{metta: new_shim.metta.clone(), repl_params: repl_params.clone()}); + let extendpy_atom = Atom::gnd(py_mod_loading::ImportPyOp{metta: new_shim.metta.clone()}); new_shim.metta.tokenizer().borrow_mut().register_token_with_regex_str("extend-py!", move |_| { extendpy_atom.clone() }); } @@ -101,8 +99,9 @@ impl MettaShim { new_shim.metta.tokenizer().borrow_mut().register_token_with_regex_str("extend-py!", move |_| { Atom::gnd(py_mod_err::ImportPyErr) }); //Run the init.metta file - let repl_params = repl_params.borrow(); - new_shim.load_metta_module(repl_params.init_metta_path.clone()); + if let Some(init_meta_file) = Environment::platform_env().initialization_metta_file_path() { + new_shim.load_metta_module(init_meta_file.into()); + } new_shim } @@ -259,8 +258,6 @@ mod py_mod_loading { use hyperon::matcher::MatchResultIter; use hyperon::metta::*; use hyperon::metta::runner::Metta; - use hyperon::common::shared::Shared; - use crate::ReplParams; /// Load the hyperon module, and get the "__version__" attribute pub fn get_hyperonpy_version() -> Result { @@ -288,7 +285,7 @@ mod py_mod_loading { } } - pub fn load_python_module_from_mod_or_file(repl_params: &ReplParams, metta: &Metta, module_name: &str) -> Result<(), String> { + pub fn load_python_module_from_mod_or_file(metta: &Metta, module_name: &str) -> Result<(), String> { // First, see if the module is already registered with Python match load_python_module(metta, module_name) { @@ -298,7 +295,7 @@ mod py_mod_loading { //Check each include directory in order, until we find the module we're looking for let file_name = PathBuf::from(module_name).with_extension("py"); let mut found_path = None; - for include_path in repl_params.modules_search_paths() { + for include_path in Environment::platform_env().modules_search_paths() { let path = include_path.join(&file_name); if path.exists() { found_path = Some(path); @@ -372,7 +369,6 @@ mod py_mod_loading { #[derive(Clone, PartialEq, Debug)] pub struct ImportPyOp { pub metta: Metta, - pub repl_params: Shared, } impl Display for ImportPyOp { @@ -393,7 +389,7 @@ mod py_mod_loading { .ok_or_else(arg_error)? .try_into().map_err(|_| arg_error())?; - match load_python_module_from_mod_or_file(&self.repl_params.borrow(), &self.metta, module_path_sym_atom.name()) { + match load_python_module_from_mod_or_file(&self.metta, module_path_sym_atom.name()) { Ok(()) => Ok(vec![]), Err(err) => Err(ExecError::from(err)), } From 00d62fc428de7d0df919d13cdc3118b5ae902d42 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Sat, 23 Sep 2023 17:48:02 +0800 Subject: [PATCH 02/28] Transitioning Environment config API to a "builder" style API --- lib/src/metta/environment.rs | 197 ++++++++++++++++++++--------------- lib/src/metta/mod.rs | 3 +- repl/src/config_params.rs | 2 +- repl/src/main.rs | 15 +-- repl/src/metta_shim.rs | 3 +- 5 files changed, 122 insertions(+), 98 deletions(-) diff --git a/lib/src/metta/environment.rs b/lib/src/metta/environment.rs index 9839b00f7..555cb6377 100644 --- a/lib/src/metta/environment.rs +++ b/lib/src/metta/environment.rs @@ -26,29 +26,7 @@ impl Environment { /// Returns a reference to the shared "platform" Environment pub fn platform_env() -> &'static Self { - PLATFORM_ENV.get_or_init(|| Self::new_with_defaults(None)) - } - - /// Initializes the shared "platform" Environment with with the OS-Specific platform configuration - /// - /// Config directory locations will be: - /// Linux: ~/.config/metta/ - /// Windows: ~\AppData\Roaming\TrueAGI\metta\config\ - /// Mac: ~/Library/Application Support/io.TrueAGI.metta/ - /// - /// TODO: Repeat this documentation somewhere more prominent, like the top-level README - /// - /// NOTE: This method will panic if the platform Environment has already been initialized - pub fn init(working_dir: Option<&Path>) { - PLATFORM_ENV.set(Self::new_with_defaults(working_dir)).expect("Fatal Error: Platform Environment already initialized"); - } - - /// Initializes the shared "platform" Environment with with the configuration stored in the `config_dir`. Will create - /// `config_dir` and its contents if it does not exist - /// - /// NOTE: This method will panic if the platform Environment has already been initialized - pub fn init_with_cfg_dir(working_dir: Option<&Path>, config_dir: &Path) { - PLATFORM_ENV.set(Self::new_with_config_dir(working_dir, config_dir)).expect("Fatal Error: Platform Environment already initialized"); + PLATFORM_ENV.get_or_init(|| EnvBuilder::new().build()) } /// Returns the Path to the config dir, in an OS-specific location @@ -69,80 +47,131 @@ impl Environment { .chain(self.extra_include_paths.iter().map(|path| path.borrow())) } - /// Returns a newly created Environment with the OS-specific platform defaults - /// - /// NOTE: Creating owned Environments is usually not necessary. It is usually sufficient to use the [platform_env]. - pub fn new_with_defaults(working_dir: Option<&Path>) -> Self { - match ProjectDirs::from("io", "TrueAGI", "metta") { - Some(proj_dirs) => { - Self::new_with_config_dir(working_dir, proj_dirs.config_dir()) - }, - None => { - eprint!("Failed to initialize config!"); - Self::new_without_config_dir(working_dir) - } + /// Private "default" function + fn new() -> Self { + Self { + config_dir: None, + init_metta_path: None, + working_dir: None, + extra_include_paths: vec![], } } +} - /// Returns a newly created Environment with the configuration stored in the `config_dir`. Will create - /// `config_dir` and its contents if it does not exist +pub struct EnvBuilder { + env: Environment, + no_cfg_dir: bool, +} + +impl EnvBuilder { + + /// Returns a new EnvBuilder, to set the parameters for the MeTTa Environment /// - /// NOTE: Creating owned Environments is usually not necessary. It is usually sufficient to use the [platform_env]. - pub fn new_with_config_dir(working_dir: Option<&Path>, config_dir: &Path) -> Self { - - //Create the modules dir inside the config dir, if it doesn't already exist. - // This will create the cfg_dir iteslf in the process - let modules_dir = config_dir.join("modules"); - std::fs::create_dir_all(&modules_dir).unwrap(); - - //Create the default init.metta file if they don't already exist - let init_metta_path = config_dir.join("init.metta"); - if !init_metta_path.exists() { - let mut file = fs::OpenOptions::new() - .create(true) - .write(true) - .open(&init_metta_path) - .expect(&format!("Error creating default init file at {init_metta_path:?}")); - file.write_all(&DEFAULT_INIT_METTA).unwrap(); + /// NOTE: Unless otherwise specified by calling either [set_no_config_dir] or [set_config_dir], the + /// [Environment] will be configured using the OS-Specific platform configuration files. + /// + /// Depending on the host OS, the config directory locations will be: + /// * Linux: ~/.config/metta/ + /// * Windows: ~\AppData\Roaming\TrueAGI\metta\config\ + /// * Mac: ~/Library/Application Support/io.TrueAGI.metta/ + /// + /// TODO: Repeat this documentation somewhere more prominent, like the top-level README + pub fn new() -> Self { + Self { + env: Environment::new(), + no_cfg_dir: false, } + } - //TODO_NOW, come back here and rethink what we do with include_paths. ie. where they are stored - // //Push the "modules" dir, as the last place to search after the paths specified on the cmd line - // //TODO: the config.metta file will be able to append / modify the search paths, and can choose not to - // // include the "modules" dir in the future. - // let mut include_paths = include_paths; - // include_paths.push(modules_dir); + /// Sets (or unsets) the working_dir for the environment + pub fn set_working_dir(mut self, working_dir: Option<&Path>) -> Self { + self.env.working_dir = working_dir.map(|dir| dir.into()); + self + } - Self { - config_dir: Some(config_dir.into()), - init_metta_path: Some(init_metta_path), - working_dir: working_dir.map(|dir| dir.into()), - extra_include_paths: vec![], //TODO_NOW, need a way to pass in extra include paths. This is the straw that pushes me over to a builder API + /// Sets the `config_dir` that the environment will load. A directory at the specified path will + /// be created its contents populated with default values, if one does not already exist + pub fn set_config_dir(mut self, config_dir: &Path) -> Self { + self.env.config_dir = Some(config_dir.into()); + if self.no_cfg_dir { + panic!("Fatal Error: set_config_dir is incompatible with set_no_config_dir"); } + self } - /// Returns a newly created Environment with no specialized configuration. This method will not touch any files. - /// - /// NOTE: Creating owned Environments is usually not necessary. It is usually sufficient to use the [platform_env]. - pub fn new_without_config_dir(working_dir: Option<&Path>) -> Self { - Self { - config_dir: None, - init_metta_path: None, - working_dir: working_dir.map(|dir| dir.into()), - extra_include_paths: vec![], + /// Configures the Environment not to load nor create any config files + pub fn set_no_config_dir(mut self) -> Self { + self.no_cfg_dir = true; + if self.env.config_dir.is_some() { + panic!("Fatal Error: set_config_dir is incompatible with set_no_config_dir"); } + self + } + + /// Adds additional include paths to search for MeTTa modules + /// + /// NOTE: The most recently added paths will have the highest search priority, save for the `working_dir`, + /// and paths returned first by the iterator will have higher priority within the same call to add_include_paths. + pub fn add_include_paths, I: IntoIterator>(mut self, paths: I) -> Self { + let mut additional_paths: Vec = paths.into_iter().map(|path| path.borrow().into()).collect(); + additional_paths.extend(self.env.extra_include_paths); + self.env.extra_include_paths = additional_paths; + self + } + + /// Initializes the shared platform Environment, accessible with [platform_env] + /// + /// NOTE: This method will panic if the platform Environment has already been initialized + pub fn init_platform_env(self) { + PLATFORM_ENV.set(self.build()).expect("Fatal Error: Platform Environment already initialized"); } -} -//TODO_NOW: Transition to a builder API -// pub struct EnvBuilder { -// env: Environment -// } + /// Returns a newly created Environment from the builder configuration + /// + /// NOTE: Creating owned Environments is usually not necessary. It is usually sufficient to use the [platform_env] method. + pub fn build(self) -> Environment { + + let mut env = self.env; + + if !self.no_cfg_dir { + if env.config_dir.is_none() { + match ProjectDirs::from("io", "TrueAGI", "metta") { + Some(proj_dirs) => { + env.config_dir = Some(proj_dirs.config_dir().into()); + }, + None => { + eprint!("Failed to initialize config with OS config directory!"); + } + } + } + } -// impl EnvBuilder { + if let Some(config_dir) = &env.config_dir { + + //Create the modules dir inside the config dir, if it doesn't already exist. + // This will create the cfg_dir iteslf in the process + let modules_dir = config_dir.join("modules"); + std::fs::create_dir_all(&modules_dir).unwrap(); + + //Push the "modules" dir, as the last place to search after the other paths that were specified + //TODO: the config.metta file will be able to append / modify the search paths, and can choose not to + // include the "modules" dir in the future. + env.extra_include_paths.push(modules_dir); + + //Create the default init.metta file if it doesn't already exist + let init_metta_path = config_dir.join("init.metta"); + if !init_metta_path.exists() { + let mut file = fs::OpenOptions::new() + .create(true) + .write(true) + .open(&init_metta_path) + .expect(&format!("Error creating default init file at {init_metta_path:?}")); + file.write_all(&DEFAULT_INIT_METTA).unwrap(); + } + env.init_metta_path = Some(init_metta_path); + } -// /// Returns a new EnvBuilder, to set the parameters for the MeTTa Environment -// pub fn new() -> Self { + env + } -// } -// } \ No newline at end of file +} diff --git a/lib/src/metta/mod.rs b/lib/src/metta/mod.rs index c2aafdd77..3904bf532 100644 --- a/lib/src/metta/mod.rs +++ b/lib/src/metta/mod.rs @@ -6,8 +6,7 @@ pub mod interpreter; pub mod interpreter2; pub mod types; pub mod runner; -mod environment; -pub use environment::Environment; +pub mod environment; use text::{SExprParser, Tokenizer}; use regex::Regex; diff --git a/repl/src/config_params.rs b/repl/src/config_params.rs index dbc4ddc75..2989eaab0 100644 --- a/repl/src/config_params.rs +++ b/repl/src/config_params.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use std::io::Write; use std::fs; -use hyperon::metta::Environment; +use hyperon::metta::environment::Environment; const DEFAULT_REPL_METTA: &[u8] = include_bytes!("repl.default.metta"); diff --git a/repl/src/main.rs b/repl/src/main.rs index e567ca22d..cd3d03250 100644 --- a/repl/src/main.rs +++ b/repl/src/main.rs @@ -11,7 +11,7 @@ use anyhow::Result; use clap::Parser; use signal_hook::{consts::SIGINT, iterator::Signals}; -use hyperon::metta::Environment; +use hyperon::metta::environment::EnvBuilder; mod metta_shim; use metta_shim::*; @@ -59,16 +59,11 @@ fn main() -> Result<()> { } }; - //TODO_NOW, Pass the extra include paths into the environment creation, when we have - // the environment builder API. - // //Push the "modules" dir, as the last place to search after the paths specified on the cmd line - // //TODO: the config.metta file will be able to append / modify the search paths, and can choose not to - // // include the "modules" dir in the future. - // let mut include_paths = cli_args.include_paths; - // include_paths.push(modules_dir); - //Init our runtime environment - Environment::init(Some(&metta_working_dir)); + EnvBuilder::new() + .set_working_dir(Some(&metta_working_dir)) + .add_include_paths(cli_args.include_paths) + .init_platform_env(); let repl_params = ReplParams::new(); //Create our MeTTa runtime environment diff --git a/repl/src/metta_shim.rs b/repl/src/metta_shim.rs index c6a78a07d..d284fbc1e 100644 --- a/repl/src/metta_shim.rs +++ b/repl/src/metta_shim.rs @@ -5,7 +5,7 @@ use hyperon::ExpressionAtom; use hyperon::Atom; use hyperon::space::*; use hyperon::space::grounding::GroundingSpace; -use hyperon::metta::Environment; +use hyperon::metta::environment::Environment; use hyperon::metta::runner::{Metta, atom_is_error}; #[cfg(not(feature = "minimal"))] use hyperon::metta::runner::stdlib::register_rust_tokens; @@ -257,6 +257,7 @@ mod py_mod_loading { use hyperon::atom::{Grounded, ExecError, match_by_equality}; use hyperon::matcher::MatchResultIter; use hyperon::metta::*; + use hyperon::metta::environment::Environment; use hyperon::metta::runner::Metta; /// Load the hyperon module, and get the "__version__" attribute From c05e12e8f2fc837c6914090bea9a61d6926e2888 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Sat, 23 Sep 2023 21:40:43 +0800 Subject: [PATCH 03/28] Updating C & Python runner init code to use Environment by default --- c/src/metta.rs | 24 ++++++++++++++++++++---- c/tests/check_space.c | 8 +++----- lib/src/metta/environment.rs | 10 ++++++++++ lib/src/metta/runner/mod.rs | 26 ++++++++++++++++++-------- python/hyperon/runner.py | 4 ++-- python/hyperonpy.cpp | 4 ++-- repl/src/metta_shim.rs | 8 +------- 7 files changed, 56 insertions(+), 28 deletions(-) diff --git a/c/src/metta.rs b/c/src/metta.rs index 186b07018..1f915779f 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -468,22 +468,38 @@ impl metta_t { } } -/// @brief Creates a new MeTTa Interpreter +/// @brief Creates a new top-level MeTTa Interpreter +/// @ingroup interpreter_group +/// @return A `metta_t` handle to the newly created Interpreter +/// @note The caller must take ownership responsibility for the returned `metta_t`, and free it with `metta_free()` +/// +#[no_mangle] +pub extern "C" fn metta_new() -> metta_t { + let metta = Metta::new_top_level_runner(); + metta.into() +} + +/// @brief Creates a new MeTTa Interpreter with a provided Space and Tokenizer /// @ingroup interpreter_group /// @param[in] space A pointer to a handle for the Space for use by the Interpreter /// @param[in] tokenizer A pointer to a handle for the Tokenizer for use by the Interpreter -/// @param[in] cwd A C-style string specifying a path to a working directory, to search for modules to load /// @return A `metta_t` handle to the newly created Interpreter /// @note The caller must take ownership responsibility for the returned `metta_t`, and free it with `metta_free()` /// #[no_mangle] -pub extern "C" fn metta_new(space: *mut space_t, tokenizer: *mut tokenizer_t, cwd: *const c_char) -> metta_t { +pub extern "C" fn metta_new_with_space(space: *mut space_t, tokenizer: *mut tokenizer_t) -> metta_t { let dyn_space = unsafe{ &*space }.borrow(); let tokenizer = unsafe{ &*tokenizer }.clone_handle(); - let metta = Metta::from_space(dyn_space.clone(), tokenizer, vec![PathBuf::from(cstr_as_str(cwd))]); + let metta = Metta::new_with_space(dyn_space.clone(), tokenizer); metta.into() } +//TODO_NOW... Implement path setters for environment like this +// /// @param[in] cwd A C-style string specifying a path to a working directory, to search for modules to load +// func( cwd: *const c_char) +// vec![PathBuf::from(cstr_as_str(cwd))] + + /// @brief Frees a `metta_t` handle /// @ingroup interpreter_group /// @param[in] metta The handle to free diff --git a/c/tests/check_space.c b/c/tests/check_space.c index a808f2dd7..4724c936c 100644 --- a/c/tests/check_space.c +++ b/c/tests/check_space.c @@ -228,11 +228,10 @@ START_TEST (test_space_nested_in_atom) space_add(&nested, expr(atom_sym("A"), atom_sym("B"), atom_ref_null())); atom_t space_atom = atom_gnd_for_space(&nested); - space_t runner_space = space_new_grounding_space(); - tokenizer_t tokenizer = tokenizer_new(); - metta_t runner = metta_new(&runner_space, &tokenizer, "."); + metta_t runner = metta_new(); metta_load_module(&runner, "stdlib"); + tokenizer_t tokenizer = metta_tokenizer(&runner); tokenizer_register_token(&tokenizer, "nested", &TOKEN_API_CLONE_ATOM, &space_atom); sexpr_parser_t parser = sexpr_parser_new("!(match nested (A $x) $x)"); @@ -247,9 +246,8 @@ START_TEST (test_space_nested_in_atom) atom_vec_free(results); sexpr_parser_free(parser); - metta_free(runner); tokenizer_free(tokenizer); - space_free(runner_space); + metta_free(runner); atom_free(space_atom); space_free(nested); diff --git a/lib/src/metta/environment.rs b/lib/src/metta/environment.rs index 555cb6377..a2dfec959 100644 --- a/lib/src/metta/environment.rs +++ b/lib/src/metta/environment.rs @@ -171,6 +171,16 @@ impl EnvBuilder { env.init_metta_path = Some(init_metta_path); } + //TODO: This line below is a stop-gap to match old behavior + //As discussed with Vitaly, searching the current working dir can be a potential security hole + // However, that is mitigated by "." being the last directory searched. + // Anyway, the issue is that the Metta::import_file method in runner.py relies on using the + // same runner but being able to change the search path by changing the working dir. + // A better fix is to fork a "child runner" with access to the same space and tokenizer, + // but an updated search path. This is really hard to implement currently given the ImportOp + // actually owns a reference to the runner it's associated with. However this must be fixed soon. + env.extra_include_paths.push(".".into()); + env } diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index 73f12db54..cac339ddb 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -10,6 +10,7 @@ use std::rc::Rc; use std::path::PathBuf; use std::collections::HashMap; +use metta::environment::Environment; #[cfg(not(feature = "minimal"))] pub mod stdlib; @@ -64,20 +65,26 @@ pub struct RunnerState<'a> { } impl Metta { - pub fn new(space: DynSpace, tokenizer: Shared) -> Self { - Metta::from_space(space, tokenizer, vec![PathBuf::from(".")]) + /// A 1-liner to get a MeTTa interpreter using the default configuration + //TODO, see comment on `new_metta_rust`. That function should merge into this one + pub fn new_top_level_runner() -> Self { + let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), + Shared::new(Tokenizer::new())); + metta } - pub fn from_space(space: DynSpace, tokenizer: Shared, search_paths: Vec) -> Self { + /// Returns a new MeTTa interpreter, using the provided Space and Tokenizer + pub fn new_with_space(space: DynSpace, tokenizer: Shared) -> Self { let settings = Shared::new(HashMap::new()); let modules = Shared::new(HashMap::new()); - let contents = MettaContents{ space, tokenizer, settings, modules, search_paths }; + let contents = MettaContents{ space, tokenizer, settings, modules, search_paths: Environment::platform_env().modules_search_paths().map(|path| path.into()).collect() }; let metta = Self(Rc::new(contents)); register_runner_tokens(&metta); register_common_tokens(&metta); metta } + /// Returns a new MeTTa interpreter intended for use loading MeTTa modules during import fn new_loading_runner(metta: &Metta, path: PathBuf) -> Self { let space = DynSpace::new(GroundingSpace::new()); let tokenizer = metta.tokenizer().clone_inner(); @@ -301,8 +308,11 @@ impl<'a> RunnerState<'a> { } } +//TODO: this function should be totally subsumed into Metta::new_top_level_runner(), but +// first we have to be able to load the "rust" stdlib before the python stdlib, which requires TODO_NOW-lookup-issue-number +// to be fixed, which in-turn requires value-bridging pub fn new_metta_rust() -> Metta { - let metta = Metta::new(DynSpace::new(GroundingSpace::new()), + let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new())); register_rust_tokens(&metta); metta.load_module(PathBuf::from("stdlib")).expect("Could not load stdlib"); @@ -339,7 +349,7 @@ mod tests { (foo b) "; - let metta = Metta::new(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new())); + let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new())); metta.set_setting("type-check".into(), sym!("auto")); let result = metta.run(&mut SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); @@ -353,7 +363,7 @@ mod tests { !(foo b) "; - let metta = Metta::new(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new())); + let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new())); metta.set_setting("type-check".into(), sym!("auto")); let result = metta.run(&mut SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); @@ -408,7 +418,7 @@ mod tests { !(foo a) "; - let metta = Metta::new(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new())); + let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new())); metta.set_setting("type-check".into(), sym!("auto")); let result = metta.run(&mut SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); diff --git a/python/hyperon/runner.py b/python/hyperon/runner.py index da428df98..28dc29661 100644 --- a/python/hyperon/runner.py +++ b/python/hyperon/runner.py @@ -7,14 +7,14 @@ class MeTTa: """This class contains the MeTTa program execution utilities""" - def __init__(self, space = None, cwd = ".", cmetta = None): + def __init__(self, space = None, cmetta = None): if cmetta is not None: self.cmetta = cmetta else: if space is None: space = GroundingSpaceRef() tokenizer = Tokenizer() - self.cmetta = hp.metta_new(space.cspace, tokenizer.ctokenizer, cwd) + self.cmetta = hp.metta_new(space.cspace, tokenizer.ctokenizer) self.load_py_module("hyperon.stdlib") hp.metta_load_module(self.cmetta, "stdlib") self.register_atom('extend-py!', diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index 392214288..a09e38217 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -697,8 +697,8 @@ PYBIND11_MODULE(hyperonpy, m) { ADD_SYMBOL(VOID, "Void"); py::class_(m, "CMetta").def(py::init(&cmetta_from_inner_ptr_as_int)); - m.def("metta_new", [](CSpace space, CTokenizer tokenizer, char const* cwd) { - return CMetta(metta_new(space.ptr(), tokenizer.ptr(), cwd)); + m.def("metta_new", [](CSpace space, CTokenizer tokenizer) { + return CMetta(metta_new_with_space(space.ptr(), tokenizer.ptr())); }, "New MeTTa interpreter instance"); m.def("metta_free", [](CMetta metta) { metta_free(metta.obj); }, "Free MeTTa interpreter"); m.def("metta_space", [](CMetta metta) { return CSpace(metta_space(metta.ptr())); }, "Get space of MeTTa interpreter"); diff --git a/repl/src/metta_shim.rs b/repl/src/metta_shim.rs index d284fbc1e..49ceac1c3 100644 --- a/repl/src/metta_shim.rs +++ b/repl/src/metta_shim.rs @@ -3,17 +3,13 @@ use std::path::PathBuf; use hyperon::ExpressionAtom; use hyperon::Atom; -use hyperon::space::*; -use hyperon::space::grounding::GroundingSpace; use hyperon::metta::environment::Environment; use hyperon::metta::runner::{Metta, atom_is_error}; #[cfg(not(feature = "minimal"))] use hyperon::metta::runner::stdlib::register_rust_tokens; #[cfg(feature = "minimal")] use hyperon::metta::runner::stdlib2::register_rust_tokens; -use hyperon::metta::text::Tokenizer; use hyperon::metta::text::SExprParser; -use hyperon::common::shared::Shared; use crate::SIGINT_RECEIVED_COUNT; @@ -61,10 +57,8 @@ impl MettaShim { pub fn new() -> Self { //Init the MeTTa interpreter - let space = DynSpace::new(GroundingSpace::new()); - let tokenizer = Shared::new(Tokenizer::new()); let mut new_shim = Self { - metta: Metta::from_space(space, tokenizer, Environment::platform_env().modules_search_paths().map(|path| path.into()).collect()), + metta: Metta::new_top_level_runner(), result: vec![], }; From 95cd2ce32c54a26e8ad05dcf2f99ac6b2b5a6f0a Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Sun, 24 Sep 2023 15:39:48 +0400 Subject: [PATCH 04/28] Adding C API for platform environment config --- c/doc/mainpage.md | 7 ++ c/src/metta.rs | 144 +++++++++++++++++++++++++++++++++-- lib/src/metta/environment.rs | 3 + 3 files changed, 148 insertions(+), 6 deletions(-) diff --git a/c/doc/mainpage.md b/c/doc/mainpage.md index a3ecfcf77..dbc55b89b 100644 --- a/c/doc/mainpage.md +++ b/c/doc/mainpage.md @@ -127,6 +127,13 @@ More complete documentation on the MeTTa language, type system, and the MeTTa st This Interface includes the types and functions to instantiate a MeTTa interpreter and step through MeTTa code. +[//]: # (Platform Environment Interface) + +@defgroup environment_group Platform Environment Interface +@brief Configuration and settings shared by MeTTa runners + +This interface allows configuration of shared properties for MeTTa interpreters + [//]: # (Misc. Interfaces) @defgroup misc_group Misc Interfaces diff --git a/c/src/metta.rs b/c/src/metta.rs index 1f915779f..19f3fe9ce 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -4,12 +4,15 @@ use hyperon::metta::text::*; use hyperon::metta::interpreter; use hyperon::metta::interpreter::InterpreterState; use hyperon::metta::runner::Metta; +use hyperon::metta::environment::{Environment, EnvBuilder}; use hyperon::rust_type_atom; use crate::util::*; use crate::atom::*; use crate::space::*; +use core::borrow::Borrow; +use std::sync::Mutex; use std::os::raw::*; use regex::Regex; use std::path::PathBuf; @@ -494,12 +497,6 @@ pub extern "C" fn metta_new_with_space(space: *mut space_t, tokenizer: *mut toke metta.into() } -//TODO_NOW... Implement path setters for environment like this -// /// @param[in] cwd A C-style string specifying a path to a working directory, to search for modules to load -// func( cwd: *const c_char) -// vec![PathBuf::from(cstr_as_str(cwd))] - - /// @brief Frees a `metta_t` handle /// @ingroup interpreter_group /// @param[in] metta The handle to free @@ -585,3 +582,138 @@ pub extern "C" fn metta_load_module(metta: *mut metta_t, name: *const c_char) { metta.load_module(PathBuf::from(cstr_as_str(name))) .expect("Returning errors from C API is not implemented yet"); } + +// =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- +// Environment Interface +// =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + +/// @brief Renders the config_dir path from the platform environment into a text buffer +/// @ingroup environment_group +/// @param[out] buf A buffer into which the text will be written +/// @param[in] buf_len The maximum allocated size of `buf` +/// @return The length of the path string, minus the string terminator character. If +/// `return_value > buf_len + 1`, then the text was not fully written and this function should be +/// called again with a larger buffer. This function will return 0 if there is no config_dir. +/// +#[no_mangle] +pub extern "C" fn environment_config_dir(buf: *mut c_char, buf_len: usize) -> usize { + match Environment::platform_env().config_dir() { + Some(path) => write_into_buf(path.display(), buf, buf_len), + None => 0 + } +} + +//QUESTION / TODO?: It would be nice to wrap the environment_init all in a va_args call, unfortunately cbindgen +// doesn't work with va_args and it's not worth requiring an additional build step for this. +// Also, C doesn't have a natural way to express key-value pairs or named args, so we'd need to invent +// one, which is probably more trouble than it's worth. So I'll leave the stateful builder API the +// way it is for now, but create a nicer API for Python using kwargs + +static CURRENT_ENV_BUILDER: Mutex<(EnvInitState, Option)> = std::sync::Mutex::new((EnvInitState::Uninitialized, None)); + +#[derive(Default, PartialEq, Eq)] +enum EnvInitState { + #[default] + Uninitialized, + InProcess, + Finished +} + +fn take_current_builder(expected_state: EnvInitState) -> EnvBuilder { + let mut builder_state = CURRENT_ENV_BUILDER.lock().unwrap(); + if builder_state.0 != expected_state { + panic!("Fatal Error: no active initialization in process. Call environment_init_start first"); + } + core::mem::take(&mut builder_state.1).unwrap() +} + +fn replace_current_builder(new_state: EnvInitState, builder: Option) { + let mut builder_state = CURRENT_ENV_BUILDER.lock().unwrap(); + builder_state.0 = new_state; + builder_state.1 = builder; +} + +/// @brief Begins the initialization of the platform environment +/// @note environment_init_finish must be called after environment initialization is finished +/// @ingroup environment_group +/// +#[no_mangle] +pub extern "C" fn environment_init_start() { + let mut builder_state = CURRENT_ENV_BUILDER.lock().unwrap(); + if builder_state.0 != EnvInitState::Uninitialized { + panic!("Fatal Error: environment_init_start must be called only once"); + } + builder_state.0 = EnvInitState::InProcess; + builder_state.1 = Some(EnvBuilder::new()); +} + +/// @brief Finishes initialization of the platform environment +/// @ingroup environment_group +/// +#[no_mangle] +pub extern "C" fn environment_init_finish() { + let builder = take_current_builder(EnvInitState::InProcess); + builder.init_platform_env(); + replace_current_builder(EnvInitState::Finished, None); +} + +/// @brief Sets the working directory for the platform environment +/// @ingroup environment_group +/// @param[in] path A C-style string specifying a path to a working directory, to search for modules to load. +/// Passing `NULL` will unset the working directory +/// @note This working directory is not required to be the same as the process working directory, and +/// it will not change as the process' working directory is changed +/// +#[no_mangle] +pub extern "C" fn environment_init_set_working_dir(path: *const c_char) { + let builder = take_current_builder(EnvInitState::InProcess); + let builder = if path.is_null() { + builder.set_working_dir(None) + } else { + builder.set_working_dir(Some(&PathBuf::from(cstr_as_str(path)))) + }; + replace_current_builder(EnvInitState::InProcess, Some(builder)); +} + +/// @brief Sets the config directory for the platform environment. A directory at the specified path +/// will be created its contents populated with default values, if one does not already exist +/// @ingroup environment_group +/// @param[in] path A C-style string specifying a path to a working directory, to search for modules to load +/// +#[no_mangle] +pub extern "C" fn environment_init_set_config_dir(path: *const c_char) { + let builder = take_current_builder(EnvInitState::InProcess); + let builder = if path.is_null() { + panic!("Fatal Error: path cannot be NULL"); + } else { + builder.set_config_dir(&PathBuf::from(cstr_as_str(path))) + }; + replace_current_builder(EnvInitState::InProcess, Some(builder)); +} + +/// @brief Configures the platform environment so that no config directory will be read nor created +/// @ingroup environment_group +/// +#[no_mangle] +pub extern "C" fn environment_init_disable_config_dir() { + let builder = take_current_builder(EnvInitState::InProcess); + let builder = builder.set_no_config_dir(); + replace_current_builder(EnvInitState::InProcess, Some(builder)); +} + +/// @brief Adds a config directory to search for imports. The most recently added paths will be searched +/// first, continuing in inverse order +/// @ingroup environment_group +/// @param[in] path A C-style string specifying a path to a working directory, to search for modules to load +/// +#[no_mangle] +pub extern "C" fn environment_init_add_include_path(path: *const c_char) { + let builder = take_current_builder(EnvInitState::InProcess); + let builder = if path.is_null() { + panic!("Fatal Error: path cannot be NULL"); + } else { + builder.add_include_paths(vec![PathBuf::from(cstr_as_str(path).borrow())]) + }; + replace_current_builder(EnvInitState::InProcess, Some(builder)); +} + diff --git a/lib/src/metta/environment.rs b/lib/src/metta/environment.rs index a2dfec959..2e500f185 100644 --- a/lib/src/metta/environment.rs +++ b/lib/src/metta/environment.rs @@ -58,6 +58,9 @@ impl Environment { } } +/// Used to customize the [Environment] configuration +/// +/// NOTE: It is not necessary to use the EnvBuilder if the default environment is acceptable pub struct EnvBuilder { env: Environment, no_cfg_dir: bool, From 24a0d6338d53438abc1e9cf129fd02a96f6983c9 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Sun, 24 Sep 2023 16:54:24 +0400 Subject: [PATCH 05/28] Exposing environment configuration as a Python API --- python/hyperon/__init__.py | 2 +- python/hyperon/runner.py | 20 ++++++++++++++++++++ python/hyperonpy.cpp | 25 +++++++++++++++++++++++++ python/tests/CMakeLists.txt | 1 + 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/python/hyperon/__init__.py b/python/hyperon/__init__.py index 0949e2e42..8b01c6daf 100644 --- a/python/hyperon/__init__.py +++ b/python/hyperon/__init__.py @@ -1,6 +1,6 @@ from .atoms import * from .base import * -from .runner import MeTTa +from .runner import * def _version(dist_name): try: diff --git a/python/hyperon/runner.py b/python/hyperon/runner.py index 28dc29661..20a3e96a3 100644 --- a/python/hyperon/runner.py +++ b/python/hyperon/runner.py @@ -91,3 +91,23 @@ def run(self, program, flat=False): return [Atom._from_catom(catom) for result in results for catom in result] else: return [[Atom._from_catom(catom) for catom in result] for result in results] + +class Environment: + """This class contains the API for shared platform configuration""" + + def config_dir(): + """Returns the config dir in the platform environment""" + return hp.environment_config_dir() + def init_platform_env(working_dir = None, config_dir = None, disable_config = False, include_paths = []): + """Initialize the platform environment with the supplied args""" + hp.environment_init_start() + if (working_dir is not None): + hp.environment_init_set_working_dir(working_dir) + if (config_dir is not None): + hp.environment_init_set_config_dir(config_dir) + if (disable_config): + hp.environment_init_disable_config_dir() + for path in reversed(include_paths): + hp.environment_init_add_include_path(path) + hp.environment_init_finish() + diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index a09e38217..923e0fb78 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -63,6 +63,22 @@ std::string func_to_string(write_to_buf_func_t func, void* arg) { } } +// Similar to func_to_string, but for functions that don't take any args +typedef size_t (*write_to_buf_no_arg_func_t)(char*, size_t); +std::string func_to_string_no_arg(write_to_buf_no_arg_func_t func) { + //First try with a 1K stack buffer, because that will work in the vast majority of cases + char dst_buf[1024]; + size_t len = func(dst_buf, 1024); + if (len < 1024) { + return std::string(dst_buf); + } else { + char* data = new char[len+1]; + func(data, len+1); + std::string new_string = std::string(data); + return new_string; + } +} + static void copy_atoms(const atom_vec_t* atoms, void* context) { py::list* list = static_cast(context); for (size_t i = 0; i < atom_vec_len(atoms); ++i) { @@ -718,6 +734,15 @@ PYBIND11_MODULE(hyperonpy, m) { metta_load_module(metta.ptr(), text.c_str()); }, "Load MeTTa module"); + m.def("environment_config_dir", []() { + return func_to_string_no_arg((write_to_buf_no_arg_func_t)&environment_config_dir); + }, "Return the config_dir for the platform environment"); + m.def("environment_init_start", []() { environment_init_start(); }, "Begin initialization of the platform environment"); + m.def("environment_init_finish", []() { environment_init_finish(); }, "Finish initialization of the platform environment"); + m.def("environment_init_set_working_dir", [](std::string path) { environment_init_set_working_dir(path.c_str()); }, "Sets the working dir in the platform environment"); + m.def("environment_init_set_config_dir", [](std::string path) { environment_init_set_config_dir(path.c_str()); }, "Sets the config dir in the platform environment"); + m.def("environment_init_disable_config_dir", []() { environment_init_disable_config_dir(); }, "Disables the config dir in the platform environment"); + m.def("environment_init_add_include_path", [](std::string path) { environment_init_add_include_path(path.c_str()); }, "Adds an include path to the platform environment"); } __attribute__((constructor)) diff --git a/python/tests/CMakeLists.txt b/python/tests/CMakeLists.txt index 01aea69ff..08a06b805 100644 --- a/python/tests/CMakeLists.txt +++ b/python/tests/CMakeLists.txt @@ -23,3 +23,4 @@ ADD_TESTS("test_minelogy.py") ADD_TESTS("test_pln_tv.py") ADD_TESTS("test_stdlib.py") ADD_TESTS("test_run_metta.py") +ADD_TESTS("test_environment.py") From 6d9d06e308f2feb388d2aef76cbe47571bc5272a Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Sun, 24 Sep 2023 16:55:12 +0400 Subject: [PATCH 06/28] Oops forgot to git add file --- python/tests/test_environment.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 python/tests/test_environment.py diff --git a/python/tests/test_environment.py b/python/tests/test_environment.py new file mode 100644 index 000000000..f728d604c --- /dev/null +++ b/python/tests/test_environment.py @@ -0,0 +1,12 @@ +import unittest + +from hyperon import Environment + +class HyperonTestCase(unittest.TestCase): + + def __init__(self, methodName): + super().__init__(methodName) + + def testEnvironment(self): + Environment.init_platform_env(config_dir = "/tmp/test_dir") + self.assertEqual(Environment.config_dir(), "/tmp/test_dir") From adee13ec290f8e6f31eac3ec40ce847f6191b44a Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Sun, 24 Sep 2023 23:05:25 +0400 Subject: [PATCH 07/28] Exposing C Api for parsing into syntax nodes --- c/src/metta.rs | 155 +++++++++++++++++++++++++++++++++++ c/tests/check_sexpr_parser.c | 39 +++++++++ lib/src/metta/text.rs | 6 +- 3 files changed, 197 insertions(+), 3 deletions(-) diff --git a/c/src/metta.rs b/c/src/metta.rs index 19f3fe9ce..e0248507e 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -214,6 +214,160 @@ pub extern "C" fn sexpr_parser_parse( parser.parse(tokenizer).unwrap().into() } +/// @brief Represents a component in a syntax tree created by parsing MeTTa code +/// @ingroup tokenizer_and_parser_group +/// @note `syntax_node_t` objects must be freed with `syntax_node_free()` +/// +#[repr(C)] +pub struct syntax_node_t { + /// Internal. Should not be accessed directly + node: *mut RustSyntaxNode, +} + +struct RustSyntaxNode(SyntaxNode); + +impl From for syntax_node_t { + fn from(node: SyntaxNode) -> Self { + Self{ node: Box::into_raw(Box::new(RustSyntaxNode(node))) } + } +} + +impl syntax_node_t { + fn into_inner(self) -> SyntaxNode { + unsafe{ (*Box::from_raw(self.node)).0 } + } + fn borrow(&self) -> &SyntaxNode { + &unsafe{ &*(&*self).node }.0 + } +} + +/// @brief The type of language construct respresented by a syntax_node_t +/// @ingroup tokenizer_and_parser_group +/// +#[repr(C)] +pub enum syntax_node_type_t { + /// @brief A Comment, beginning with a ';' character + COMMENT, + /// @brief A variable. A symbol immediately preceded by a '$' sigil + VARIABLE_TOKEN, + /// @brief A String Literal. All text between non-escaped '"' (double quote) characters + STRING_TOKEN, + /// @brief Word Token. Any other whitespace-delimited token that isn't a VARIABLE_TOKEN or STRING_TOKEN + WORD_TOKEN, + /// @brief Open Parenthesis. A non-escaped '(' character indicating the beginning of an expression + OPEN_PAREN, + /// @brief Close Parenthesis. A non-escaped ')' character indicating the end of an expression + CLOSE_PAREN, + /// @brief Whitespace. One or more whitespace chars + WHITESPACE, + /// @brief Leftover Text that remains unparsed after a parse error has occurred + LEFTOVER_TEXT, + /// @brief A Group of nodes between an `OPEN_PAREN` and a matching `CLOSE_PAREN` + EXPRESSION_GROUP, + /// @brief A Group of nodes that cannot be combined into a coherent atom due to a parse error, + /// even if some of the individual nodes could represent valid atoms + ERROR_GROUP, +} + +impl From for syntax_node_type_t { + fn from(node_type: SyntaxNodeType) -> Self { + match node_type { + SyntaxNodeType::Comment => Self::COMMENT, + SyntaxNodeType::VariableToken => Self::VARIABLE_TOKEN, + SyntaxNodeType::StringToken => Self::STRING_TOKEN, + SyntaxNodeType::WordToken => Self::WORD_TOKEN, + SyntaxNodeType::OpenParen => Self::OPEN_PAREN, + SyntaxNodeType::CloseParen => Self::CLOSE_PAREN, + SyntaxNodeType::Whitespace => Self::WHITESPACE, + SyntaxNodeType::LeftoverText => Self::LEFTOVER_TEXT, + SyntaxNodeType::ExpressionGroup => Self::EXPRESSION_GROUP, + SyntaxNodeType::ErrorGroup => Self::ERROR_GROUP, + } + } +} + +/// @brief Function signature for a callback providing access to a `syntax_node_t` +/// @ingroup tokenizer_and_parser_group +/// @param[in] node The `syntax_node_t` being provided. This node should not be modified or freed by the callback. +/// @param[in] context The context state pointer initially passed to the upstream function initiating the callback. +/// +pub type c_syntax_node_callback_t = extern "C" fn(node: *const syntax_node_t, context: *mut c_void); + +/// @brief Parses the text associated with an `sexpr_parser_t`, and creates a syntax tree +/// @ingroup tokenizer_and_parser_group +/// @param[in] parser A pointer to the Parser, which is associated with the text to parse +/// @return The new `syntax_node_t` representing the root of the parsed tree +/// @note The caller must take ownership responsibility for the returned `syntax_node_t`, and ultimately free +/// it with `syntax_node_free()` +/// +#[no_mangle] +pub extern "C" fn sexpr_parser_parse_to_syntax_tree(parser: *mut sexpr_parser_t) -> syntax_node_t +{ + let parser = unsafe{ &*parser }.borrow_inner(); + parser.parse_to_syntax_tree().unwrap().into() +} + +/// @brief Frees a syntax_node_t +/// @ingroup tokenizer_and_parser_group +/// @param[in] node The `sexpr_parser_t` handle to free +/// +#[no_mangle] +pub extern "C" fn syntax_node_free(node: syntax_node_t) { + let node = node.into_inner(); + drop(node); +} + +/// @brief Performs a depth-first iteration of all child syntax nodes within a syntax tree +/// @ingroup tokenizer_and_parser_group +/// @param[in] node A pointer to the top-level `syntax_node_t` representing the syntax tree +/// @param[in] callback A function that will be called to provide a vector of all type atoms associated with the `atom` argument atom +/// @param[in] context A pointer to a caller-defined structure to facilitate communication with the `callback` function +/// +#[no_mangle] +pub extern "C" fn syntax_node_iterate(node: *const syntax_node_t, + callback: c_syntax_node_callback_t, context: *mut c_void) { + let node = unsafe{ &*node }.borrow(); + node.visit_depth_first(|node| { + let node = syntax_node_t{node: (node as *const SyntaxNode).cast_mut().cast()}; + callback(&node, context); + }); +} + +/// @brief Returns the type of a `syntax_node_t` +/// @ingroup tokenizer_and_parser_group +/// @param[in] node A pointer to the `syntax_node_t` +/// @return The `syntax_node_type_t` representing the type of the syntax node +/// +#[no_mangle] +pub extern "C" fn syntax_node_type(node: *const syntax_node_t) -> syntax_node_type_t { + let node = unsafe{ &*node }.borrow(); + node.node_type.into() +} + +/// @brief Returns `true` if a syntax node is a leaf (has no children) and `false` otherwise +/// @ingroup tokenizer_and_parser_group +/// @param[in] node A pointer to the `syntax_node_t` +/// @return The boolean value indicating if the node is a leaf +/// +#[no_mangle] +pub extern "C" fn syntax_node_is_leaf(node: *const syntax_node_t) -> bool { + let node = unsafe{ &*node }.borrow(); + node.node_type.is_leaf() +} + +/// @brief Returns the beginning and end positions in the parsed source of the text represented by the syntax node +/// @ingroup tokenizer_and_parser_group +/// @param[in] node A pointer to the `syntax_node_t` +/// @param[out] range_start A pointer to a value, into which the starting offset of the range will be written +/// @param[out] range_end A pointer to a value, into which the ending offset of the range will be written +/// +#[no_mangle] +pub extern "C" fn syntax_node_src_range(node: *const syntax_node_t, range_start: *mut usize, range_end: *mut usize) { + let node = unsafe{ &*node }.borrow(); + unsafe{ *range_start = node.src_range.start; } + unsafe{ *range_end = node.src_range.end; } +} + // =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- // MeTTa Language and Types // =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- @@ -609,6 +763,7 @@ pub extern "C" fn environment_config_dir(buf: *mut c_char, buf_len: usize) -> us // one, which is probably more trouble than it's worth. So I'll leave the stateful builder API the // way it is for now, but create a nicer API for Python using kwargs +/// cbindgen:ignore static CURRENT_ENV_BUILDER: Mutex<(EnvInitState, Option)> = std::sync::Mutex::new((EnvInitState::Uninitialized, None)); #[derive(Default, PartialEq, Eq)] diff --git a/c/tests/check_sexpr_parser.c b/c/tests/check_sexpr_parser.c index 7e6978f4b..2227a487c 100644 --- a/c/tests/check_sexpr_parser.c +++ b/c/tests/check_sexpr_parser.c @@ -4,6 +4,8 @@ #include "util.h" #include "int_gnd.h" +#include //GOAT + void setup(void) { } @@ -41,10 +43,47 @@ START_TEST (test_tokenizer_parser) } END_TEST +typedef struct node_types { + int32_t count; + syntax_node_type_t type_buf[32]; +} node_types; + +void save_node_types(const syntax_node_t* node, void *context) { + node_types* nodes = (node_types*)context; + nodes->type_buf[nodes->count] = syntax_node_type(node); + nodes->count++; +}; + +START_TEST (test_syntax_tree_parser) +{ + sexpr_parser_t parser = sexpr_parser_new("(+ $one \"one\") ; Is it 2?"); + + syntax_node_t root_node = sexpr_parser_parse_to_syntax_tree(&parser); + + node_types nodes; + nodes.count = 0; + syntax_node_iterate(&root_node, &save_node_types, &nodes); + + ck_assert_int_eq(nodes.count, 8); + ck_assert_int_eq(nodes.type_buf[0], OPEN_PAREN); + ck_assert_int_eq(nodes.type_buf[1], WORD_TOKEN); + ck_assert_int_eq(nodes.type_buf[2], WHITESPACE); + ck_assert_int_eq(nodes.type_buf[3], VARIABLE_TOKEN); + ck_assert_int_eq(nodes.type_buf[4], WHITESPACE); + ck_assert_int_eq(nodes.type_buf[5], STRING_TOKEN); + ck_assert_int_eq(nodes.type_buf[6], CLOSE_PAREN); + ck_assert_int_eq(nodes.type_buf[7], EXPRESSION_GROUP); + + syntax_node_free(root_node); + sexpr_parser_free(parser); +} +END_TEST + void init_test(TCase* test_case) { tcase_set_timeout(test_case, 300); //300s = 5min. To test for memory leaks tcase_add_checked_fixture(test_case, setup, teardown); tcase_add_test(test_case, test_tokenizer_parser); + tcase_add_test(test_case, test_syntax_tree_parser); } TEST_MAIN(init_test); diff --git a/lib/src/metta/text.rs b/lib/src/metta/text.rs index e2c14a191..9a806ecc7 100644 --- a/lib/src/metta/text.rs +++ b/lib/src/metta/text.rs @@ -63,7 +63,7 @@ impl Tokenizer { } /// The meaning of a parsed syntactic element, generated from a substring in the input text -#[derive(Clone, Debug)] +#[derive(Clone, Copy, Debug)] pub enum SyntaxNodeType { /// Comment line. All text between a non-escaped ';' and a newline Comment, @@ -80,13 +80,13 @@ pub enum SyntaxNodeType { CloseParen, /// Whitespace. One or more whitespace chars Whitespace, - /// Symbol Atom. A Symbol Atom + /// Text that remains unparsed after a parse error has occurred LeftoverText, /// A Group of [SyntaxNode]s between an [OpenParen](SyntaxNodeType::OpenParen) and a matching /// [CloseParen](SyntaxNodeType::CloseParen) ExpressionGroup, /// Syntax Nodes that cannot be combined into a coherent atom due to a parse error, even if some - /// of the individual nodes are valid + /// of the individual nodes could represent valid atoms ErrorGroup, } From fd3b6c1df7f0a1a706d7e6342c6593246ef99a4c Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Mon, 25 Sep 2023 15:46:35 +0400 Subject: [PATCH 08/28] Adding Python interface to SyntaxNode parsing --- c/src/metta.rs | 40 ++++++++++++++++++++++++++++- python/hyperon/base.py | 47 ++++++++++++++++++++++++++++++++++ python/hyperonpy.cpp | 46 ++++++++++++++++++++++++++++++++- python/tests/CMakeLists.txt | 1 + python/tests/test_sexparser.py | 26 +++++++++++++++++++ 5 files changed, 158 insertions(+), 2 deletions(-) create mode 100644 python/tests/test_sexparser.py diff --git a/c/src/metta.rs b/c/src/metta.rs index e0248507e..11b744504 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -232,6 +232,15 @@ impl From for syntax_node_t { } } +impl From> for syntax_node_t { + fn from(node: Option) -> Self { + match node { + Some(node) => Self{ node: Box::into_raw(Box::new(RustSyntaxNode(node))) }, + None => syntax_node_t::null() + } + } +} + impl syntax_node_t { fn into_inner(self) -> SyntaxNode { unsafe{ (*Box::from_raw(self.node)).0 } @@ -239,6 +248,12 @@ impl syntax_node_t { fn borrow(&self) -> &SyntaxNode { &unsafe{ &*(&*self).node }.0 } + fn is_null(&self) -> bool { + self.node == core::ptr::null_mut() + } + fn null() -> Self { + Self{node: core::ptr::null_mut()} + } } /// @brief The type of language construct respresented by a syntax_node_t @@ -304,7 +319,7 @@ pub type c_syntax_node_callback_t = extern "C" fn(node: *const syntax_node_t, co pub extern "C" fn sexpr_parser_parse_to_syntax_tree(parser: *mut sexpr_parser_t) -> syntax_node_t { let parser = unsafe{ &*parser }.borrow_inner(); - parser.parse_to_syntax_tree().unwrap().into() + parser.parse_to_syntax_tree().into() } /// @brief Frees a syntax_node_t @@ -317,6 +332,19 @@ pub extern "C" fn syntax_node_free(node: syntax_node_t) { drop(node); } +/// @brief Creates a deep copy of a `syntax_node_t` +/// @ingroup tokenizer_and_parser_group +/// @param[in] node A pointer to the `syntax_node_t` +/// @return The `syntax_node_t` representing the cloned syntax node +/// @note The caller must take ownership responsibility for the returned `syntax_node_t`, and ultimately free +/// it with `syntax_node_free()` +/// +#[no_mangle] +pub extern "C" fn syntax_node_clone(node: *const syntax_node_t) -> syntax_node_t { + let node = unsafe{ &*node }.borrow(); + node.clone().into() +} + /// @brief Performs a depth-first iteration of all child syntax nodes within a syntax tree /// @ingroup tokenizer_and_parser_group /// @param[in] node A pointer to the top-level `syntax_node_t` representing the syntax tree @@ -344,6 +372,16 @@ pub extern "C" fn syntax_node_type(node: *const syntax_node_t) -> syntax_node_ty node.node_type.into() } +/// @brief Returns `true` if a syntax node represents the end of the stream +/// @ingroup tokenizer_and_parser_group +/// @param[in] node A pointer to the `syntax_node_t` +/// @return The boolean value indicating if the node is a a null node +/// +#[no_mangle] +pub extern "C" fn syntax_node_is_null(node: *const syntax_node_t) -> bool { + unsafe{ &*node }.is_null() +} + /// @brief Returns `true` if a syntax node is a leaf (has no children) and `false` otherwise /// @ingroup tokenizer_and_parser_group /// @param[in] node A pointer to the `syntax_node_t` diff --git a/python/hyperon/base.py b/python/hyperon/base.py index 7b49edde9..12d4e9e91 100644 --- a/python/hyperon/base.py +++ b/python/hyperon/base.py @@ -1,6 +1,7 @@ import hyperonpy as hp from .atoms import Atom, BindingsSet +from hyperonpy import SyntaxNodeType class AbstractSpace: """ @@ -323,6 +324,45 @@ def register_token(self, regex, constr): """ hp.tokenizer_register_token(self.ctokenizer, regex, constr) +class SyntaxNode: + """ + A class representing a node in a parsed syntax tree + """ + + def __init__(self, cnode): + """ + Initialize a new Tokenizer. + """ + self.cnode = cnode + + def __del__(self): + """ + Destructor for the SyntaxNode + """ + hp.syntax_node_free(self.cnode) + + def get_type(self): + """ + Returns the type of a SyntaxNode + """ + return hp.syntax_node_type(self.cnode) + + def src_range(self): + """ + Returns the range of offsets into the source code of the text represented by the SyntaxNode + """ + range_tuple = hp.syntax_node_src_range(self.cnode) + return range(range_tuple[0], range_tuple[1]) + + def unroll(self): + """ + Returns a list of all leaf nodes recursively contained within a SyntaxNode + """ + syntax_nodes = [] + for cnode in hp.syntax_node_unroll(self.cnode): + syntax_nodes.append(SyntaxNode(cnode)) + return syntax_nodes + class SExprParser: """ A class responsible for parsing S-expressions (Symbolic Expressions). @@ -340,6 +380,13 @@ def parse(self, tokenizer): catom = self.cparser.parse(tokenizer.ctokenizer) return Atom._from_catom(catom) if catom is not None else None + def parse_to_syntax_tree(self): + """ + Parses the S-expression into a SyntaxNode representing the top-level of a syntax tree. + """ + cnode = self.cparser.parse_to_syntax_tree() + return SyntaxNode(cnode) if cnode is not None else None + class Interpreter: """ A wrapper class for the MeTTa interpreter that handles the interpretation of expressions in a given grounding space. diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index 923e0fb78..af16031fd 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -33,6 +33,7 @@ using CBindings = CStruct; using CBindingsSet = CStruct; using CSpace = CStruct; using CTokenizer = CStruct; +using CSyntaxNode = CStruct; using CStepResult = CStruct; using CMetta = CStruct; @@ -402,6 +403,13 @@ void bindings_copy_to_list_callback(bindings_t* bindings, void* context){ bindings_list.append(CBindings(bindings_clone(bindings))); } +void syntax_node_copy_to_list_callback(const syntax_node_t* node, void *context) { + pybind11::list& nodes_list = *( (pybind11::list*)(context) ); + if (syntax_node_is_leaf(node)) { + nodes_list.append(CSyntaxNode(syntax_node_clone(node))); + } +}; + struct CConstr { py::function pyconstr; @@ -439,6 +447,11 @@ struct CSExprParser { atom_t atom = sexpr_parser_parse(&this->parser, tokenizer.ptr()); return !atom_is_null(&atom) ? py::cast(CAtom(atom)) : py::none(); } + + py::object parse_to_syntax_tree() { + syntax_node_t root_node = sexpr_parser_parse_to_syntax_tree(&this->parser); + return !syntax_node_is_null(&root_node) ? py::cast(CSyntaxNode(root_node)) : py::none(); + } }; struct CAtomType {}; @@ -661,9 +674,40 @@ PYBIND11_MODULE(hyperonpy, m) { tokenizer_register_token(tokenizer.ptr(), regex, &TOKEN_API, new CConstr(constr)); }, "Register token"); + py::enum_(m, "SyntaxNodeType") + .value("COMMENT", syntax_node_type_t::COMMENT) + .value("VARIABLE_TOKEN", syntax_node_type_t::VARIABLE_TOKEN) + .value("STRING_TOKEN", syntax_node_type_t::STRING_TOKEN) + .value("WORD_TOKEN", syntax_node_type_t::WORD_TOKEN) + .value("OPEN_PAREN", syntax_node_type_t::OPEN_PAREN) + .value("CLOSE_PAREN", syntax_node_type_t::CLOSE_PAREN) + .value("WHITESPACE", syntax_node_type_t::WHITESPACE) + .value("LEFTOVER_TEXT", syntax_node_type_t::LEFTOVER_TEXT) + .value("EXPRESSION_GROUP", syntax_node_type_t::EXPRESSION_GROUP) + .value("ERROR_GROUP", syntax_node_type_t::ERROR_GROUP) + .export_values(); + + py::class_(m, "CSyntaxNode"); + m.def("syntax_node_free", [](CSyntaxNode node) { syntax_node_free(node.obj); }, "Free a syntax node at the top level of a syntax tree"); + m.def("syntax_node_clone", [](CSyntaxNode& node) { return CSyntaxNode(syntax_node_clone(node.ptr())); }, "Create a deep copy of the syntax node"); + m.def("syntax_node_type", [](CSyntaxNode& node) { return syntax_node_type(node.ptr()); }, "Get type of the syntax node"); + m.def("syntax_node_is_null", [](CSyntaxNode& node) { return syntax_node_is_null(node.ptr()); }, "Returns True if a syntax node is Null"); + m.def("syntax_node_is_leaf", [](CSyntaxNode& node) { return syntax_node_is_leaf(node.ptr()); }, "Returns True if a syntax node is Null"); + m.def("syntax_node_src_range", [](CSyntaxNode& node) -> py::object { + size_t start, end; + syntax_node_src_range(node.ptr(), &start, &end); + return py::make_tuple(start, end); + }, "Get range in source code offsets for the text represented by the node"); + m.def("syntax_node_unroll", [](CSyntaxNode& node) { + pybind11::list nodes_list; + syntax_node_iterate(node.ptr(), syntax_node_copy_to_list_callback, &nodes_list); + return nodes_list; + }, "Returns a list of all leaf nodes recursively contained within a SyntaxNode"); + py::class_(m, "CSExprParser") .def(py::init()) - .def("parse", &CSExprParser::parse, "Return next parser atom or None"); + .def("parse", &CSExprParser::parse, "Return next parser atom or None") + .def("parse_to_syntax_tree", &CSExprParser::parse_to_syntax_tree, "Return next parser atom or None, as a syntax node at the root of a syntax tree"); py::class_(m, "CStepResult") .def("__str__", [](CStepResult step) { diff --git a/python/tests/CMakeLists.txt b/python/tests/CMakeLists.txt index 08a06b805..bd495f2ad 100644 --- a/python/tests/CMakeLists.txt +++ b/python/tests/CMakeLists.txt @@ -17,6 +17,7 @@ ADD_TESTS("test_examples.py") ADD_TESTS("test_extend.py") ADD_TESTS("test_grounded_type.py") ADD_TESTS("test_grounding_space.py") +ADD_TESTS("test_sexparser.py") ADD_TESTS("test_metta.py") ADD_TESTS("test_minecraft.py") ADD_TESTS("test_minelogy.py") diff --git a/python/tests/test_sexparser.py b/python/tests/test_sexparser.py new file mode 100644 index 000000000..368aa37cb --- /dev/null +++ b/python/tests/test_sexparser.py @@ -0,0 +1,26 @@ +import unittest + +from hyperon import * + +class HyperonTestCase(unittest.TestCase): + + def __init__(self, methodName): + super().__init__(methodName) + + def testParseToSyntaxNodes(self): + parser = SExprParser("(+ one \"one\")") + syntax_node = parser.parse_to_syntax_tree() + leaf_node_list = syntax_node.unroll() + leaf_node_types = []; + for node in leaf_node_list: + leaf_node_types.append(node.get_type()) + + expected_node_types = [SyntaxNodeType.OPEN_PAREN, + SyntaxNodeType.WORD_TOKEN, + SyntaxNodeType.WHITESPACE, + SyntaxNodeType.WORD_TOKEN, + SyntaxNodeType.WHITESPACE, + SyntaxNodeType.STRING_TOKEN, + SyntaxNodeType.CLOSE_PAREN]; + + self.assertEqual(leaf_node_types, expected_node_types) From 9906182cd570d06076ae0ac94a3939a991909b92 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Tue, 26 Sep 2023 12:46:11 +0400 Subject: [PATCH 09/28] Adding C interface to incremental runner --- c/src/metta.rs | 110 ++++++++++++++++++++++++++++++++++- c/tests/CMakeLists.txt | 4 ++ c/tests/check_runner.c | 67 +++++++++++++++++++++ c/tests/check_sexpr_parser.c | 2 - lib/src/metta/runner/mod.rs | 4 +- repl/src/metta_shim.rs | 2 +- 6 files changed, 183 insertions(+), 6 deletions(-) create mode 100644 c/tests/check_runner.c diff --git a/c/src/metta.rs b/c/src/metta.rs index 11b744504..386223ef2 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -3,7 +3,7 @@ use hyperon::space::DynSpace; use hyperon::metta::text::*; use hyperon::metta::interpreter; use hyperon::metta::interpreter::InterpreterState; -use hyperon::metta::runner::Metta; +use hyperon::metta::runner::{Metta, RunnerState, stdlib}; use hyperon::metta::environment::{Environment, EnvBuilder}; use hyperon::rust_type_atom; @@ -324,7 +324,7 @@ pub extern "C" fn sexpr_parser_parse_to_syntax_tree(parser: *mut sexpr_parser_t) /// @brief Frees a syntax_node_t /// @ingroup tokenizer_and_parser_group -/// @param[in] node The `sexpr_parser_t` handle to free +/// @param[in] node The `syntax_node_t` to free /// #[no_mangle] pub extern "C" fn syntax_node_free(node: syntax_node_t) { @@ -530,6 +530,10 @@ pub extern "C" fn get_atom_types(space: *const space_t, atom: *const atom_ref_t, // MeTTa Intperpreter Interface // =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- +//QUESTION: It feels like a runner_state_t and a step_result_t are getting at the same functionality, +// but at different levels. I think it probably makes sense to remove step_result_t from the C +// and Python APIs to cut down on API surface area + /// @brief Contains the state for an in-flight interpreter operation /// @ingroup interpreter_group /// @note A `step_result_t` is initially created by `interpret_init()`. Each call to `interpret_step()`, in @@ -744,6 +748,102 @@ pub extern "C" fn metta_run(metta: *mut metta_t, parser: *mut sexpr_parser_t, } } +/// @brief Represents the state of a MeTTa runner +/// @ingroup interpreter_group +/// @note `runner_state_t` handles must be freed with `runner_state_free()` +/// +#[repr(C)] +pub struct runner_state_t { + /// Internal. Should not be accessed directly + state: *mut RustRunnerState, +} + +struct RustRunnerState(RunnerState<'static>); + +impl From> for runner_state_t { + fn from(state: RunnerState<'static>) -> Self { + Self{ state: Box::into_raw(Box::new(RustRunnerState(state))) } + } +} + +impl runner_state_t { + fn into_inner(self) -> RunnerState<'static> { + unsafe{ Box::from_raw(self.state).0 } + } + fn borrow(&self) -> &RunnerState<'static> { + &unsafe{ &*(&*self).state }.0 + } + fn borrow_mut(&mut self) -> &mut RunnerState<'static> { + &mut unsafe{ &mut *(&*self).state }.0 + } +} + +/// @brief Creates a runner_state_t, to use for step-wise execution +/// @ingroup interpreter_group +/// @param[in] metta A pointer to the Interpreter handle +/// @return The newly created `runner_state_t`, which can begin evaluating MeTTa code +/// @note The returned `runner_state_t` handle must be freed with `runner_state_free()` +/// +#[no_mangle] +pub extern "C" fn metta_start_run(metta: *mut metta_t) -> runner_state_t { + let metta = unsafe{ &*metta }.borrow(); + let state = metta.start_run(); + state.into() +} + +/// @brief Frees a runner_state_t +/// @ingroup interpreter_group +/// @param[in] node The `runner_state_t` to free +/// +#[no_mangle] +pub extern "C" fn runner_state_free(state: runner_state_t) { + let state = state.into_inner(); + drop(state); +} + +/// @brief Runs one step of the interpreter +/// @ingroup interpreter_group +/// @param[in] metta A pointer to the Interpreter handle +/// @param[in] parser A pointer to the S-Expression Parser handle, containing the expression text +/// @param[in] state A pointer to the in-flight runner state +/// +#[no_mangle] +pub extern "C" fn metta_run_step(metta: *mut metta_t, parser: *mut sexpr_parser_t, state: *mut runner_state_t) { + let metta = unsafe{ &*metta }.borrow(); + let parser = unsafe{ &*parser }.borrow_inner(); + let state = unsafe{ &mut *state }.borrow_mut(); + metta.run_step(parser, state).unwrap_or_else(|err| panic!("Unhandled MeTTa error: {}", err)); +} + +/// @brief Returns whether or not the runner_state_t has completed all outstanding work +/// @ingroup interpreter_group +/// @param[in] state The `runner_state_t` to inspect +/// @return `true` if the runner has already concluded, or `false` if there is more work to do +/// +#[no_mangle] +pub extern "C" fn runner_state_is_complete(state: *const runner_state_t) -> bool { + let state = unsafe{ &*state }.borrow(); + state.is_complete() +} + +/// @brief Accesses the current in-flight results in the runner_state_t +/// @ingroup interpreter_group +/// @param[in] state The `runner_state_t` within which to preview results +/// @param[in] callback A function that will be called to provide a vector of atoms produced by the evaluation +/// @param[in] context A pointer to a caller-defined structure to facilitate communication with the `callback` function +/// @warning The provided results will be overwritten by the next call to `metta_run_step`, so the caller +/// must clone the result atoms if they are needed for an extended period of time +/// +#[no_mangle] +pub extern "C" fn runner_state_current_results(state: *const runner_state_t, + callback: c_atom_vec_callback_t, context: *mut c_void) { + let state = unsafe{ &*state }.borrow(); + let results = state.current_results(); + for result in results { + return_atoms(result, callback, context); + } +} + /// @brief Runs the MeTTa Interpreter to evaluate an input Atom /// @ingroup interpreter_group /// @param[in] metta A pointer to the Interpreter handle @@ -773,6 +873,12 @@ pub extern "C" fn metta_load_module(metta: *mut metta_t, name: *const c_char) { // TODO: return erorrs properly metta.load_module(PathBuf::from(cstr_as_str(name))) .expect("Returning errors from C API is not implemented yet"); + + // TODO: This is a hack, We need a way to register grounded tokens with a module + let name_cstr = unsafe{ std::ffi::CStr::from_ptr(name) }; + if name_cstr.to_str().unwrap() == "stdlib" { + stdlib::register_rust_tokens(&metta); + } } // =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- diff --git a/c/tests/CMakeLists.txt b/c/tests/CMakeLists.txt index df8f68ad4..35601fad1 100644 --- a/c/tests/CMakeLists.txt +++ b/c/tests/CMakeLists.txt @@ -19,3 +19,7 @@ add_test(NAME check_sexpr_parser COMMAND check_sexpr_parser) add_executable(check_types check_types.c ${TEST_SOURCES}) target_link_libraries(check_types hyperonc-static CONAN_PKG::libcheck) add_test(NAME check_types COMMAND check_types) + +add_executable(check_runner check_runner.c ${TEST_SOURCES}) +target_link_libraries(check_runner hyperonc-static CONAN_PKG::libcheck) +add_test(NAME check_runner COMMAND check_runner) diff --git a/c/tests/check_runner.c b/c/tests/check_runner.c new file mode 100644 index 000000000..a6b2dd236 --- /dev/null +++ b/c/tests/check_runner.c @@ -0,0 +1,67 @@ + +#include +#include + +#include "test.h" +#include "util.h" + +void setup(void) { +} + +void teardown(void) { +} + +void copy_atom_vec(const atom_vec_t* atoms, void* context) { + atom_vec_t** dst_ptr = (atom_vec_t**)context; + if (*dst_ptr != NULL) { + abort(); + } + *dst_ptr = malloc(sizeof(atom_vec_t)); + **dst_ptr = atom_vec_clone(atoms); +} + +START_TEST (test_incremental_runner) +{ + metta_t runner = metta_new(); + metta_load_module(&runner, "stdlib"); + + runner_state_t runner_state = metta_start_run(&runner); + + sexpr_parser_t parser = sexpr_parser_new("!(+ 1 (+ 2 (+ 3 4)))"); + + int step_count = 0; + char atom_str_buf[64]; + atom_str_buf[0] = 0; + while (!runner_state_is_complete(&runner_state)) { + metta_run_step(&runner, &parser, &runner_state); + + atom_vec_t* results = NULL; + runner_state_current_results(&runner_state, ©_atom_vec, &results); + + if (results != NULL) { + if (atom_vec_len(results) > 0) { + atom_ref_t result_atom = atom_vec_get(results, 0); + size_t len = atom_to_str(&result_atom, atom_str_buf, 64); + } + atom_vec_free(*results); + free(results); + } + + step_count++; + } + ck_assert_str_eq(atom_str_buf, "10"); + + sexpr_parser_free(parser); + + runner_state_free(runner_state); + metta_free(runner); +} +END_TEST + +void init_test(TCase* test_case) { + tcase_set_timeout(test_case, 300); //300s = 5min. To test for memory leaks + tcase_add_checked_fixture(test_case, setup, teardown); + tcase_add_test(test_case, test_incremental_runner); +} + +TEST_MAIN(init_test); diff --git a/c/tests/check_sexpr_parser.c b/c/tests/check_sexpr_parser.c index 2227a487c..240038618 100644 --- a/c/tests/check_sexpr_parser.c +++ b/c/tests/check_sexpr_parser.c @@ -4,8 +4,6 @@ #include "util.h" #include "int_gnd.h" -#include //GOAT - void setup(void) { } diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index cac339ddb..d0c50991d 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -300,9 +300,11 @@ impl<'a> RunnerState<'a> { pub fn is_complete(&self) -> bool { self.mode == MettaRunnerMode::TERMINATE } - pub fn intermediate_results(&self) -> &Vec> { + /// Returns a reference to the current in-progress results within the RunnerState + pub fn current_results(&self) -> &Vec> { &self.results } + /// Consumes the RunnerState and returns the final results pub fn into_results(self) -> Vec> { self.results } diff --git a/repl/src/metta_shim.rs b/repl/src/metta_shim.rs index 49ceac1c3..b0e65e776 100644 --- a/repl/src/metta_shim.rs +++ b/repl/src/metta_shim.rs @@ -129,7 +129,7 @@ impl MettaShim { //Run the next step self.metta.run_step(&mut parser, &mut runner_state) .unwrap_or_else(|err| panic!("Unhandled MeTTa error: {}", err)); - self.result = runner_state.intermediate_results().clone(); + self.result = runner_state.current_results().clone(); } }} } From 1de9d45b20fa0ab35f734e40b9dca854ec96dd4b Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Tue, 26 Sep 2023 16:08:31 +0400 Subject: [PATCH 10/28] Exposing incremental runner in Python API --- python/hyperon/runner.py | 31 +++++++++++++++++++++++++++++++ python/hyperonpy.cpp | 15 ++++++++++++++- python/tests/test_metta.py | 15 +++++++++++++++ 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/python/hyperon/runner.py b/python/hyperon/runner.py index 20a3e96a3..36e69d091 100644 --- a/python/hyperon/runner.py +++ b/python/hyperon/runner.py @@ -4,6 +4,29 @@ from .atoms import Atom, AtomType, OperationAtom from .base import GroundingSpaceRef, Tokenizer, SExprParser +class RunnerState: + def __init__(self, cstate): + """Initialize a RunnerState""" + self.cstate = cstate + + def __del__(self): + """Frees a RunnerState and all associated resources.""" + hp.runner_state_free(self.cstate) + + def run_step(self): + hp.metta_run_step(self.runner.cmetta, self.parser.cparser, self.cstate) + + def is_complete(self): + return hp.runner_state_is_complete(self.cstate) + + def current_results(self, flat=False): + """Returns the current in-progress results from an in-flight program evaluation""" + results = hp.runner_state_current_results(self.cstate) + if flat: + return [Atom._from_catom(catom) for result in results for catom in result] + else: + return [[Atom._from_catom(catom) for catom in result] for result in results] + class MeTTa: """This class contains the MeTTa program execution utilities""" @@ -92,6 +115,14 @@ def run(self, program, flat=False): else: return [[Atom._from_catom(catom) for catom in result] for result in results] + def start_run(self, program): + """Initializes a RunnerState to begin evaluation of MeTTa code""" + parser = SExprParser(program) + state = RunnerState(hp.metta_start_run(self.cmetta)) + state.parser = parser + state.runner = self + return state + class Environment: """This class contains the API for shared platform configuration""" diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index af16031fd..20c5d4f5c 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -35,6 +35,7 @@ using CSpace = CStruct; using CTokenizer = CStruct; using CSyntaxNode = CStruct; using CStepResult = CStruct; +using CRunnerState = CStruct; using CMetta = CStruct; //TODO: This entire CStruct template, and especially these functions should go away when hyperonpy is @@ -443,6 +444,8 @@ struct CSExprParser { sexpr_parser_free(parser); } + sexpr_parser_t* ptr () { return &(this->parser); } + py::object parse(CTokenizer tokenizer) { atom_t atom = sexpr_parser_parse(&this->parser, tokenizer.ptr()); return !atom_is_null(&atom) ? py::cast(CAtom(atom)) : py::none(); @@ -773,11 +776,21 @@ PYBIND11_MODULE(hyperonpy, m) { metta_evaluate_atom(metta.ptr(), atom_clone(atom.ptr()), copy_atoms, &atoms); return atoms; }, "Run MeTTa interpreter on an atom"); - + m.def("metta_start_run", [](CMetta& metta) { return CRunnerState(metta_start_run(metta.ptr())); }, "Initializes the MeTTa interpreter for incremental execution"); + m.def("metta_run_step", [](CMetta& metta, CSExprParser& parser, CRunnerState& state) { metta_run_step(metta.ptr(), parser.ptr(), state.ptr()); }, "Runs one incremental step of the MeTTa interpreter"); m.def("metta_load_module", [](CMetta metta, std::string text) { metta_load_module(metta.ptr(), text.c_str()); }, "Load MeTTa module"); + py::class_(m, "CRunnerState"); + m.def("runner_state_free", [](CRunnerState state) { runner_state_free(state.obj); }, "Frees a Runner State"); + m.def("runner_state_is_complete", [](CRunnerState& state) { return runner_state_is_complete(state.ptr()); }, "Returns whether a RunnerState is finished"); + m.def("runner_state_current_results", [](CRunnerState& state) { + py::list lists_of_atom; + runner_state_current_results(state.ptr(), copy_lists_of_atom, &lists_of_atom); + return lists_of_atom; + }, "Returns the in-flight results from a runner state"); + m.def("environment_config_dir", []() { return func_to_string_no_arg((write_to_buf_no_arg_func_t)&environment_config_dir); }, "Return the config_dir for the platform environment"); diff --git a/python/tests/test_metta.py b/python/tests/test_metta.py index e0652101e..1e0ee29e6 100644 --- a/python/tests/test_metta.py +++ b/python/tests/test_metta.py @@ -34,6 +34,21 @@ def test_metta_runner(self): self.assertEqual([[S('T')]], result) + def test_incremental_runner(self): + program = ''' + !(+ 1 (+ 2 (+ 3 4))) + ''' + runner = MeTTa() + runner_state = runner.start_run(program) + + step_count = 0 + while not runner_state.is_complete(): + runner_state.run_step() + step_count += 1 + + results = runner_state.current_results() + self.assertEqual(repr(results), "[[10]]") + def test_gnd_type_error(self): program = ''' !(+ 2 "String") From a181df9aed255399569144470269462488001f4e Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Wed, 27 Sep 2023 15:55:39 +0400 Subject: [PATCH 11/28] Adding pathway to access parse errors in C & Python APIs --- c/src/metta.rs | 51 ++++++++++++++++++++++++++++++---- lib/src/metta/runner/mod.rs | 1 - python/hyperon/base.py | 6 ++++ python/hyperonpy.cpp | 6 ++++ python/tests/test_sexparser.py | 17 ++++++++++++ 5 files changed, 75 insertions(+), 6 deletions(-) diff --git a/c/src/metta.rs b/c/src/metta.rs index 386223ef2..8edf4629e 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -153,13 +153,27 @@ pub extern "C" fn tokenizer_clone(tokenizer: *const tokenizer_t) -> tokenizer_t pub struct sexpr_parser_t { /// Internal. Should not be accessed directly parser: *const RustSExprParser, + err_string: *mut c_char, +} + +impl sexpr_parser_t { + fn free_err_string(&mut self) { + if !self.err_string.is_null() { + let string = unsafe{ std::ffi::CString::from_raw(self.err_string) }; + drop(string); + self.err_string = core::ptr::null_mut(); + } + } } struct RustSExprParser(std::cell::RefCell>); impl From>> for sexpr_parser_t { fn from(parser: Shared) -> Self { - Self{ parser: std::rc::Rc::into_raw(parser.0).cast() } + Self{ + parser: std::rc::Rc::into_raw(parser.0).cast(), + err_string: core::ptr::null_mut() + } } } @@ -209,9 +223,34 @@ pub extern "C" fn sexpr_parser_parse( parser: *mut sexpr_parser_t, tokenizer: *const tokenizer_t) -> atom_t { - let parser = unsafe{ &*parser }.borrow_inner(); + let parser = unsafe{ &mut *parser }; + parser.free_err_string(); + let rust_parser = parser.borrow_inner(); let tokenizer = unsafe{ &*tokenizer }.borrow_inner(); - parser.parse(tokenizer).unwrap().into() + match rust_parser.parse(tokenizer) { + Ok(atom) => atom.into(), + Err(err) => { + let err_cstring = std::ffi::CString::new(err).unwrap(); + parser.err_string = err_cstring.into_raw(); + atom_t::null() + } + } +} + +/// @brief Returns the error string associated with the last `sexpr_parser_parse` call +/// @ingroup tokenizer_and_parser_group +/// @param[in] parser A pointer to the Parser, which is associated with the text to parse +/// @return A pointer to the C-string containing the parse error that occurred, or NULL if no +/// parse error occurred +/// @warning The returned pointer should NOT be freed. It must never be accessed after the +/// sexpr_parser_t has been freed, or any subsequent `sexpr_parser_parse` or +/// `sexpr_parser_parse_to_syntax_tree` has been made. +/// +#[no_mangle] +pub extern "C" fn sexpr_parser_err_str( + parser: *mut sexpr_parser_t) -> *const c_char { + let parser = unsafe{ &*parser }; + parser.err_string } /// @brief Represents a component in a syntax tree created by parsing MeTTa code @@ -318,8 +357,10 @@ pub type c_syntax_node_callback_t = extern "C" fn(node: *const syntax_node_t, co #[no_mangle] pub extern "C" fn sexpr_parser_parse_to_syntax_tree(parser: *mut sexpr_parser_t) -> syntax_node_t { - let parser = unsafe{ &*parser }.borrow_inner(); - parser.parse_to_syntax_tree().into() + let parser = unsafe{ &mut *parser }; + parser.free_err_string(); + let rust_parser = parser.borrow_inner(); + rust_parser.parse_to_syntax_tree().into() } /// @brief Frees a syntax_node_t diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index d0c50991d..b0a95e772 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -316,7 +316,6 @@ impl<'a> RunnerState<'a> { pub fn new_metta_rust() -> Metta { let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new())); - register_rust_tokens(&metta); metta.load_module(PathBuf::from("stdlib")).expect("Could not load stdlib"); metta } diff --git a/python/hyperon/base.py b/python/hyperon/base.py index 12d4e9e91..53ee4f48b 100644 --- a/python/hyperon/base.py +++ b/python/hyperon/base.py @@ -380,6 +380,12 @@ def parse(self, tokenizer): catom = self.cparser.parse(tokenizer.ctokenizer) return Atom._from_catom(catom) if catom is not None else None + def parse_err(self): + """ + Returns the parse error string from the previous parse, or None. + """ + return self.cparser.sexpr_parser_err_str() + def parse_to_syntax_tree(self): """ Parses the S-expression into a SyntaxNode representing the top-level of a syntax tree. diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index 20c5d4f5c..7cbe0e96c 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -451,6 +451,11 @@ struct CSExprParser { return !atom_is_null(&atom) ? py::cast(CAtom(atom)) : py::none(); } + py::object err_str() { + const char* err_str = sexpr_parser_err_str(&this->parser); + return err_str != NULL ? py::cast(std::string(err_str)) : py::none(); + } + py::object parse_to_syntax_tree() { syntax_node_t root_node = sexpr_parser_parse_to_syntax_tree(&this->parser); return !syntax_node_is_null(&root_node) ? py::cast(CSyntaxNode(root_node)) : py::none(); @@ -710,6 +715,7 @@ PYBIND11_MODULE(hyperonpy, m) { py::class_(m, "CSExprParser") .def(py::init()) .def("parse", &CSExprParser::parse, "Return next parser atom or None") + .def("sexpr_parser_err_str", &CSExprParser::err_str, "Return the parse error from the previous parse operation or None") .def("parse_to_syntax_tree", &CSExprParser::parse_to_syntax_tree, "Return next parser atom or None, as a syntax node at the root of a syntax tree"); py::class_(m, "CStepResult") diff --git a/python/tests/test_sexparser.py b/python/tests/test_sexparser.py index 368aa37cb..e916377a5 100644 --- a/python/tests/test_sexparser.py +++ b/python/tests/test_sexparser.py @@ -24,3 +24,20 @@ def testParseToSyntaxNodes(self): SyntaxNodeType.CLOSE_PAREN]; self.assertEqual(leaf_node_types, expected_node_types) + + def testParseErr(self): + tokenizer = Tokenizer() + parser = SExprParser("(+ one \"one") + parsed_atom = parser.parse(tokenizer) + self.assertEqual(parsed_atom, None) + self.assertEqual(parser.parse_err(), "Unclosed String Literal") + + parser = SExprParser("(+ one \"one\"") + parsed_atom = parser.parse(tokenizer) + self.assertEqual(parsed_atom, None) + self.assertEqual(parser.parse_err(), "Unexpected end of expression") + + parser = SExprParser("(+ one \"one\")") + parsed_atom = parser.parse(tokenizer) + self.assertTrue(parsed_atom is not None) + self.assertEqual(parser.parse_err(), None) From 6679de979487169917f44cee0354f593534d1a1d Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Mon, 2 Oct 2023 16:09:26 +0900 Subject: [PATCH 12/28] Refactoring the repl app to call through python for every MeTTa interaction, in the Python build path --- c/src/metta.rs | 68 ++- c/tests/check_runner.c | 2 +- c/tests/check_space.c | 2 +- lib/src/metta/runner/mod.rs | 59 ++- lib/src/metta/runner/stdlib.rs | 10 +- lib/src/metta/runner/stdlib2.rs | 9 +- lib/tests/case.rs | 6 +- lib/tests/metta.rs | 4 +- python/hyperon/atoms.py | 3 + python/hyperon/runner.py | 57 +- python/hyperonpy.cpp | 19 +- python/tests/scripts/f1_imports.metta | 24 +- repl/Cargo.toml | 7 +- repl/src/config_params.rs | 6 +- repl/src/interactive_helper.rs | 198 +++---- repl/src/main.rs | 26 +- repl/src/metta_shim.rs | 728 +++++++++++++++----------- repl/src/py_shim.py | 71 +++ 18 files changed, 788 insertions(+), 511 deletions(-) create mode 100644 repl/src/py_shim.py diff --git a/c/src/metta.rs b/c/src/metta.rs index 8edf4629e..12cbf6bf0 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -3,7 +3,7 @@ use hyperon::space::DynSpace; use hyperon::metta::text::*; use hyperon::metta::interpreter; use hyperon::metta::interpreter::InterpreterState; -use hyperon::metta::runner::{Metta, RunnerState, stdlib}; +use hyperon::metta::runner::{Metta, RunnerState}; use hyperon::metta::environment::{Environment, EnvBuilder}; use hyperon::rust_type_atom; @@ -455,6 +455,17 @@ pub extern "C" fn syntax_node_src_range(node: *const syntax_node_t, range_start: // atom_t fully repr(C), then I will change this, so the caller doesn't need to worry about freeing the // return values +/// @brief Checks if an atom is a MeTTa error expression +/// @ingroup metta_language_group +/// @param[in] atom A pointer to an `atom_t` or an `atom_ref_t` representing the atom to check +/// @return `true` is the atom is a MeTTa error expression +/// +#[no_mangle] +pub extern "C" fn atom_is_error(atom: *const atom_ref_t) -> bool { + let atom = unsafe{ &*atom }.borrow(); + hyperon::metta::runner::atom_is_error(atom) +} + /// @brief Creates a Symbol atom for the special MeTTa symbol: "%Undefined%" /// @ingroup metta_language_group /// @return The `atom_t` representing the Symbol atom @@ -708,14 +719,14 @@ impl metta_t { } } -/// @brief Creates a new top-level MeTTa Interpreter +/// @brief Creates a new top-level MeTTa Interpreter, with only the Rust stdlib loaded /// @ingroup interpreter_group /// @return A `metta_t` handle to the newly created Interpreter /// @note The caller must take ownership responsibility for the returned `metta_t`, and free it with `metta_free()` /// #[no_mangle] -pub extern "C" fn metta_new() -> metta_t { - let metta = Metta::new_top_level_runner(); +pub extern "C" fn metta_new_rust() -> metta_t { + let metta = Metta::new_rust(); metta.into() } @@ -725,6 +736,7 @@ pub extern "C" fn metta_new() -> metta_t { /// @param[in] tokenizer A pointer to a handle for the Tokenizer for use by the Interpreter /// @return A `metta_t` handle to the newly created Interpreter /// @note The caller must take ownership responsibility for the returned `metta_t`, and free it with `metta_free()` +/// @note This function does not load any stdlib, nor does it run the `init.metta` file from the environment /// #[no_mangle] pub extern "C" fn metta_new_with_space(space: *mut space_t, tokenizer: *mut tokenizer_t) -> metta_t { @@ -746,6 +758,18 @@ pub extern "C" fn metta_free(metta: metta_t) { drop(metta); } +/// @brief Initializes a MeTTa interpreter, for manually bootstrapping a multi-language interpreter +/// @ingroup interpreter_group +/// @param[in] metta A pointer to the Interpreter handle +/// @note Most callers can simply call `metta_new_rust`. This function is provided to support languages +/// with their own stdlib, that needs to be loaded before the init.metta file is run +/// +#[no_mangle] +pub extern "C" fn metta_init_with_platform_env(metta: *mut metta_t) { + let metta = unsafe{ &*metta }.borrow(); + metta.init_with_platform_env() +} + /// @brief Provides access to the Space associated with a MeTTa Interpreter /// @ingroup interpreter_group /// @param[in] metta A pointer to the Interpreter handle @@ -914,12 +938,6 @@ pub extern "C" fn metta_load_module(metta: *mut metta_t, name: *const c_char) { // TODO: return erorrs properly metta.load_module(PathBuf::from(cstr_as_str(name))) .expect("Returning errors from C API is not implemented yet"); - - // TODO: This is a hack, We need a way to register grounded tokens with a module - let name_cstr = unsafe{ std::ffi::CStr::from_ptr(name) }; - if name_cstr.to_str().unwrap() == "stdlib" { - stdlib::register_rust_tokens(&metta); - } } // =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- @@ -942,6 +960,36 @@ pub extern "C" fn environment_config_dir(buf: *mut c_char, buf_len: usize) -> us } } +/// @brief Returns the number of module search paths in the environment +/// @ingroup environment_group +/// @return The number of paths in the Environment +/// +#[no_mangle] +pub extern "C" fn environment_search_path_cnt() -> usize { + Environment::platform_env().modules_search_paths().count() +} + +/// @brief Renders nth module search path from the platform environment into a text buffer +/// @ingroup environment_group +/// @param[in] idx The index of the search path to render, with 0 being the highest priority search +/// path, and subsequent indices having decreasing search priority. +/// If `i > environment_search_path_cnt()`, this function will return 0. +/// @param[out] buf A buffer into which the text will be written +/// @param[in] buf_len The maximum allocated size of `buf` +/// @return The length of the path string, minus the string terminator character. If +/// `return_value > buf_len + 1`, then the text was not fully written and this function should be +/// called again with a larger buffer. This function will return 0 if the input arguments don't +/// specify a valid search path. +/// +#[no_mangle] +pub extern "C" fn environment_nth_search_path(idx: usize, buf: *mut c_char, buf_len: usize) -> usize { + let path = Environment::platform_env().modules_search_paths().nth(idx); + match path { + Some(path) => write_into_buf(path.display(), buf, buf_len), + None => 0 + } +} + //QUESTION / TODO?: It would be nice to wrap the environment_init all in a va_args call, unfortunately cbindgen // doesn't work with va_args and it's not worth requiring an additional build step for this. // Also, C doesn't have a natural way to express key-value pairs or named args, so we'd need to invent diff --git a/c/tests/check_runner.c b/c/tests/check_runner.c index a6b2dd236..9ed3d05d9 100644 --- a/c/tests/check_runner.c +++ b/c/tests/check_runner.c @@ -22,7 +22,7 @@ void copy_atom_vec(const atom_vec_t* atoms, void* context) { START_TEST (test_incremental_runner) { - metta_t runner = metta_new(); + metta_t runner = metta_new_rust(); metta_load_module(&runner, "stdlib"); runner_state_t runner_state = metta_start_run(&runner); diff --git a/c/tests/check_space.c b/c/tests/check_space.c index 4724c936c..76fcd7824 100644 --- a/c/tests/check_space.c +++ b/c/tests/check_space.c @@ -228,7 +228,7 @@ START_TEST (test_space_nested_in_atom) space_add(&nested, expr(atom_sym("A"), atom_sym("B"), atom_ref_null())); atom_t space_atom = atom_gnd_for_space(&nested); - metta_t runner = metta_new(); + metta_t runner = metta_new_rust(); metta_load_module(&runner, "stdlib"); tokenizer_t tokenizer = metta_tokenizer(&runner); diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index b0a95e772..1b4ca72d2 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -65,15 +65,44 @@ pub struct RunnerState<'a> { } impl Metta { + /// A 1-liner to get a MeTTa interpreter using the default configuration - //TODO, see comment on `new_metta_rust`. That function should merge into this one - pub fn new_top_level_runner() -> Self { + /// + /// NOTE: This function is appropriate for Rust or C clients, but if other language-specific + /// stdlibs are involved then see the documentation for [Metta::init] + pub fn new_rust() -> Metta { let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), - Shared::new(Tokenizer::new())); + Shared::new(Tokenizer::new())); + metta.load_module(PathBuf::from("stdlib")).expect("Could not load stdlib"); + metta.init_with_platform_env(); metta } + /// Performs initialization of a MeTTa interpreter. Presently this involves running the `init.metta` + /// file from the platform environment. + /// + /// DISCUSSION: Creating a fully-initialized MeTTa environment should usually be done with with + /// a top-level initialization function, such as [new_metta_rust] or `MeTTa.new()` in Python. + /// + /// Doing it manually involves several steps: + /// 1. Create the MeTTa runner, using [new_with_space]. This provides a working interpreter, but + /// doesn't load any stdlibs + /// 2. Load each language-specific stdlib (Currently only Python has an extended stdlib) + /// 3. Load the Rust `stdlib` (TODO: Conceptually I'd like to load Rust's stdlib first, so other + /// stdlibs can utilize the Rust stdlib's atoms, but that depends on value bridging) + /// 4. Run the `init.metta` file by calling this function + /// + /// TODO: When we are able to load the Rust stdlib before the Python stdlib, which requires value-bridging, + /// we can refactor this function to load the appropriate stdlib(s) and simplify the init process + pub fn init_with_platform_env(&self) { + if let Some(init_meta_file) = Environment::platform_env().initialization_metta_file_path() { + self.load_module(init_meta_file.into()).unwrap(); + } + } + /// Returns a new MeTTa interpreter, using the provided Space and Tokenizer + /// + /// NOTE: This function does not load any stdlib atoms, nor run the [Environment]'s 'init.metta' pub fn new_with_space(space: DynSpace, tokenizer: Shared) -> Self { let settings = Shared::new(HashMap::new()); let modules = Shared::new(HashMap::new()); @@ -125,6 +154,12 @@ impl Metta { self.0.modules.borrow_mut().insert(path.clone(), runner.space().clone()); runner.run(&mut SExprParser::new(program.as_str())) .map_err(|err| format!("Cannot import module, path: {}, error: {}", path.display(), err))?; + + // TODO: This is a hack. We need a way to register tokens at module-load-time, for any module + if path.to_str().unwrap() == "stdlib" { + register_rust_tokens(self); + } + Ok(runner.space().clone()) } } @@ -137,7 +172,7 @@ impl Metta { // TODO: check if it is already there (if the module is newly loaded) let module_space = self.load_module_space(path)?; let space_atom = Atom::gnd(module_space); - self.0.space.borrow_mut().add(space_atom); // self.add_atom(space_atom) + self.0.space.borrow_mut().add(space_atom); Ok(()) } @@ -310,16 +345,6 @@ impl<'a> RunnerState<'a> { } } -//TODO: this function should be totally subsumed into Metta::new_top_level_runner(), but -// first we have to be able to load the "rust" stdlib before the python stdlib, which requires TODO_NOW-lookup-issue-number -// to be fixed, which in-turn requires value-bridging -pub fn new_metta_rust() -> Metta { - let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), - Shared::new(Tokenizer::new())); - metta.load_module(PathBuf::from("stdlib")).expect("Could not load stdlib"); - metta -} - #[cfg(test)] mod tests { use super::*; @@ -337,7 +362,7 @@ mod tests { !(green Fritz) "; - let metta = new_metta_rust(); + let metta = Metta::new_rust(); let result = metta.run(&mut SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![Atom::sym("T")]])); } @@ -401,7 +426,7 @@ mod tests { !(foo) "; - let metta = new_metta_rust(); + let metta = Metta::new_rust(); metta.tokenizer().borrow_mut().register_token(Regex::new("error").unwrap(), |_| Atom::gnd(ErrorOp{})); let result = metta.run(&mut SExprParser::new(program)); @@ -452,7 +477,7 @@ mod tests { !(empty) "; - let metta = new_metta_rust(); + let metta = Metta::new_rust(); metta.tokenizer().borrow_mut().register_token(Regex::new("empty").unwrap(), |_| Atom::gnd(ReturnAtomOp(expr!()))); let result = metta.run(&mut SExprParser::new(program)); diff --git a/lib/src/metta/runner/stdlib.rs b/lib/src/metta/runner/stdlib.rs index 46744b516..56b42d103 100644 --- a/lib/src/metta/runner/stdlib.rs +++ b/lib/src/metta/runner/stdlib.rs @@ -1189,11 +1189,11 @@ pub static METTA_CODE: &'static str = " #[cfg(test)] mod tests { use super::*; - use crate::metta::runner::new_metta_rust; + use crate::metta::runner::Metta; use crate::metta::types::validate_atom; fn run_program(program: &str) -> Result>, String> { - let metta = new_metta_rust(); + let metta = Metta::new_rust(); metta.run(&mut SExprParser::new(program)) } @@ -1406,7 +1406,7 @@ mod tests { #[test] fn superpose_op_multiple_interpretations() { - let metta = new_metta_rust(); + let metta = Metta::new_rust(); let mut parser = SExprParser::new(" (= (f) A) (= (f) B) @@ -1422,7 +1422,7 @@ mod tests { #[test] fn superpose_op_superposed_with_collapse() { - let metta = new_metta_rust(); + let metta = Metta::new_rust(); let mut parser = SExprParser::new(" (= (f) A) (= (f) B) @@ -1436,7 +1436,7 @@ mod tests { #[test] fn superpose_op_consumes_interpreter_errors() { - let metta = new_metta_rust(); + let metta = Metta::new_rust(); let mut parser = SExprParser::new(" (: f (-> A B)) (= (f $x) $x) diff --git a/lib/src/metta/runner/stdlib2.rs b/lib/src/metta/runner/stdlib2.rs index 2a0659047..ab844b32a 100644 --- a/lib/src/metta/runner/stdlib2.rs +++ b/lib/src/metta/runner/stdlib2.rs @@ -399,13 +399,12 @@ pub static METTA_CODE: &'static str = include_str!("stdlib.metta"); #[cfg(test)] mod tests { use super::*; - use crate::metta::runner::new_metta_rust; use crate::matcher::atoms_are_equivalent; use std::convert::TryFrom; fn run_program(program: &str) -> Result>, String> { - let metta = new_metta_rust(); + let metta = Metta::new_rust(); metta.run(&mut SExprParser::new(program)) } @@ -607,7 +606,7 @@ mod tests { #[test] fn metta_assert_equal_op() { - let metta = new_metta_rust(); + let metta = Metta::new_rust(); let assert = AssertEqualOp::new(metta.space().clone()); let program = " (= (foo $x) $x) @@ -627,7 +626,7 @@ mod tests { #[test] fn metta_assert_equal_to_result_op() { - let metta = new_metta_rust(); + let metta = Metta::new_rust(); let assert = AssertEqualToResultOp::new(metta.space().clone()); let program = " (= (foo) A) @@ -832,7 +831,7 @@ mod tests { !(eval (interpret (id_a myAtom) %Undefined% &self)) "; - let metta = new_metta_rust(); + let metta = Metta::new_rust(); metta.tokenizer().borrow_mut().register_token(Regex::new("id_num").unwrap(), |_| Atom::gnd(ID_NUM)); diff --git a/lib/tests/case.rs b/lib/tests/case.rs index 5f486f0fe..d3ad851a7 100644 --- a/lib/tests/case.rs +++ b/lib/tests/case.rs @@ -1,10 +1,10 @@ use hyperon::assert_eq_metta_results; use hyperon::metta::text::SExprParser; -use hyperon::metta::runner::new_metta_rust; +use hyperon::metta::runner::Metta; #[test] fn test_case_operation() { - let metta = new_metta_rust(); + let metta = Metta::new_rust(); let result = metta.run(&mut SExprParser::new(" ; cases are processed sequentially !(case (+ 1 5) @@ -39,7 +39,7 @@ fn test_case_operation() { ")); assert_eq!(result, expected); - let metta = new_metta_rust(); + let metta = Metta::new_rust(); let result = metta.run(&mut SExprParser::new(" (Rel-P A B) (Rel-Q A C) diff --git a/lib/tests/metta.rs b/lib/tests/metta.rs index cb86aa415..0f8758993 100644 --- a/lib/tests/metta.rs +++ b/lib/tests/metta.rs @@ -1,5 +1,5 @@ use hyperon::metta::text::*; -use hyperon::metta::runner::new_metta_rust; +use hyperon::metta::runner::Metta; #[test] fn test_reduce_higher_order() { @@ -13,7 +13,7 @@ fn test_reduce_higher_order() { !(assertEqualToResult ((inc) 2) (3)) "; - let metta = new_metta_rust(); + let metta = Metta::new_rust(); let result = metta.run(&mut SExprParser::new(program)); diff --git a/python/hyperon/atoms.py b/python/hyperon/atoms.py index 6dcd4c0e1..55df3ad24 100644 --- a/python/hyperon/atoms.py +++ b/python/hyperon/atoms.py @@ -27,6 +27,9 @@ def __repr__(self): """Renders a human-readable text description of the Atom.""" return hp.atom_to_str(self.catom) + def is_error(self): + return hp.atom_is_error(self.catom) + def get_type(self): """Gets the type of the current Atom instance""" return hp.atom_get_type(self.catom) diff --git a/python/hyperon/runner.py b/python/hyperon/runner.py index 36e69d091..47d277098 100644 --- a/python/hyperon/runner.py +++ b/python/hyperon/runner.py @@ -1,5 +1,7 @@ import os from importlib import import_module +import importlib.util +import sys import hyperonpy as hp from .atoms import Atom, AtomType, OperationAtom from .base import GroundingSpaceRef, Tokenizer, SExprParser @@ -42,8 +44,9 @@ def __init__(self, space = None, cmetta = None): hp.metta_load_module(self.cmetta, "stdlib") self.register_atom('extend-py!', OperationAtom('extend-py!', - lambda name: self.load_py_module(name) or [], + lambda name: self.load_py_module_from_mod_or_file(name) or [], [AtomType.UNDEFINED, AtomType.ATOM], unwrap=False)) + hp.metta_init_with_platform_env(self.cmetta) def __del__(self): hp.metta_free(self.cmetta) @@ -84,11 +87,53 @@ def load_py_module(self, name): """Loads the given python module""" if not isinstance(name, str): name = repr(name) - mod = import_module(name) - for n in dir(mod): - obj = getattr(mod, n) - if '__name__' in dir(obj) and obj.__name__ == 'metta_register': - obj(self) + try: + mod = import_module(name) + for n in dir(mod): + obj = getattr(mod, n) + if '__name__' in dir(obj) and obj.__name__ == 'metta_register': + obj(self) + return mod + except: + return None + + def load_py_module_from_mod_or_file(self, mod_name): + """Loads the given python-implemented MeTTa module, first using python's module-namespace logic, + then by searching for files in the MeTTa environment's search path""" + + # First, see if the module is already available to Python + if not isinstance(mod_name, str): + mod_name = repr(mod_name) + mod = MeTTa.load_py_module(self, mod_name) + if (mod is None): + # If that failed, try and load the module from a file + file_name = mod_name + ".py" + + # Check each search path directory in order, until we find the module we're looking for + num_search_paths = hp.environment_search_path_cnt() + search_path_idx = 0 + found_path = None + while (search_path_idx < num_search_paths): + search_path = hp.environment_nth_search_path(search_path_idx) + test_path = os.path.join(search_path, file_name) + if (os.path.exists(test_path)): + found_path = test_path + break + search_path_idx += 1 + + if (found_path is not None): + MeTTa.load_py_module_from_path(self, mod_name, found_path) + else: + raise RuntimeError("Failed to load module " + mod_name + "; could not locate file: " + file_name) + + def load_py_module_from_path(self, mod_name, path): + """Loads the given python-implemented MeTTa module from a file at the specified path""" + + spec = importlib.util.spec_from_file_location(mod_name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[mod_name] = module + spec.loader.exec_module(module) + MeTTa.load_py_module(self, mod_name) def import_file(self, fname): """Loads the program file and runs it""" diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index 7cbe0e96c..753c2f432 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -38,17 +38,6 @@ using CStepResult = CStruct; using CRunnerState = CStruct; using CMetta = CStruct; -//TODO: This entire CStruct template, and especially these functions should go away when hyperonpy is -// implemented directly on Rust, rather than on top of hyperonc -//This method is an ugly hack to push C structs through pyo3 and pybind11 without a type that -// Python can understand. Ironically these C structs just wrap Rust objects originating in -// Rust, which are then converted by calling hyperonc directly. -static CMetta cmetta_from_inner_ptr_as_int(size_t buf_as_int) { - metta_t tmp_metta_t; - tmp_metta_t.metta = (RustMettaInterpreter*)buf_as_int; - return CMetta(tmp_metta_t); -} - // Returns a string, created by executing a function that writes string data into a buffer typedef size_t (*write_to_buf_func_t)(void*, char*, size_t); std::string func_to_string(write_to_buf_func_t func, void* arg) { @@ -509,6 +498,7 @@ PYBIND11_MODULE(hyperonpy, m) { m.def("atom_free", [](CAtom atom) { atom_free(atom.obj); }, "Free C atom"); m.def("atom_eq", [](CAtom& a, CAtom& b) -> bool { return atom_eq(a.ptr(), b.ptr()); }, "Test if two atoms are equal"); + m.def("atom_is_error", [](CAtom& atom) -> bool { return atom_is_error(atom.ptr()); }, "Returns True if an atom is a MeTTa error expression"); m.def("atom_to_str", [](CAtom& atom) { return func_to_string((write_to_buf_func_t)&atom_to_str, atom.ptr()); }, "Convert atom to human readable string"); @@ -765,11 +755,12 @@ PYBIND11_MODULE(hyperonpy, m) { py::class_(m, "CAtoms") ADD_SYMBOL(VOID, "Void"); - py::class_(m, "CMetta").def(py::init(&cmetta_from_inner_ptr_as_int)); + py::class_(m, "CMetta"); m.def("metta_new", [](CSpace space, CTokenizer tokenizer) { return CMetta(metta_new_with_space(space.ptr(), tokenizer.ptr())); }, "New MeTTa interpreter instance"); m.def("metta_free", [](CMetta metta) { metta_free(metta.obj); }, "Free MeTTa interpreter"); + m.def("metta_init_with_platform_env", [](CMetta metta) { metta_init_with_platform_env(metta.ptr()); }, "Inits a MeTTa interpreter by running the init.metta file from the environment"); m.def("metta_space", [](CMetta metta) { return CSpace(metta_space(metta.ptr())); }, "Get space of MeTTa interpreter"); m.def("metta_tokenizer", [](CMetta metta) { return CTokenizer(metta_tokenizer(metta.ptr())); }, "Get tokenizer of MeTTa interpreter"); m.def("metta_run", [](CMetta metta, CSExprParser& parser) { @@ -800,6 +791,10 @@ PYBIND11_MODULE(hyperonpy, m) { m.def("environment_config_dir", []() { return func_to_string_no_arg((write_to_buf_no_arg_func_t)&environment_config_dir); }, "Return the config_dir for the platform environment"); + m.def("environment_search_path_cnt", []() { return environment_search_path_cnt(); }, "Returns the number of module search paths in the environment"); + m.def("environment_nth_search_path", [](size_t idx) { + return func_to_string((write_to_buf_func_t)&environment_nth_search_path, (void*)idx); + }, "Returns the module search path at the specified index, in the environment"); m.def("environment_init_start", []() { environment_init_start(); }, "Begin initialization of the platform environment"); m.def("environment_init_finish", []() { environment_init_finish(); }, "Finish initialization of the platform environment"); m.def("environment_init_set_working_dir", [](std::string path) { environment_init_set_working_dir(path.c_str()); }, "Sets the working dir in the platform environment"); diff --git a/python/tests/scripts/f1_imports.metta b/python/tests/scripts/f1_imports.metta index 36e560669..68c05b80d 100644 --- a/python/tests/scripts/f1_imports.metta +++ b/python/tests/scripts/f1_imports.metta @@ -1,13 +1,19 @@ ; NOTE: this behavior can change in the future ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; Even at the very beginning of the main script `(get-atoms &self)` -; returns one atom, which wraps the space of stdlib. +; At the very beginning of the main script, the space +; contains an atom wrapping the space of stdlib. ; The type of this atom is the same as of `&self` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +; QUESTION: Are space implementations really required to preserve ordering of atoms? +; QUESTION / TODO: Is there a better way to find this sub-space than `get-atoms`? +; Now that we load the init.metta file as a module, the previous comment about the +; &self space containing only one atom at the beginning is no longer true !(assertEqual - ((let $x (get-atoms &self) (get-type $x))) - ((get-type &self))) + (let* (($x (collapse (get-atoms &self))) + ($y (car-atom $x))) + (get-type $y)) + (get-type &self)) ; stdlib is already loaded !(assertEqual @@ -20,7 +26,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; !(import! &m f1_moduleA.metta) -; It's first atom is a space +; Its first atom is a space !(assertEqual (let* (($x (collapse (get-atoms &m))) ($y (car-atom $x))) @@ -53,15 +59,17 @@ !(assertEqual (g 2) 102) !(assertEqual (f 2) 103) -; `&self` contains 3 atoms-spaces now: +; `&self` contains 4 atoms-spaces now: ; - stdlib +; - init.metta ; - moduleC imported by moduleA and removed from A after its import to &self ; - moduleA itself, which is the same as &m !(assertEqual &m (let* (($a (collapse (get-atoms &self))) ($x (cdr-atom $a)) - ($y (cdr-atom $x))) - (car-atom $y))) + ($y (cdr-atom $x)) + ($z (cdr-atom $y))) + (car-atom $z))) ; NOTE: now the first atom, which was a space, is removed from `&m`, ; because we load modules only once, and we collect atoms-spaces to diff --git a/repl/Cargo.toml b/repl/Cargo.toml index b205ec93e..afaf592a6 100644 --- a/repl/Cargo.toml +++ b/repl/Cargo.toml @@ -6,7 +6,6 @@ description = "A shell to execute MeTTa" [dependencies] anyhow = { version = "1.0.75", features = ["std"] } -hyperon = { path = "../lib/" } # rustyline = { version = "12.0.0", features = ["derive"] } # rustyline = {git = "https://github.com/luketpeterson/rustyline", version = "12.0.0", features = ["derive"] } # TODO: Yay, our fix landed in main. Still needs to publish however. One step closer @@ -15,12 +14,14 @@ clap = { version = "4.4.0", features = ["derive"] } signal-hook = "0.3.17" pyo3 = { version = "0.19.2", features = ["auto-initialize"], optional = true } pep440_rs = { version = "0.3.11", optional = true } +hyperon = { path = "../lib/", optional = true } #TODO: We can only link Hyperon directly or through Python, but not both at the same time. The right fix is to allow HyperonPy to be built within Hyperon, See https://github.com/trueagi-io/hyperon-experimental/issues/283 [[bin]] name = "metta" path = "src/main.rs" [features] -# default = ["python", "minimal"] +default = ["python"] +no_python = ["hyperon"] python = ["pyo3", "pep440_rs"] -minimal = ["hyperon/minimal"] +minimal = ["hyperon/minimal", "no_python"] diff --git a/repl/src/config_params.rs b/repl/src/config_params.rs index 2989eaab0..f55a67979 100644 --- a/repl/src/config_params.rs +++ b/repl/src/config_params.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use std::io::Write; use std::fs; -use hyperon::metta::environment::Environment; +use crate::MettaShim; const DEFAULT_REPL_METTA: &[u8] = include_bytes!("repl.default.metta"); @@ -29,9 +29,9 @@ pub struct ReplParams { } impl ReplParams { - pub fn new() -> Self { + pub fn new(shim: &MettaShim) -> Self { - if let Some(config_dir) = Environment::platform_env().config_dir() { + if let Some(config_dir) = shim.config_dir() { //Create the default repl.meta file, if it doesn't already exist let repl_config_metta_path = config_dir.join("repl.metta"); diff --git a/repl/src/interactive_helper.rs b/repl/src/interactive_helper.rs index af784ecff..9c63e694d 100644 --- a/repl/src/interactive_helper.rs +++ b/repl/src/interactive_helper.rs @@ -10,10 +10,9 @@ use rustyline::validate::{Validator, ValidationContext, ValidationResult}; use rustyline::error::ReadlineError; use rustyline::{Completer, Helper, Hinter}; -use hyperon::metta::text::{SExprParser, SyntaxNodeType}; - use crate::config_params::*; use crate::metta_shim::MettaShim; +use crate::metta_shim::metta_interface_mod::*; #[derive(Helper, Completer, Hinter)] pub struct ReplHelper { @@ -71,89 +70,75 @@ impl Highlighter for ReplHelper { //Iterate over the syntax nodes generated by the parser, coloring them appropriately let mut colored_line = String::with_capacity(line.len() * 2); let mut bracket_depth = 0; - self.metta.borrow_mut().inside_env(|_metta| { - let mut parser = SExprParser::new(line); - loop { - match parser.parse_to_syntax_tree() { - Some(root_node) => { - root_node.visit_depth_first(|node| { - // We will only render the leaf nodes in the syntax tree - if !node.node_type.is_leaf() { - return; - } - - let mut style_sequence: Vec<&str> = vec![]; - - // TODO: In the future, We'll want to be able to use the type system to assign styling, - // which is going to mean looking at Atoms, and not the tokens they were built from - - //Set up the style for the node - match node.node_type { - SyntaxNodeType::Comment => { - style_sequence.push(&self.style.comment_style); - }, - SyntaxNodeType::VariableToken => { - style_sequence.push(&self.style.variable_style); - }, - SyntaxNodeType::StringToken => { - style_sequence.push(&self.style.string_style); - }, - SyntaxNodeType::WordToken => { - style_sequence.push(&self.style.symbol_style); - }, - SyntaxNodeType::OpenParen => { - style_sequence.push(&self.style.bracket_styles[bracket_depth%self.style.bracket_styles.len()]); - bracket_depth += 1; - }, - SyntaxNodeType::CloseParen => { - if bracket_depth > 0 { - bracket_depth -= 1; - style_sequence.push(&self.style.bracket_styles[bracket_depth%self.style.bracket_styles.len()]); - } else { - style_sequence.push(&self.style.error_style); - } - }, - SyntaxNodeType::LeftoverText => { - style_sequence.push(&self.style.error_style); - } - _ => { } - } - - //See if we need to render this node with the "bracket blink" - if self.style.bracket_match_enabled { - if let Some((_matching_char, blink_idx)) = &blink_char { - if node.src_range.contains(blink_idx) { - style_sequence.push(&self.style.bracket_match_style); - } - } - } - - //Push the styles to the buffer - let style_count = style_sequence.len(); - if style_count > 0 { - colored_line.push_str("\x1b["); - for (style_idx, style) in style_sequence.into_iter().enumerate() { - colored_line.push_str(style); - if style_idx < style_count-1 { - colored_line.push(';'); - } - } - colored_line.push('m'); - } - - //Push the node itself to the buffer - colored_line.push_str(&line[node.src_range.clone()]); - - //And push an undo sequence, if the node was stylized - if style_count > 0 { - colored_line.push_str("\x1b[0m"); - } - }); - }, - None => break, + for (node_type, range) in self.metta.borrow().parse_and_unroll_syntax_tree(line) { + + let mut style_sequence: Vec<&str> = vec![]; + + // TODO: In the future, We'll want to be able to use the type system to assign styling, + // which is going to mean looking at Atoms, and not the tokens they were built from + + //Set up the style for the node + match node_type { + SyntaxNodeType::Comment => { + style_sequence.push(&self.style.comment_style); + }, + SyntaxNodeType::VariableToken => { + style_sequence.push(&self.style.variable_style); + }, + SyntaxNodeType::StringToken => { + style_sequence.push(&self.style.string_style); + }, + SyntaxNodeType::WordToken => { + style_sequence.push(&self.style.symbol_style); + }, + SyntaxNodeType::OpenParen => { + style_sequence.push(&self.style.bracket_styles[bracket_depth%self.style.bracket_styles.len()]); + bracket_depth += 1; + }, + SyntaxNodeType::CloseParen => { + if bracket_depth > 0 { + bracket_depth -= 1; + style_sequence.push(&self.style.bracket_styles[bracket_depth%self.style.bracket_styles.len()]); + } else { + style_sequence.push(&self.style.error_style); + } + }, + SyntaxNodeType::LeftoverText => { + style_sequence.push(&self.style.error_style); } + _ => { } } - }); + + //See if we need to render this node with the "bracket blink" + if self.style.bracket_match_enabled { + if let Some((_matching_char, blink_idx)) = &blink_char { + if range.contains(blink_idx) { + style_sequence.push(&self.style.bracket_match_style); + } + } + } + + //Push the styles to the buffer + let style_count = style_sequence.len(); + if style_count > 0 { + colored_line.push_str("\x1b["); + for (style_idx, style) in style_sequence.into_iter().enumerate() { + colored_line.push_str(style); + if style_idx < style_count-1 { + colored_line.push(';'); + } + } + colored_line.push('m'); + } + + //Push the node itself to the buffer + colored_line.push_str(&line[range]); + + //And push an undo sequence, if the node was stylized + if style_count > 0 { + colored_line.push_str("\x1b[0m"); + } + } Owned(colored_line) } @@ -180,36 +165,27 @@ impl Validator for ReplHelper { let force_submit = *self.force_submit.lock().unwrap(); *self.force_submit.lock().unwrap() = false; let mut validation_result = ValidationResult::Incomplete; - self.metta.borrow_mut().inside_env(|metta| { - let mut parser = SExprParser::new(ctx.input()); - loop { - let result = parser.parse(&metta.metta.tokenizer().borrow()); - - match result { - Ok(Some(_atom)) => (), - Ok(None) => { - validation_result = ValidationResult::Valid(None); - *self.checked_line.borrow_mut() = "".to_string(); - break - }, - Err(err) => { - let input = ctx.input(); - if input.len() < 1 { - break; - } - if !force_submit && - (*self.checked_line.borrow() != &input[0..input.len()-1] || input.as_bytes()[input.len()-1] != b'\n') { - *self.checked_line.borrow_mut() = ctx.input().to_string(); - } else { - validation_result = ValidationResult::Invalid(Some( - format!(" - \x1b[0;{}m{}\x1b[0m", self.style.error_style, err) - )); - } - break; + + match self.metta.borrow_mut().parse_line(ctx.input()) { + Ok(_) => { + validation_result = ValidationResult::Valid(None); + *self.checked_line.borrow_mut() = "".to_string(); + }, + Err(err) => { + let input = ctx.input(); + if input.len() > 0 { + if !force_submit && + (*self.checked_line.borrow() != &input[0..input.len()-1] || input.as_bytes()[input.len()-1] != b'\n') { + *self.checked_line.borrow_mut() = ctx.input().to_string(); + } else { + validation_result = ValidationResult::Invalid(Some( + format!(" - \x1b[0;{}m{}\x1b[0m", self.style.error_style, err) + )); } } } - }); + } + Ok(validation_result) } @@ -244,7 +220,7 @@ impl StyleSettings { string_style: metta_shim.get_config_string(CFG_STRING_STYLE).expect(Self::ERR_STR), error_style: metta_shim.get_config_string(CFG_ERROR_STYLE).expect(Self::ERR_STR), bracket_match_style: metta_shim.get_config_string(CFG_BRACKET_MATCH_STYLE).expect(Self::ERR_STR), - bracket_match_enabled: metta_shim.get_config_atom(CFG_BRACKET_MATCH_ENABLED).map(|_bool_atom| true).unwrap_or(true), //TODO, make this work when we can bridge value atoms + bracket_match_enabled: true, //metta_shim.get_config_atom(CFG_BRACKET_MATCH_ENABLED).map(|_bool_atom| true).unwrap_or(true), //TODO, make this work when we can bridge value atoms } } } diff --git a/repl/src/main.rs b/repl/src/main.rs index cd3d03250..9a6b1753f 100644 --- a/repl/src/main.rs +++ b/repl/src/main.rs @@ -11,8 +11,6 @@ use anyhow::Result; use clap::Parser; use signal_hook::{consts::SIGINT, iterator::Signals}; -use hyperon::metta::environment::EnvBuilder; - mod metta_shim; use metta_shim::*; @@ -59,15 +57,11 @@ fn main() -> Result<()> { } }; - //Init our runtime environment - EnvBuilder::new() - .set_working_dir(Some(&metta_working_dir)) - .add_include_paths(cli_args.include_paths) - .init_platform_env(); - let repl_params = ReplParams::new(); - //Create our MeTTa runtime environment - let mut metta = MettaShim::new(); + let mut metta = MettaShim::new(metta_working_dir, cli_args.include_paths); + + //Init our runtime environment + let repl_params = ReplParams::new(&metta); //Spawn a signal handler background thread, to deal with passing interrupts to the execution loop let mut signals = Signals::new(&[SIGINT])?; @@ -100,11 +94,7 @@ fn main() -> Result<()> { //Only print the output from the primary .metta file let metta_code = std::fs::read_to_string(metta_file)?; metta.exec(metta_code.as_str()); - metta.inside_env(|metta| { - for result in metta.result.iter() { - println!("{result:?}"); - } - }); + metta.print_result(); Ok(()) } else { @@ -177,11 +167,7 @@ fn start_interactive_mode(repl_params: ReplParams, mut metta: MettaShim) -> rust let mut metta = rl.helper().unwrap().metta.borrow_mut(); metta.exec(line.as_str()); - metta.inside_env(|metta| { - for result in metta.result.iter() { - println!("{result:?}"); - } - }); + metta.print_result(); } Err(ReadlineError::Interrupted) | Err(ReadlineError::Eof) => { diff --git a/repl/src/metta_shim.rs b/repl/src/metta_shim.rs index b0e65e776..213908eb2 100644 --- a/repl/src/metta_shim.rs +++ b/repl/src/metta_shim.rs @@ -1,393 +1,495 @@ -use std::path::PathBuf; - -use hyperon::ExpressionAtom; -use hyperon::Atom; -use hyperon::metta::environment::Environment; -use hyperon::metta::runner::{Metta, atom_is_error}; -#[cfg(not(feature = "minimal"))] -use hyperon::metta::runner::stdlib::register_rust_tokens; -#[cfg(feature = "minimal")] -use hyperon::metta::runner::stdlib2::register_rust_tokens; -use hyperon::metta::text::SExprParser; +//! MettaShim is responsible for **ALL** calls between the repl and MeTTa, and is in charge of keeping +//! Python happy (and perhaps other languages in the future). +//! +//! Because so much functionality can be extended with Python, MettaShim must handle many operations +//! you might not expect, such as rendering atoms to text or even dropping atoms +//! -use crate::SIGINT_RECEIVED_COUNT; +pub use metta_interface_mod::MettaShim; -/// MettaShim is responsible for **ALL** calls between the repl and MeTTa, and is in charge of keeping -/// Python happy (and perhaps other languages in the future). -/// -/// Because so much functionality can be extended with Python, MettaShim must handle many operations -/// you might not expect, such as rendering atoms to text or even dropping atoms -/// -pub struct MettaShim { - pub metta: Metta, - pub result: Vec>, -} +use crate::SIGINT_RECEIVED_COUNT; -#[macro_export] -macro_rules! metta_shim_env { - ( $body:block ) => { - { - #[cfg(feature = "python")] - { - use pyo3::prelude::*; - Python::with_gil(|_py| -> PyResult<()> { - $body - Ok(()) - }).unwrap(); - } - #[cfg(not(feature = "python"))] - { - $body - } - } - }; +/// Prepares to enter an interruptible exec loop +pub fn exec_state_prepare() { + // Clear any leftover count that might have happened if the user pressed Ctrl+C just after MeTTa + // interpreter finished processing, but before control returned to rustyline's prompt. That signal is + // not intended for the new execution we are about to begin. + //See https://github.com/trueagi-io/hyperon-experimental/pull/419#discussion_r1315598220 for more details + *SIGINT_RECEIVED_COUNT.lock().unwrap() = 0; } -impl Drop for MettaShim { - fn drop(&mut self) { - metta_shim_env!{{ - self.result = vec![]; - }} +/// Check whether an exec loop should break based on an interrupt, and clear the interrupt state +pub fn exec_state_should_break() -> bool { + let mut signal_received_cnt = SIGINT_RECEIVED_COUNT.lock().unwrap(); + if *signal_received_cnt > 0 { + *signal_received_cnt = 0; + true + } else { + false } } -impl MettaShim { - - pub fn new() -> Self { - - //Init the MeTTa interpreter - let mut new_shim = Self { - metta: Metta::new_top_level_runner(), - result: vec![], - }; +#[cfg(all(feature = "python", not(feature = "no_python")))] +pub mod metta_interface_mod { + use std::str::FromStr; + use std::path::PathBuf; + use pep440_rs::{parse_version_specifiers, Version}; + use pyo3::prelude::*; + use pyo3::types::{PyTuple, PyString, PyBool, PyList, PyDict}; + use super::{strip_quotes, exec_state_prepare, exec_state_should_break}; - //Init HyperonPy if the repl includes Python support - #[cfg(feature = "python")] - if let Err(err) = || -> Result<(), String> { - //Confirm the hyperonpy version is compatible - py_mod_loading::confirm_hyperonpy_version(">=0.1.0, <0.2.0")?; + /// Load the hyperon module, and get the "__version__" attribute + pub fn get_hyperonpy_version() -> Result { + Python::with_gil(|py| -> PyResult { + let hyperon_mod = PyModule::import(py, "hyperon")?; + let version_obj = hyperon_mod.getattr("__version__")?; + Ok(version_obj.str()?.to_str()?.into()) + }).map_err(|err| { + format!("{err}") + }) + } - //Load the hyperonpy Python stdlib - py_mod_loading::load_python_module(&new_shim.metta, "hyperon.stdlib")?; + pub fn confirm_hyperonpy_version(req_str: &str) -> Result<(), String> { + let req = parse_version_specifiers(req_str).unwrap(); + let version_string = get_hyperonpy_version()?; + //NOTE: Version parsing errors will be encountered by users building hyperonpy from source with an abnormal configuration + // Therefore references to the "hyperon source directory" are ok. Users who get hyperonpy from PyPi won't hit this issue + let version = Version::from_str(&version_string) + .map_err(|_e| format!("Invalid HyperonPy version found: '{version_string}'.\nPlease update the package by running `python -m pip install -e ./python[dev]` from your hyperon source directory."))?; + if req.iter().all(|specifier| specifier.contains(&version)) { Ok(()) - }() { - eprintln!("Fatal Error: {err}"); - std::process::exit(-1); + } else { + Err(format!("MeTTa repl requires HyperonPy version matching '{req_str}'. Found version: '{version}'")) } + } - //Load the Rust stdlib - register_rust_tokens(&new_shim.metta); - new_shim.load_metta_module("stdlib".into()); + pub struct MettaShim { + py_mod: Py, + py_metta: Py, + result: Vec>>, + } - //Add the extend-py! token, if we have Python support - #[cfg(feature = "python")] - { - let extendpy_atom = Atom::gnd(py_mod_loading::ImportPyOp{metta: new_shim.metta.clone()}); - new_shim.metta.tokenizer().borrow_mut().register_token_with_regex_str("extend-py!", move |_| { extendpy_atom.clone() }); - } + impl MettaShim { + const PY_CODE: &str = include_str!("py_shim.py"); - //extend-py! should throw an error if we don't - #[cfg(not(feature = "python"))] - new_shim.metta.tokenizer().borrow_mut().register_token_with_regex_str("extend-py!", move |_| { Atom::gnd(py_mod_err::ImportPyErr) }); + pub fn new(working_dir: PathBuf, include_paths: Vec) -> Self { - //Run the init.metta file - if let Some(init_meta_file) = Environment::platform_env().initialization_metta_file_path() { - new_shim.load_metta_module(init_meta_file.into()); - } + match || -> Result<_, String> { + //Confirm the hyperonpy version is compatible + confirm_hyperonpy_version(">=0.1.0, <0.2.0")?; - new_shim - } + //Initialize the Hyperon environment + let new_shim = MettaShim::init_platform_env(working_dir, include_paths)?; - pub fn load_metta_module(&mut self, module: PathBuf) { - metta_shim_env!{{ - self.metta.load_module(module).unwrap(); - }} - } + Ok(new_shim) + }() { + Ok(shim) => shim, + Err(err) => { + eprintln!("Fatal Error: {err}"); + std::process::exit(-1); + } + } + } - pub fn exec(&mut self, line: &str) { - metta_shim_env!{{ - let mut parser = SExprParser::new(line); - let mut runner_state = self.metta.start_run(); + pub fn init_platform_env(working_dir: PathBuf, include_paths: Vec) -> Result { + match Python::with_gil(|py| -> PyResult<(Py, Py)> { + let py_mod = PyModule::from_code(py, Self::PY_CODE, "", "")?; + let init_func = py_mod.getattr("init_metta")?; + let kwargs = PyDict::new(py); + kwargs.set_item("working_dir", working_dir)?; + kwargs.set_item("include_paths", include_paths)?; + let py_metta = init_func.call((), Some(kwargs))?; + Ok((py_mod.into(), py_metta.into())) + }) { + Err(err) => Err(format!("{err}")), + Ok((py_mod, py_metta)) => Ok(Self { py_mod, py_metta, result: vec![] }), + } + } - // This clears any leftover count that might have happened if the user pressed Ctrl+C just after MeTTa - // interpreter finished processing, but before control returned to rustyline's prompt. That signal is - // not intended for the new execution we are about to begin. - //See https://github.com/trueagi-io/hyperon-experimental/pull/419#discussion_r1315598220 for more details - *SIGINT_RECEIVED_COUNT.lock().unwrap() = 0; + pub fn load_metta_module(&mut self, module: PathBuf) { + Python::with_gil(|py| -> PyResult<()> { + let path = PyString::new(py, &module.into_os_string().into_string().unwrap()); + let py_metta = self.py_metta.as_ref(py); + let args = PyTuple::new(py, &[py_metta, path]); + let module: &PyModule = self.py_mod.as_ref(py); + let func = module.getattr("load_metta_module")?; + func.call1(args)?; + Ok(()) + }).unwrap(); + } - while !runner_state.is_complete() { - //If we received an interrupt, then clear it and break the loop - let mut signal_received_cnt = SIGINT_RECEIVED_COUNT.lock().unwrap(); - if *signal_received_cnt > 0 { - *signal_received_cnt = 0; + pub fn exec(&mut self, line: &str) { + + //Initialize the runner state + let runner_state = Python::with_gil(|py| -> PyResult> { + let line = PyString::new(py, line); + let py_metta = self.py_metta.as_ref(py); + let args = PyTuple::new(py, &[py_metta, line]); + let module: &PyModule = self.py_mod.as_ref(py); + let func = module.getattr("start_run")?; + let result = func.call1(args)?; + Ok(result.into()) + }).unwrap(); + + exec_state_prepare(); + + loop { + //See if we've already finished processing + if Python::with_gil(|py| -> PyResult { + let module: &PyModule = self.py_mod.as_ref(py); + let func = module.getattr("run_is_complete")?; + let args = PyTuple::new(py, &[&runner_state]); + let result = func.call1(args)?; + Ok(result.downcast::().unwrap().is_true()) + }).unwrap() { + break; + } + + //See if we should exit + if exec_state_should_break() { break; } - drop(signal_received_cnt); //Run the next step - self.metta.run_step(&mut parser, &mut runner_state) - .unwrap_or_else(|err| panic!("Unhandled MeTTa error: {}", err)); - self.result = runner_state.current_results().clone(); + self.result = Python::with_gil(|py| -> PyResult>>> { + let module: &PyModule = self.py_mod.as_ref(py); + let func = module.getattr("run_step")?; + let args = PyTuple::new(py, &[&runner_state]); + let result = func.call1(args)?; + let results_list = result.downcast::().unwrap(); + let mut results: Vec>> = vec![]; + for result in results_list { + let inner_list = result.downcast::().unwrap(); + results.push(inner_list.iter().map(|atom| atom.into()).collect()); + } + Ok(results) + }).unwrap(); } - }} - } - - pub fn inside_env(&mut self, func: F) { - metta_shim_env!{{ - func(self) - }} - } + } - pub fn get_config_atom(&mut self, config_name: &str) -> Option { - self.exec(&format!("!(get-state {config_name})")); + pub fn print_result(&self) { + Python::with_gil(|py| -> PyResult<()> { + for result_vec in self.result.iter() { + let result_vec: Vec<&PyAny> = result_vec.iter().map(|atom| atom.as_ref(py)).collect(); + println!("{result_vec:?}"); + } + Ok(()) + }).unwrap() + } - #[allow(unused_assignments)] - let mut result = None; - metta_shim_env!{{ - result = self.result.get(0) - .and_then(|vec| vec.get(0)) - .and_then(|atom| (!atom_is_error(atom)).then_some(atom)) - .cloned() - }} - result - } + pub fn parse_line(&mut self, line: &str) -> Result<(), String> { + Python::with_gil(|py| -> PyResult> { + let py_line = PyString::new(py, line); + let py_metta = self.py_metta.as_ref(py); + let args = PyTuple::new(py, &[py_metta, py_line]); + let module: &PyModule = self.py_mod.as_ref(py); + let func = module.getattr("parse_line")?; + let result = func.call1(args)?; + Ok(if result.is_none() { + Ok(()) + } else { + Err(result.to_string()) + }) + }).unwrap() + } - pub fn get_config_string(&mut self, config_name: &str) -> Option { - let atom = self.get_config_atom(config_name)?; + pub fn parse_and_unroll_syntax_tree(&self, line: &str) -> Vec<(SyntaxNodeType, std::ops::Range)> { + + Python::with_gil(|py| -> PyResult)>> { + let mut result_nodes = vec![]; + let py_line = PyString::new(py, line); + let args = PyTuple::new(py, &[py_line]); + let module: &PyModule = self.py_mod.as_ref(py); + let func = module.getattr("parse_line_to_syntax_tree")?; + let nodes = func.call1(args)?; + let result_list = nodes.downcast::().unwrap(); + for result in result_list { + let result = result.downcast::().unwrap(); + let node_type = SyntaxNodeType::from_py_str(&result[0].to_string()); + let start: usize = result[1].getattr("start")?.extract().unwrap(); + let end: usize = result[1].getattr("stop")?.extract().unwrap(); + let range = core::ops::Range{start, end}; + result_nodes.push((node_type, range)) + } + Ok(result_nodes) + }).unwrap() + } - #[allow(unused_assignments)] - let mut result = None; - metta_shim_env!{{ - //TODO: We need to do atom type checking here - result = Some(Self::strip_quotes(atom.to_string())); - }} - result - } + pub fn config_dir(&self) -> Option { + Python::with_gil(|py| -> PyResult> { + let module: &PyModule = self.py_mod.as_ref(py); + let func = module.getattr("get_config_dir")?; + let result = func.call0()?; + Ok(if result.is_none() { + None + } else { + Some(PathBuf::from(result.to_string())) + }) + }).unwrap() + } - /// A utility function to return the part of a string in between starting and ending quotes - // TODO: Roll this into a stdlib grounded string module, maybe as a test case for - // https://github.com/trueagi-io/hyperon-experimental/issues/351 - fn strip_quotes(the_string: String) -> String { - if let Some(first) = the_string.chars().next() { - if first == '"' { - if let Some(last) = the_string.chars().last() { - if last == '"' { - if the_string.len() > 1 { - return String::from_utf8(the_string.as_bytes()[1..the_string.len()-1].to_vec()).unwrap(); - } + pub fn get_config_expr_vec(&mut self, config_name: &str) -> Option> { + Python::with_gil(|py| -> PyResult>> { + let config_name = PyString::new(py, config_name); + let py_metta = self.py_metta.as_ref(py); + let args = PyTuple::new(py, &[py_metta, config_name]); + let module: &PyModule = self.py_mod.as_ref(py); + let func = module.getattr("get_config_expr_vec")?; + let result = func.call1(args)?; + + Ok(if result.is_none() { + None + } else { + match result.downcast::() { + Ok(result_list) => { + Some(result_list.iter().map(|atom| strip_quotes(atom.to_string())).collect()) + }, + Err(_) => None } - } - } + }) + }).unwrap() + } + + pub fn get_config_string(&mut self, config_name: &str) -> Option { + Python::with_gil(|py| -> PyResult> { + let config_name = PyString::new(py, config_name); + let py_metta = self.py_metta.as_ref(py); + let args = PyTuple::new(py, &[py_metta, config_name]); + let module: &PyModule = self.py_mod.as_ref(py); + let func = module.getattr("get_config_string")?; + let result = func.call1(args)?; + + Ok(if result.is_none() { + None + } else { + Some(strip_quotes(result.to_string())) + }) + }).unwrap() + } + + pub fn get_config_int(&mut self, _config_name: &str) -> Option { + None //TODO. Make this work when I have reliable value atom bridging } - the_string } - pub fn get_config_expr_vec(&mut self, config_name: &str) -> Option> { - let atom = self.get_config_atom(config_name)?; - let mut result = None; - metta_shim_env!{{ - if let Ok(expr) = ExpressionAtom::try_from(atom) { - result = Some(expr.into_children() - .into_iter() - .map(|atom| { - //TODO: We need to do atom type checking here - Self::strip_quotes(atom.to_string()) - }) - .collect()) - } - }} - result + /// Duplicated from Hyperon because linking hyperon directly is not yet allowed + #[derive(Debug)] + pub enum SyntaxNodeType { + Comment, + VariableToken, + StringToken, + WordToken, + OpenParen, + CloseParen, + Whitespace, + LeftoverText, + ExpressionGroup, + ErrorGroup, } - pub fn get_config_int(&mut self, _config_name: &str) -> Option { - None //TODO. Make this work when I have reliable value atom bridging + impl SyntaxNodeType { + fn from_py_str(the_str: &str) -> Self { + match the_str { + "SyntaxNodeType.COMMENT" => Self::Comment, + "SyntaxNodeType.VARIABLE_TOKEN" => Self::VariableToken, + "SyntaxNodeType.STRING_TOKEN" => Self::StringToken, + "SyntaxNodeType.WORD_TOKEN" => Self::WordToken, + "SyntaxNodeType.OPEN_PAREN" => Self::OpenParen, + "SyntaxNodeType.CLOSE_PAREN" => Self::CloseParen, + "SyntaxNodeType.WHITESPACE" => Self::Whitespace, + "SyntaxNodeType.LEFTOVER_TEXT" => Self::LeftoverText, + "SyntaxNodeType.EXPRESSION_GROUP" => Self::ExpressionGroup, + "SyntaxNodeType.ERROR_GROUP" => Self::ErrorGroup, + _ => panic!("Unrecognized syntax node type: {the_str}") + } + } } + } -#[cfg(not(feature = "python"))] -mod py_mod_err { +/// The "no_python" path involves a reimplementation of all of the MeTTa interface points calling MeTTa +/// directly instead of through Python. Maintaining two paths is a temporary stop-gap solution because +/// we can only link the Hyperon Rust library through one pathway and the HyperonPy module is that path +/// when the Python repl is used. +/// +/// When we have the ability to statically link HyperonPy, we can remove this shim and call +/// Hyperon and MeTTa from everywhere in the code. This will likely mean we can get rid of the clumsy +/// implementations in the "python" version of metta_interface_mod. See See https://github.com/trueagi-io/hyperon-experimental/issues/283 +#[cfg(feature = "no_python")] +pub mod metta_interface_mod { + use std::path::{PathBuf, Path}; use std::fmt::Display; - use hyperon::Atom; use hyperon::atom::{Grounded, ExecError, match_by_equality}; use hyperon::matcher::MatchResultIter; use hyperon::metta::*; + use hyperon::metta::environment::EnvBuilder; + use hyperon::metta::text::SExprParser; + use hyperon::ExpressionAtom; + use hyperon::Atom; + use hyperon::metta::environment::Environment; + use hyperon::metta::runner::{Metta, atom_is_error}; + use super::{strip_quotes, exec_state_prepare, exec_state_should_break}; - #[derive(Clone, PartialEq, Debug)] - pub struct ImportPyErr; + pub use hyperon::metta::text::SyntaxNodeType as SyntaxNodeType; - impl Display for ImportPyErr { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "extend-py!") - } + pub struct MettaShim { + pub metta: Metta, + pub result: Vec>, } - impl Grounded for ImportPyErr { - fn type_(&self) -> Atom { - Atom::expr([ARROW_SYMBOL, ATOM_TYPE_SYMBOL, ATOM_TYPE_UNDEFINED]) + impl MettaShim { + + pub fn new(working_dir: PathBuf, include_paths: Vec) -> Self { + match || -> Result<_, String> { + let new_shim = MettaShim::init_platform_env(working_dir, include_paths)?; + Ok(new_shim) + }() { + Ok(shim) => shim, + Err(err) => { + eprintln!("Fatal Error: {err}"); + std::process::exit(-1); + } + } } - fn execute(&self, _args: &[Atom]) -> Result, ExecError> { - Err(ExecError::from("extend-py! not available in metta repl without Python support")) - } + pub fn init_platform_env(working_dir: PathBuf, include_paths: Vec) -> Result { + EnvBuilder::new() + .set_working_dir(Some(&working_dir)) + .add_include_paths(include_paths) + .init_platform_env(); - fn match_(&self, other: &Atom) -> MatchResultIter { - match_by_equality(self, other) - } - } -} + let new_shim = MettaShim { + metta: Metta::new_rust(), + result: vec![], + }; + new_shim.metta.tokenizer().borrow_mut().register_token_with_regex_str("extend-py!", move |_| { Atom::gnd(ImportPyErr) }); -#[cfg(feature = "python")] -mod py_mod_loading { - use std::fmt::Display; - use std::str::FromStr; - use std::path::PathBuf; - use pep440_rs::{parse_version_specifiers, Version}; - use pyo3::prelude::*; - use pyo3::types::{PyTuple, PyDict}; - use hyperon::*; - use hyperon::Atom; - use hyperon::atom::{Grounded, ExecError, match_by_equality}; - use hyperon::matcher::MatchResultIter; - use hyperon::metta::*; - use hyperon::metta::environment::Environment; - use hyperon::metta::runner::Metta; + Ok(new_shim) + } - /// Load the hyperon module, and get the "__version__" attribute - pub fn get_hyperonpy_version() -> Result { - Python::with_gil(|py| -> PyResult { - let hyperon_mod = PyModule::import(py, "hyperon")?; - let version_obj = hyperon_mod.getattr("__version__")?; - Ok(version_obj.str()?.to_str()?.into()) - }).map_err(|err| { - format!("{err}") - }) - } + pub fn load_metta_module(&mut self, module: PathBuf) { + self.metta.load_module(module).unwrap(); + } - pub fn confirm_hyperonpy_version(req_str: &str) -> Result<(), String> { + pub fn parse_and_unroll_syntax_tree(&self, line: &str) -> Vec<(SyntaxNodeType, std::ops::Range)> { - let req = parse_version_specifiers(req_str).unwrap(); - let version_string = get_hyperonpy_version()?; - //NOTE: Version parsing errors will be encountered by users building hyperonpy from source with an abnormal configuration - // Therefore references to the "hyperon source directory" are ok. Users who get hyperonpy from PyPi won't hit this issue - let version = Version::from_str(&version_string) - .map_err(|_e| format!("Invalid HyperonPy version found: '{version_string}'.\nPlease update the package by running `python -m pip install -e ./python[dev]` from your hyperon source directory."))?; - if req.iter().all(|specifier| specifier.contains(&version)) { - Ok(()) - } else { - Err(format!("MeTTa repl requires HyperonPy version matching '{req_str}'. Found version: '{version}'")) + let mut nodes = vec![]; + let mut parser = SExprParser::new(line); + loop { + match parser.parse_to_syntax_tree() { + Some(root_node) => { + root_node.visit_depth_first(|node| { + // We will only render the leaf nodes in the syntax tree + if !node.node_type.is_leaf() { + return; + } + + nodes.push((node.node_type, node.src_range.clone())) + }); + }, + None => break, + } + } + nodes } - } - pub fn load_python_module_from_mod_or_file(metta: &Metta, module_name: &str) -> Result<(), String> { - - // First, see if the module is already registered with Python - match load_python_module(metta, module_name) { - Err(_) => { - // If that failed, try and load the module from a file - - //Check each include directory in order, until we find the module we're looking for - let file_name = PathBuf::from(module_name).with_extension("py"); - let mut found_path = None; - for include_path in Environment::platform_env().modules_search_paths() { - let path = include_path.join(&file_name); - if path.exists() { - found_path = Some(path); - break; + pub fn parse_line(&mut self, line: &str) -> Result<(), String> { + let mut parser = SExprParser::new(line); + loop { + let result = parser.parse(&self.metta.tokenizer().borrow()); + + match result { + Ok(Some(_atom)) => (), + Ok(None) => { + return Ok(()); + }, + Err(err) => { + return Err(err); } } - - match found_path { - Some(path) => load_python_module_from_known_path(metta, module_name, &path), - None => Err(format!("Failed to load module {module_name}; could not locate file: {file_name:?}")) - } } - _ => Ok(()) } - } - pub fn load_python_module_from_known_path(metta: &Metta, module_name: &str, path: &PathBuf) -> Result<(), String> { + pub fn exec(&mut self, line: &str) { + let mut parser = SExprParser::new(line); + let mut runner_state = self.metta.start_run(); - let code = std::fs::read_to_string(&path).or_else(|err| Err(format!("Error reading file {path:?} - {err}")))?; - Python::with_gil(|py| -> PyResult<()> { - let _py_mod = PyModule::from_code(py, &code, path.to_str().unwrap(), module_name)?; - Ok(()) - }).map_err(|err| { - format!("{err}") - })?; + exec_state_prepare(); - // If we suceeded in loading the module from source, then register the MeTTa extensions - load_python_module(metta, module_name) - } + while !runner_state.is_complete() { + if exec_state_should_break() { + break; + } - pub fn load_python_module(metta: &Metta, module_name: &str) -> Result<(), String> { + //Run the next step + self.metta.run_step(&mut parser, &mut runner_state) + .unwrap_or_else(|err| panic!("Unhandled MeTTa error: {}", err)); + self.result = runner_state.current_results().clone(); + } + } - Python::with_gil(|py| -> PyResult<()> { + pub fn print_result(&self) { + for result in self.result.iter() { + println!("{result:?}"); + } + } - // Load the module - let py_mod = PyModule::import(py, module_name)?; + pub fn config_dir(&self) -> Option<&Path> { + Environment::platform_env().config_dir() + } - // Clone the Rust Metta handle and turn it into a CMetta object that hyperonpy can work with - let boxed_metta = Box::into_raw(Box::new(metta.clone())); - let hyperonpy_mod = PyModule::import(py, "hyperonpy")?; - let metta_class_obj = hyperonpy_mod.getattr("CMetta")?; - let args = PyTuple::new(py, &[boxed_metta as usize]); - let wrapped_metta = metta_class_obj.call1(args)?; + pub fn get_config_atom(&mut self, config_name: &str) -> Option { + self.exec(&format!("!(get-state {config_name})")); + self.result.get(0) + .and_then(|vec| vec.get(0)) + .and_then(|atom| (!atom_is_error(atom)).then_some(atom)) + .cloned() + } - // Init a MeTTa Python object from our CMetta - let hyperon_mod = PyModule::import(py, "hyperon")?; - let metta_class_obj = hyperon_mod.getattr("MeTTa")?; - let kwargs = PyDict::new(py); - kwargs.set_item("cmetta", wrapped_metta)?; - let py_metta = metta_class_obj.call((), Some(kwargs))?; - - // Register all the items in the module - for item in py_mod.dir() { - let obj = py_mod.getattr(item.str()?)?; - - if let Ok(obj_name) = obj.getattr("__name__") { - if obj_name.eq("metta_register")? { - let args = PyTuple::new(py, &[py_metta]); - obj.call1(args)?; - } - } - } + pub fn get_config_string(&mut self, config_name: &str) -> Option { + let atom = self.get_config_atom(config_name)?; + //TODO: We need to do atom type checking here + Some(strip_quotes(atom.to_string())) + } - Ok(()) - }).map_err(|err| { - format!("{err}") - }) + pub fn get_config_expr_vec(&mut self, config_name: &str) -> Option> { + let atom = self.get_config_atom(config_name)?; + if let Ok(expr) = ExpressionAtom::try_from(atom) { + Some(expr.into_children() + .into_iter() + .map(|atom| { + //TODO: We need to do atom type checking here + strip_quotes(atom.to_string()) + }) + .collect()) + } else { + None + } + } + pub fn get_config_int(&mut self, _config_name: &str) -> Option { + None //TODO. Make this work when I have reliable value atom bridging + } } #[derive(Clone, PartialEq, Debug)] - pub struct ImportPyOp { - pub metta: Metta, - } + pub struct ImportPyErr; - impl Display for ImportPyOp { + impl Display for ImportPyErr { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "extend-py!") } } - impl Grounded for ImportPyOp { + impl Grounded for ImportPyErr { fn type_(&self) -> Atom { - //TODO: The Repl std atoms should include a "RESOURCE_PATH" atom type Atom::expr([ARROW_SYMBOL, ATOM_TYPE_SYMBOL, ATOM_TYPE_UNDEFINED]) } - fn execute(&self, args: &[Atom]) -> Result, ExecError> { - let arg_error = || ExecError::from("extend-py! expects a resource path argument"); - let module_path_sym_atom: &SymbolAtom = args.get(0) - .ok_or_else(arg_error)? - .try_into().map_err(|_| arg_error())?; - - match load_python_module_from_mod_or_file(&self.metta, module_path_sym_atom.name()) { - Ok(()) => Ok(vec![]), - Err(err) => Err(ExecError::from(err)), - } + fn execute(&self, _args: &[Atom]) -> Result, ExecError> { + Err(ExecError::from("extend-py! not available in metta repl without Python support")) } fn match_(&self, other: &Atom) -> MatchResultIter { @@ -395,3 +497,21 @@ mod py_mod_loading { } } } + +/// A utility function to return the part of a string in between starting and ending quotes +// TODO: Roll this into a stdlib grounded string module, maybe as a test case for +// https://github.com/trueagi-io/hyperon-experimental/issues/351 +fn strip_quotes(the_string: String) -> String { + if let Some(first) = the_string.chars().next() { + if first == '"' { + if let Some(last) = the_string.chars().last() { + if last == '"' { + if the_string.len() > 1 { + return String::from_utf8(the_string.as_bytes()[1..the_string.len()-1].to_vec()).unwrap(); + } + } + } + } + } + the_string +} diff --git a/repl/src/py_shim.py b/repl/src/py_shim.py new file mode 100644 index 000000000..d0100bc6d --- /dev/null +++ b/repl/src/py_shim.py @@ -0,0 +1,71 @@ + +from hyperon import * + +def init_metta(working_dir, include_paths): + Environment.init_platform_env(working_dir = working_dir, include_paths = include_paths) + return MeTTa() + +def load_metta_module(metta, mod_path): + metta.import_file(mod_path) + +def start_run(metta, program): + return metta.start_run(program) + +def run_is_complete(runner_state): + return runner_state.is_complete() + +def run_step(runner_state): + runner_state.run_step() + return runner_state.current_results() + +def parse_line(metta, line): + tokenizer = metta.tokenizer() + parser = SExprParser(line) + while True: + parsed_atom = parser.parse(tokenizer) + if (parsed_atom is None): + if (parser.parse_err() is None): + return None + else: + return parser.parse_err() + +def parse_line_to_syntax_tree(line): + leaf_node_types = []; + parser = SExprParser(line) + while True: + syntax_node = parser.parse_to_syntax_tree() + if syntax_node is None: + break + else: + leaf_node_list = syntax_node.unroll() + for node in leaf_node_list: + leaf_node_types.append((node.get_type(), node.src_range())) + return leaf_node_types + +def get_config_dir(): + return Environment.config_dir() + +def get_config_atom(metta, config_name): + result = metta.run("!(get-state " + config_name + ")") + try: + atom = result[0][0] + if (atom.is_error()): + return None + else: + return atom + except: + return None + +def get_config_expr_vec(metta, config_name): + try: + atom = get_config_atom(metta, config_name) + return atom.get_children() + except: + return None + +def get_config_string(metta, config_name): + atom = get_config_atom(metta, config_name) + if atom is not None: + return atom.__repr__() + else: + return None From 9c6dd2cfeae204bcaa63e4f22a68005c1950eda2 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Mon, 2 Oct 2023 16:32:42 +0900 Subject: [PATCH 13/28] Getting rid of ctor, since we now have a top-level object from which to initialize the logger (the platform Environment) See https://github.com/trueagi-io/hyperon-experimental/pull/314 --- c/src/lib.rs | 1 + lib/Cargo.toml | 1 - lib/src/common/mod.rs | 1 + lib/src/lib.rs | 7 ------- lib/src/metta/environment.rs | 3 +++ 5 files changed, 5 insertions(+), 8 deletions(-) diff --git a/c/src/lib.rs b/c/src/lib.rs index 9781a495f..d8ac20283 100644 --- a/c/src/lib.rs +++ b/c/src/lib.rs @@ -12,6 +12,7 @@ pub mod metta; //TODO: Is there a way we can get rid of this function in the external API? // Discussion about alternative ways to init the logger here: https://github.com/trueagi-io/hyperon-experimental/pull/314 // and here: https://github.com/trueagi-io/hyperon-experimental/issues/146 +//UPDATE: Logger init parameters should be moved into the Environment API. #[no_mangle] pub extern "C" fn init_logger() { hyperon::common::init_logger(false); diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 2e173b68b..ca281ebb0 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -10,7 +10,6 @@ regex = "1.5.4" log = "0.4.0" env_logger = "0.8.4" directories = "5.0.1" # For Environment to find platform-specific config location -ctor = "0.2.0" smallvec = "1.10.0" [lib] diff --git a/lib/src/common/mod.rs b/lib/src/common/mod.rs index 8c5721c49..900a72bee 100644 --- a/lib/src/common/mod.rs +++ b/lib/src/common/mod.rs @@ -20,6 +20,7 @@ use std::collections::HashMap; use crate::metta::metta_atom; +//TODO: logger init params should be moved into the Environment pub fn init_logger(is_test: bool) { let _ = env_logger::builder().is_test(is_test).try_init(); } diff --git a/lib/src/lib.rs b/lib/src/lib.rs index fa309d765..1213ee9af 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -7,10 +7,3 @@ pub mod space; pub mod metta; pub use atom::*; - -use ctor::ctor; - -#[ctor] -fn on_load() { - common::init_logger(false); -} diff --git a/lib/src/metta/environment.rs b/lib/src/metta/environment.rs index 2e500f185..ec4ded183 100644 --- a/lib/src/metta/environment.rs +++ b/lib/src/metta/environment.rs @@ -6,6 +6,8 @@ use std::borrow::Borrow; use directories::ProjectDirs; +use crate::common; + /// Contains state and platform interfaces shared by all MeTTa runners. This includes config settings /// and logger /// @@ -127,6 +129,7 @@ impl EnvBuilder { /// NOTE: This method will panic if the platform Environment has already been initialized pub fn init_platform_env(self) { PLATFORM_ENV.set(self.build()).expect("Fatal Error: Platform Environment already initialized"); + common::init_logger(false); } /// Returns a newly created Environment from the builder configuration From b80a23fd806489820b916562c6f2fe19a00ccc1d Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Tue, 3 Oct 2023 14:50:23 +0900 Subject: [PATCH 14/28] Fixing bug where logger wasn't initialized when platform environment default initialization was used --- lib/src/metta/environment.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/src/metta/environment.rs b/lib/src/metta/environment.rs index ec4ded183..96598f945 100644 --- a/lib/src/metta/environment.rs +++ b/lib/src/metta/environment.rs @@ -28,7 +28,7 @@ impl Environment { /// Returns a reference to the shared "platform" Environment pub fn platform_env() -> &'static Self { - PLATFORM_ENV.get_or_init(|| EnvBuilder::new().build()) + PLATFORM_ENV.get_or_init(|| EnvBuilder::new().build_platform_env()) } /// Returns the Path to the config dir, in an OS-specific location @@ -128,8 +128,13 @@ impl EnvBuilder { /// /// NOTE: This method will panic if the platform Environment has already been initialized pub fn init_platform_env(self) { - PLATFORM_ENV.set(self.build()).expect("Fatal Error: Platform Environment already initialized"); + PLATFORM_ENV.set(self.build_platform_env()).expect("Fatal Error: Platform Environment already initialized"); + } + + /// Internal function to finalize the building of the shared platform environment + fn build_platform_env(self) -> Environment { common::init_logger(false); + self.build() } /// Returns a newly created Environment from the builder configuration From 586e2b2a050fe79bd7b7edb7055e5d2593730200 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Wed, 4 Oct 2023 11:38:46 +0900 Subject: [PATCH 15/28] Adding `try_init` so platform environment initialization can gracefully fail Reworking C API to use builder object, rather than relying on a static state. This allows flexibility to initialize the environment from multiple threads - although that is still ugly and discouraged --- c/src/metta.rs | 101 ++++++++++++++++--------------- lib/src/metta/environment.rs | 10 ++- lib/src/metta/runner/mod.rs | 8 ++- lib/src/metta/runner/stdlib.rs | 6 +- python/hyperon/runner.py | 12 ++-- python/hyperonpy.cpp | 14 +++-- python/tests/test_environment.py | 4 +- 7 files changed, 87 insertions(+), 68 deletions(-) diff --git a/c/src/metta.rs b/c/src/metta.rs index 12cbf6bf0..3e2203fba 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -12,7 +12,6 @@ use crate::atom::*; use crate::space::*; use core::borrow::Borrow; -use std::sync::Mutex; use std::os::raw::*; use regex::Regex; use std::path::PathBuf; @@ -990,118 +989,120 @@ pub extern "C" fn environment_nth_search_path(idx: usize, buf: *mut c_char, buf_ } } -//QUESTION / TODO?: It would be nice to wrap the environment_init all in a va_args call, unfortunately cbindgen -// doesn't work with va_args and it's not worth requiring an additional build step for this. -// Also, C doesn't have a natural way to express key-value pairs or named args, so we'd need to invent -// one, which is probably more trouble than it's worth. So I'll leave the stateful builder API the -// way it is for now, but create a nicer API for Python using kwargs - -/// cbindgen:ignore -static CURRENT_ENV_BUILDER: Mutex<(EnvInitState, Option)> = std::sync::Mutex::new((EnvInitState::Uninitialized, None)); - -#[derive(Default, PartialEq, Eq)] -enum EnvInitState { - #[default] - Uninitialized, - InProcess, - Finished +/// @brief Represents the environment initialization, in progress +/// @ingroup environment_group +/// @note `env_builder_t` must be given to `environment_init_finish()` to properly release it +/// +#[repr(C)] +pub struct env_builder_t { + /// Internal. Should not be accessed directly + builder: *mut RustEnvBuilder, } -fn take_current_builder(expected_state: EnvInitState) -> EnvBuilder { - let mut builder_state = CURRENT_ENV_BUILDER.lock().unwrap(); - if builder_state.0 != expected_state { - panic!("Fatal Error: no active initialization in process. Call environment_init_start first"); +struct RustEnvBuilder(EnvBuilder); + +impl From for env_builder_t { + fn from(builder: EnvBuilder) -> Self { + Self{ builder: Box::into_raw(Box::new(RustEnvBuilder(builder))) } } - core::mem::take(&mut builder_state.1).unwrap() } -fn replace_current_builder(new_state: EnvInitState, builder: Option) { - let mut builder_state = CURRENT_ENV_BUILDER.lock().unwrap(); - builder_state.0 = new_state; - builder_state.1 = builder; +impl env_builder_t { + fn into_inner(self) -> EnvBuilder { + unsafe{ Box::from_raw(self.builder).0 } + } + fn null() -> Self { + Self {builder: core::ptr::null_mut()} + } } /// @brief Begins the initialization of the platform environment -/// @note environment_init_finish must be called after environment initialization is finished /// @ingroup environment_group +/// @return The `env_builder_t` object representing the in-process environment initialization +/// @note environment_init_finish must be called after environment initialization is finished /// #[no_mangle] -pub extern "C" fn environment_init_start() { - let mut builder_state = CURRENT_ENV_BUILDER.lock().unwrap(); - if builder_state.0 != EnvInitState::Uninitialized { - panic!("Fatal Error: environment_init_start must be called only once"); - } - builder_state.0 = EnvInitState::InProcess; - builder_state.1 = Some(EnvBuilder::new()); +pub extern "C" fn environment_init_start() -> env_builder_t { + EnvBuilder::new().into() } /// @brief Finishes initialization of the platform environment /// @ingroup environment_group +/// @param[in] builder The in-process environment builder state to install as the platform environment +/// @return True if the environment was sucessfully initialized with the provided builder state. False +/// if the environment had already been initialized by a prior call /// #[no_mangle] -pub extern "C" fn environment_init_finish() { - let builder = take_current_builder(EnvInitState::InProcess); - builder.init_platform_env(); - replace_current_builder(EnvInitState::Finished, None); +pub extern "C" fn environment_init_finish(builder: env_builder_t) -> bool { + let builder = builder.into_inner(); + builder.try_init_platform_env().is_ok() } /// @brief Sets the working directory for the platform environment /// @ingroup environment_group +/// @param[in] builder A pointer to the in-process environment builder state /// @param[in] path A C-style string specifying a path to a working directory, to search for modules to load. /// Passing `NULL` will unset the working directory /// @note This working directory is not required to be the same as the process working directory, and /// it will not change as the process' working directory is changed /// #[no_mangle] -pub extern "C" fn environment_init_set_working_dir(path: *const c_char) { - let builder = take_current_builder(EnvInitState::InProcess); +pub extern "C" fn environment_init_set_working_dir(builder: *mut env_builder_t, path: *const c_char) { + let builder_arg_ref = unsafe{ &mut *builder }; + let builder = core::mem::replace(builder_arg_ref, env_builder_t::null()).into_inner(); let builder = if path.is_null() { builder.set_working_dir(None) } else { builder.set_working_dir(Some(&PathBuf::from(cstr_as_str(path)))) }; - replace_current_builder(EnvInitState::InProcess, Some(builder)); + *builder_arg_ref = builder.into(); } /// @brief Sets the config directory for the platform environment. A directory at the specified path -/// will be created its contents populated with default values, if one does not already exist +/// will be created, and its contents populated with default values, if one does not already exist /// @ingroup environment_group +/// @param[in] builder A pointer to the in-process environment builder state /// @param[in] path A C-style string specifying a path to a working directory, to search for modules to load /// #[no_mangle] -pub extern "C" fn environment_init_set_config_dir(path: *const c_char) { - let builder = take_current_builder(EnvInitState::InProcess); +pub extern "C" fn environment_init_set_config_dir(builder: *mut env_builder_t, path: *const c_char) { + let builder_arg_ref = unsafe{ &mut *builder }; + let builder = core::mem::replace(builder_arg_ref, env_builder_t::null()).into_inner(); let builder = if path.is_null() { panic!("Fatal Error: path cannot be NULL"); } else { builder.set_config_dir(&PathBuf::from(cstr_as_str(path))) }; - replace_current_builder(EnvInitState::InProcess, Some(builder)); + *builder_arg_ref = builder.into(); } /// @brief Configures the platform environment so that no config directory will be read nor created /// @ingroup environment_group +/// @param[in] builder A pointer to the in-process environment builder state /// #[no_mangle] -pub extern "C" fn environment_init_disable_config_dir() { - let builder = take_current_builder(EnvInitState::InProcess); +pub extern "C" fn environment_init_disable_config_dir(builder: *mut env_builder_t) { + let builder_arg_ref = unsafe{ &mut *builder }; + let builder = core::mem::replace(builder_arg_ref, env_builder_t::null()).into_inner(); let builder = builder.set_no_config_dir(); - replace_current_builder(EnvInitState::InProcess, Some(builder)); + *builder_arg_ref = builder.into(); } /// @brief Adds a config directory to search for imports. The most recently added paths will be searched /// first, continuing in inverse order /// @ingroup environment_group +/// @param[in] builder A pointer to the in-process environment builder state /// @param[in] path A C-style string specifying a path to a working directory, to search for modules to load /// #[no_mangle] -pub extern "C" fn environment_init_add_include_path(path: *const c_char) { - let builder = take_current_builder(EnvInitState::InProcess); +pub extern "C" fn environment_init_add_include_path(builder: *mut env_builder_t, path: *const c_char) { + let builder_arg_ref = unsafe{ &mut *builder }; + let builder = core::mem::replace(builder_arg_ref, env_builder_t::null()).into_inner(); let builder = if path.is_null() { panic!("Fatal Error: path cannot be NULL"); } else { builder.add_include_paths(vec![PathBuf::from(cstr_as_str(path).borrow())]) }; - replace_current_builder(EnvInitState::InProcess, Some(builder)); + *builder_arg_ref = builder.into(); } diff --git a/lib/src/metta/environment.rs b/lib/src/metta/environment.rs index 96598f945..3b4c08d68 100644 --- a/lib/src/metta/environment.rs +++ b/lib/src/metta/environment.rs @@ -128,10 +128,16 @@ impl EnvBuilder { /// /// NOTE: This method will panic if the platform Environment has already been initialized pub fn init_platform_env(self) { - PLATFORM_ENV.set(self.build_platform_env()).expect("Fatal Error: Platform Environment already initialized"); + self.try_init_platform_env().expect("Fatal Error: Platform Environment already initialized"); } - /// Internal function to finalize the building of the shared platform environment + /// Initializes the shared platform Environment. Non-panicking version of [init_platform_env] + pub fn try_init_platform_env(self) -> Result<(), &'static str> { + PLATFORM_ENV.set(self.build_platform_env()).map_err(|_| "Platform Environment already initialized") + } + + /// Internal function to finalize the building of the shared platform environment. Will always be called + /// from inside the init closure of a OnceLock, so it will only be called once per process. fn build_platform_env(self) -> Environment { common::init_logger(false); self.build() diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index 1b4ca72d2..439424b7e 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -39,10 +39,10 @@ pub fn atom_is_error(atom: &Atom) -> bool { } } -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug)] pub struct Metta(Rc); -#[derive(Debug, PartialEq)] +#[derive(Debug)] pub struct MettaContents { space: DynSpace, tokenizer: Shared, @@ -121,6 +121,10 @@ impl Metta { let modules = metta.0.modules.clone(); //Search only the parent directory of the module we're loading + //TODO: I think we want all the same search path behavior as the parent runner, but with + // a different working_dir. Consider using the working_dir from the environment only to + // init the top-level runner, and moving search-path composition logic from the environment + // to the runner. let mut path = path; path.pop(); let search_paths = vec![path]; diff --git a/lib/src/metta/runner/stdlib.rs b/lib/src/metta/runner/stdlib.rs index 56b42d103..8f5bc7957 100644 --- a/lib/src/metta/runner/stdlib.rs +++ b/lib/src/metta/runner/stdlib.rs @@ -33,11 +33,15 @@ fn interpret_no_error(space: DynSpace, expr: &Atom) -> Result, String> } } -#[derive(Clone, PartialEq, Debug)] +#[derive(Clone, Debug)] pub struct ImportOp { metta: Metta, } +impl PartialEq for ImportOp { + fn eq(&self, _other: &Self) -> bool { true } +} + impl ImportOp { pub fn new(metta: Metta) -> Self { Self{ metta } diff --git a/python/hyperon/runner.py b/python/hyperon/runner.py index 47d277098..a8e7b7afa 100644 --- a/python/hyperon/runner.py +++ b/python/hyperon/runner.py @@ -176,14 +176,14 @@ def config_dir(): return hp.environment_config_dir() def init_platform_env(working_dir = None, config_dir = None, disable_config = False, include_paths = []): """Initialize the platform environment with the supplied args""" - hp.environment_init_start() + builder = hp.environment_init_start() if (working_dir is not None): - hp.environment_init_set_working_dir(working_dir) + hp.environment_init_set_working_dir(builder, working_dir) if (config_dir is not None): - hp.environment_init_set_config_dir(config_dir) + hp.environment_init_set_config_dir(builder, config_dir) if (disable_config): - hp.environment_init_disable_config_dir() + hp.environment_init_disable_config_dir(builder) for path in reversed(include_paths): - hp.environment_init_add_include_path(path) - hp.environment_init_finish() + hp.environment_init_add_include_path(builder, path) + return hp.environment_init_finish(builder) diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index 753c2f432..8e07846c5 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -37,6 +37,7 @@ using CSyntaxNode = CStruct; using CStepResult = CStruct; using CRunnerState = CStruct; using CMetta = CStruct; +using CEnvBuilder = CStruct; // Returns a string, created by executing a function that writes string data into a buffer typedef size_t (*write_to_buf_func_t)(void*, char*, size_t); @@ -788,6 +789,7 @@ PYBIND11_MODULE(hyperonpy, m) { return lists_of_atom; }, "Returns the in-flight results from a runner state"); + py::class_(m, "CEnvBuilder"); m.def("environment_config_dir", []() { return func_to_string_no_arg((write_to_buf_no_arg_func_t)&environment_config_dir); }, "Return the config_dir for the platform environment"); @@ -795,12 +797,12 @@ PYBIND11_MODULE(hyperonpy, m) { m.def("environment_nth_search_path", [](size_t idx) { return func_to_string((write_to_buf_func_t)&environment_nth_search_path, (void*)idx); }, "Returns the module search path at the specified index, in the environment"); - m.def("environment_init_start", []() { environment_init_start(); }, "Begin initialization of the platform environment"); - m.def("environment_init_finish", []() { environment_init_finish(); }, "Finish initialization of the platform environment"); - m.def("environment_init_set_working_dir", [](std::string path) { environment_init_set_working_dir(path.c_str()); }, "Sets the working dir in the platform environment"); - m.def("environment_init_set_config_dir", [](std::string path) { environment_init_set_config_dir(path.c_str()); }, "Sets the config dir in the platform environment"); - m.def("environment_init_disable_config_dir", []() { environment_init_disable_config_dir(); }, "Disables the config dir in the platform environment"); - m.def("environment_init_add_include_path", [](std::string path) { environment_init_add_include_path(path.c_str()); }, "Adds an include path to the platform environment"); + m.def("environment_init_start", []() { return CEnvBuilder(environment_init_start()); }, "Begin initialization of the platform environment"); + m.def("environment_init_finish", [](CEnvBuilder builder) { return environment_init_finish(builder.obj); }, "Finish initialization of the platform environment"); + m.def("environment_init_set_working_dir", [](CEnvBuilder& builder, std::string path) { environment_init_set_working_dir(builder.ptr(), path.c_str()); }, "Sets the working dir in the platform environment"); + m.def("environment_init_set_config_dir", [](CEnvBuilder& builder, std::string path) { environment_init_set_config_dir(builder.ptr(), path.c_str()); }, "Sets the config dir in the platform environment"); + m.def("environment_init_disable_config_dir", [](CEnvBuilder& builder) { environment_init_disable_config_dir(builder.ptr()); }, "Disables the config dir in the platform environment"); + m.def("environment_init_add_include_path", [](CEnvBuilder& builder, std::string path) { environment_init_add_include_path(builder.ptr(), path.c_str()); }, "Adds an include path to the platform environment"); } __attribute__((constructor)) diff --git a/python/tests/test_environment.py b/python/tests/test_environment.py index f728d604c..4bcdb18aa 100644 --- a/python/tests/test_environment.py +++ b/python/tests/test_environment.py @@ -8,5 +8,7 @@ def __init__(self, methodName): super().__init__(methodName) def testEnvironment(self): - Environment.init_platform_env(config_dir = "/tmp/test_dir") + self.assertTrue(Environment.init_platform_env(config_dir = "/tmp/test_dir")) self.assertEqual(Environment.config_dir(), "/tmp/test_dir") + + self.assertFalse(Environment.init_platform_env(disable_config = True)) From dc15fc359ebb7b7dad9014891da563a6dfda709d Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Fri, 6 Oct 2023 12:36:58 +0900 Subject: [PATCH 16/28] Extensive rework of Environment API, to permit multiple coexisting environments to be initialized and used from C & Python layers --- c/src/lib.rs | 14 --- c/src/metta.rs | 145 ++++++++++++++++++-------- c/tests/check_runner.c | 3 +- c/tests/check_space.c | 3 +- c/tests/util.c | 12 +++ c/tests/util.h | 3 + lib/src/common/mod.rs | 5 - lib/src/metta/environment.rs | 63 +++++++---- lib/src/metta/runner/mod.rs | 83 +++++++++------ lib/src/metta/runner/stdlib.rs | 14 +-- lib/src/metta/runner/stdlib2.rs | 9 +- lib/tests/case.rs | 5 +- lib/tests/metta.rs | 3 +- python/hyperon/runner.py | 46 +++++--- python/hyperonpy.cpp | 55 ++++++---- python/tests/scripts/f1_imports.metta | 28 ++--- python/tests/test_custom_space.py | 4 +- python/tests/test_examples.py | 26 ++--- python/tests/test_extend.py | 4 +- python/tests/test_grounded_type.py | 8 +- python/tests/test_grounding_space.py | 2 +- python/tests/test_metta.py | 8 +- python/tests/test_minecraft.py | 4 +- python/tests/test_minelogy.py | 14 +-- python/tests/test_pln_tv.py | 2 +- python/tests/test_run_metta.py | 54 +++++----- python/tests/test_stdlib.py | 4 +- repl/src/metta_shim.rs | 2 +- 28 files changed, 371 insertions(+), 252 deletions(-) diff --git a/c/src/lib.rs b/c/src/lib.rs index d8ac20283..968811268 100644 --- a/c/src/lib.rs +++ b/c/src/lib.rs @@ -4,17 +4,3 @@ pub mod util; pub mod atom; pub mod space; pub mod metta; - -/// @brief Initializes the logger -/// @ingroup misc_group -/// @note This function should be called once, prior to any other calls to Hyperon C functions -/// -//TODO: Is there a way we can get rid of this function in the external API? -// Discussion about alternative ways to init the logger here: https://github.com/trueagi-io/hyperon-experimental/pull/314 -// and here: https://github.com/trueagi-io/hyperon-experimental/issues/146 -//UPDATE: Logger init parameters should be moved into the Environment API. -#[no_mangle] -pub extern "C" fn init_logger() { - hyperon::common::init_logger(false); -} - diff --git a/c/src/metta.rs b/c/src/metta.rs index 3e2203fba..d91bf3857 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -725,23 +725,29 @@ impl metta_t { /// #[no_mangle] pub extern "C" fn metta_new_rust() -> metta_t { - let metta = Metta::new_rust(); + let metta = Metta::new_rust(None); metta.into() } -/// @brief Creates a new MeTTa Interpreter with a provided Space and Tokenizer +/// @brief Creates a new MeTTa Interpreter with a provided Space, Tokenizer and Environment /// @ingroup interpreter_group /// @param[in] space A pointer to a handle for the Space for use by the Interpreter /// @param[in] tokenizer A pointer to a handle for the Tokenizer for use by the Interpreter +/// @param[in] environment An `environment_t` handle representing the environment to use /// @return A `metta_t` handle to the newly created Interpreter /// @note The caller must take ownership responsibility for the returned `metta_t`, and free it with `metta_free()` /// @note This function does not load any stdlib, nor does it run the `init.metta` file from the environment /// #[no_mangle] -pub extern "C" fn metta_new_with_space(space: *mut space_t, tokenizer: *mut tokenizer_t) -> metta_t { +pub extern "C" fn metta_new_with_space(space: *mut space_t, tokenizer: *mut tokenizer_t, env_builder: env_builder_t) -> metta_t { let dyn_space = unsafe{ &*space }.borrow(); let tokenizer = unsafe{ &*tokenizer }.clone_handle(); - let metta = Metta::new_with_space(dyn_space.clone(), tokenizer); + let env_builder = if env_builder.is_default() { + None + } else { + Some(env_builder.into_inner()) + }; + let metta = Metta::new_with_space(dyn_space.clone(), tokenizer, env_builder); metta.into() } @@ -764,9 +770,45 @@ pub extern "C" fn metta_free(metta: metta_t) { /// with their own stdlib, that needs to be loaded before the init.metta file is run /// #[no_mangle] -pub extern "C" fn metta_init_with_platform_env(metta: *mut metta_t) { +pub extern "C" fn metta_init(metta: *mut metta_t) { + let metta = unsafe{ &*metta }.borrow(); + metta.init() +} + +/// @brief Returns the number of module search paths that will be searched when importing modules into +/// the runner +/// @ingroup interpreter_group +/// @param[in] metta A pointer to the Interpreter handle +/// @return The number of paths that will be searched before the search is considered unsuccessful +/// +#[no_mangle] +pub extern "C" fn metta_search_path_cnt(metta: *const metta_t) -> usize { let metta = unsafe{ &*metta }.borrow(); - metta.init_with_platform_env() + metta.search_paths().count() +} + +/// @brief Renders nth module search path for a given runner into a text buffer +/// @ingroup interpreter_group +/// @param[in] idx The index of the search path to render, with 0 being the highest priority search +/// path, and subsequent indices having decreasing search priority. +/// If `i > metta_search_path_cnt()`, this function will return 0. +/// @param[out] buf A buffer into which the text will be written +/// @param[in] buf_len The maximum allocated size of `buf` +/// @return The length of the path string, minus the string terminator character. If +/// `return_value > buf_len + 1`, then the text was not fully written and this function should be +/// called again with a larger buffer. This function will return 0 if the input arguments don't +/// specify a valid search path. +/// @note This function is primarily useful for implementing additional language-specific module-loading +/// logic, such as the `extend-py!` operation in the Python MeTTa extensions. +/// +#[no_mangle] +pub extern "C" fn metta_nth_search_path(metta: *const metta_t, idx: usize, buf: *mut c_char, buf_len: usize) -> usize { + let metta = unsafe{ &*metta }.borrow(); + let path = metta.search_paths().nth(idx); + match path { + Some(path) => write_into_buf(path.display(), buf, buf_len), + None => 0 + } } /// @brief Provides access to the Space associated with a MeTTa Interpreter @@ -959,37 +1001,7 @@ pub extern "C" fn environment_config_dir(buf: *mut c_char, buf_len: usize) -> us } } -/// @brief Returns the number of module search paths in the environment -/// @ingroup environment_group -/// @return The number of paths in the Environment -/// -#[no_mangle] -pub extern "C" fn environment_search_path_cnt() -> usize { - Environment::platform_env().modules_search_paths().count() -} - -/// @brief Renders nth module search path from the platform environment into a text buffer -/// @ingroup environment_group -/// @param[in] idx The index of the search path to render, with 0 being the highest priority search -/// path, and subsequent indices having decreasing search priority. -/// If `i > environment_search_path_cnt()`, this function will return 0. -/// @param[out] buf A buffer into which the text will be written -/// @param[in] buf_len The maximum allocated size of `buf` -/// @return The length of the path string, minus the string terminator character. If -/// `return_value > buf_len + 1`, then the text was not fully written and this function should be -/// called again with a larger buffer. This function will return 0 if the input arguments don't -/// specify a valid search path. -/// -#[no_mangle] -pub extern "C" fn environment_nth_search_path(idx: usize, buf: *mut c_char, buf_len: usize) -> usize { - let path = Environment::platform_env().modules_search_paths().nth(idx); - match path { - Some(path) => write_into_buf(path.display(), buf, buf_len), - None => 0 - } -} - -/// @brief Represents the environment initialization, in progress +/// @brief Represents an environment initialization, in progress /// @ingroup environment_group /// @note `env_builder_t` must be given to `environment_init_finish()` to properly release it /// @@ -1008,7 +1020,13 @@ impl From for env_builder_t { } impl env_builder_t { + fn is_default(&self) -> bool { + self.builder.is_null() + } fn into_inner(self) -> EnvBuilder { + if self.is_default() { + panic!("Fatal Error, default env_builder_t cannot be accessed") + } unsafe{ Box::from_raw(self.builder).0 } } fn null() -> Self { @@ -1016,16 +1034,40 @@ impl env_builder_t { } } -/// @brief Begins the initialization of the platform environment +/// @brief Begins initialization of an environment /// @ingroup environment_group /// @return The `env_builder_t` object representing the in-process environment initialization -/// @note environment_init_finish must be called after environment initialization is finished +/// @note The `env_builder_t` must be passed to either `env_builder_init_platform_env` +/// or `metta_new_with_space` in order to properly deallocate it /// #[no_mangle] -pub extern "C" fn environment_init_start() -> env_builder_t { +pub extern "C" fn env_builder_start() -> env_builder_t { EnvBuilder::new().into() } +/// @brief Creates an `env_builder_t` to specify that the default platform environment should be used +/// @ingroup environment_group +/// @return The `env_builder_t` object specifying the default platform environment +/// @note This function exists to supply an argument to `metta_new_with_space` when no special +/// behavior is desired +/// @note The `env_builder_t` must be passed to `metta_new_with_space` +/// +#[no_mangle] +pub extern "C" fn env_builder_use_default() -> env_builder_t { + env_builder_t::null() +} + +/// @brief A convenience to create an `env_builder_t`, to specify that a unit-test environment should be used +/// @ingroup environment_group +/// @return The `env_builder_t` object specifying the unit test environment +/// @note This function exists to supply an argument to `metta_new_with_space` when performing unit testing +/// @note The `env_builder_t` must be passed to `metta_new_with_space` +/// +#[no_mangle] +pub extern "C" fn env_builder_use_test_env() -> env_builder_t { + EnvBuilder::test_env().into() +} + /// @brief Finishes initialization of the platform environment /// @ingroup environment_group /// @param[in] builder The in-process environment builder state to install as the platform environment @@ -1033,7 +1075,7 @@ pub extern "C" fn environment_init_start() -> env_builder_t { /// if the environment had already been initialized by a prior call /// #[no_mangle] -pub extern "C" fn environment_init_finish(builder: env_builder_t) -> bool { +pub extern "C" fn env_builder_init_platform_env(builder: env_builder_t) -> bool { let builder = builder.into_inner(); builder.try_init_platform_env().is_ok() } @@ -1047,7 +1089,7 @@ pub extern "C" fn environment_init_finish(builder: env_builder_t) -> bool { /// it will not change as the process' working directory is changed /// #[no_mangle] -pub extern "C" fn environment_init_set_working_dir(builder: *mut env_builder_t, path: *const c_char) { +pub extern "C" fn env_builder_set_working_dir(builder: *mut env_builder_t, path: *const c_char) { let builder_arg_ref = unsafe{ &mut *builder }; let builder = core::mem::replace(builder_arg_ref, env_builder_t::null()).into_inner(); let builder = if path.is_null() { @@ -1065,7 +1107,7 @@ pub extern "C" fn environment_init_set_working_dir(builder: *mut env_builder_t, /// @param[in] path A C-style string specifying a path to a working directory, to search for modules to load /// #[no_mangle] -pub extern "C" fn environment_init_set_config_dir(builder: *mut env_builder_t, path: *const c_char) { +pub extern "C" fn env_builder_set_config_dir(builder: *mut env_builder_t, path: *const c_char) { let builder_arg_ref = unsafe{ &mut *builder }; let builder = core::mem::replace(builder_arg_ref, env_builder_t::null()).into_inner(); let builder = if path.is_null() { @@ -1081,13 +1123,26 @@ pub extern "C" fn environment_init_set_config_dir(builder: *mut env_builder_t, p /// @param[in] builder A pointer to the in-process environment builder state /// #[no_mangle] -pub extern "C" fn environment_init_disable_config_dir(builder: *mut env_builder_t) { +pub extern "C" fn env_builder_disable_config_dir(builder: *mut env_builder_t) { let builder_arg_ref = unsafe{ &mut *builder }; let builder = core::mem::replace(builder_arg_ref, env_builder_t::null()).into_inner(); let builder = builder.set_no_config_dir(); *builder_arg_ref = builder.into(); } +/// @brief Configures the platform environment for use in unit testing +/// @ingroup environment_group +/// @param[in] builder A pointer to the in-process environment builder state +/// @param[in] is_test True if the environment is a unit-test environment, False otherwise +/// +#[no_mangle] +pub extern "C" fn env_builder_set_is_test(builder: *mut env_builder_t, is_test: bool) { + let builder_arg_ref = unsafe{ &mut *builder }; + let builder = core::mem::replace(builder_arg_ref, env_builder_t::null()).into_inner(); + let builder = builder.set_is_test(is_test); + *builder_arg_ref = builder.into(); +} + /// @brief Adds a config directory to search for imports. The most recently added paths will be searched /// first, continuing in inverse order /// @ingroup environment_group @@ -1095,7 +1150,7 @@ pub extern "C" fn environment_init_disable_config_dir(builder: *mut env_builder_ /// @param[in] path A C-style string specifying a path to a working directory, to search for modules to load /// #[no_mangle] -pub extern "C" fn environment_init_add_include_path(builder: *mut env_builder_t, path: *const c_char) { +pub extern "C" fn env_builder_add_include_path(builder: *mut env_builder_t, path: *const c_char) { let builder_arg_ref = unsafe{ &mut *builder }; let builder = core::mem::replace(builder_arg_ref, env_builder_t::null()).into_inner(); let builder = if path.is_null() { diff --git a/c/tests/check_runner.c b/c/tests/check_runner.c index 9ed3d05d9..3fccf792e 100644 --- a/c/tests/check_runner.c +++ b/c/tests/check_runner.c @@ -22,8 +22,7 @@ void copy_atom_vec(const atom_vec_t* atoms, void* context) { START_TEST (test_incremental_runner) { - metta_t runner = metta_new_rust(); - metta_load_module(&runner, "stdlib"); + metta_t runner = new_test_metta(); runner_state_t runner_state = metta_start_run(&runner); diff --git a/c/tests/check_space.c b/c/tests/check_space.c index 76fcd7824..e45560622 100644 --- a/c/tests/check_space.c +++ b/c/tests/check_space.c @@ -228,8 +228,7 @@ START_TEST (test_space_nested_in_atom) space_add(&nested, expr(atom_sym("A"), atom_sym("B"), atom_ref_null())); atom_t space_atom = atom_gnd_for_space(&nested); - metta_t runner = metta_new_rust(); - metta_load_module(&runner, "stdlib"); + metta_t runner = new_test_metta(); tokenizer_t tokenizer = metta_tokenizer(&runner); tokenizer_register_token(&tokenizer, "nested", &TOKEN_API_CLONE_ATOM, &space_atom); diff --git a/c/tests/util.c b/c/tests/util.c index a88be2311..2c7b0a29c 100644 --- a/c/tests/util.c +++ b/c/tests/util.c @@ -29,3 +29,15 @@ atom_t expr(atom_t atom, ...) { va_end(ap); return atom_expr(children, argno); } + +metta_t new_test_metta(void) { + + space_t space = space_new_grounding_space(); + tokenizer_t tokenizer = tokenizer_new(); + metta_t metta = metta_new_with_space(&space, &tokenizer, env_builder_use_test_env()); + metta_load_module(&metta, "stdlib"); + metta_init(&metta); + space_free(space); + tokenizer_free(tokenizer); + return metta; +} \ No newline at end of file diff --git a/c/tests/util.h b/c/tests/util.h index c23916809..29e2fe7d3 100644 --- a/c/tests/util.h +++ b/c/tests/util.h @@ -10,4 +10,7 @@ void str_to_buf(const char *str, void *context); char* stratom(atom_t const* atom); atom_t expr(atom_t atom, ...); +// A function that initializes a MeTTa runner with a test environment, similar to `metta_new_rust()` +metta_t new_test_metta(void); + #endif /* UTIL_H */ diff --git a/lib/src/common/mod.rs b/lib/src/common/mod.rs index 900a72bee..b512582dc 100644 --- a/lib/src/common/mod.rs +++ b/lib/src/common/mod.rs @@ -20,11 +20,6 @@ use std::collections::HashMap; use crate::metta::metta_atom; -//TODO: logger init params should be moved into the Environment -pub fn init_logger(is_test: bool) { - let _ = env_logger::builder().is_test(is_test).try_init(); -} - // TODO: move Operation and arithmetics under metta package as it uses metta_atom // Operation implements stateless operations as GroundedAtom. // Each operation has the only instance which is identified by unique name. diff --git a/lib/src/metta/environment.rs b/lib/src/metta/environment.rs index 3b4c08d68..52f480329 100644 --- a/lib/src/metta/environment.rs +++ b/lib/src/metta/environment.rs @@ -3,11 +3,10 @@ use std::path::{Path, PathBuf}; use std::io::Write; use std::fs; use std::borrow::Borrow; +use std::sync::Arc; use directories::ProjectDirs; -use crate::common; - /// Contains state and platform interfaces shared by all MeTTa runners. This includes config settings /// and logger /// @@ -18,17 +17,23 @@ pub struct Environment { init_metta_path: Option, working_dir: Option, extra_include_paths: Vec, + is_test: bool, } const DEFAULT_INIT_METTA: &[u8] = include_bytes!("init.default.metta"); -static PLATFORM_ENV: std::sync::OnceLock = std::sync::OnceLock::new(); +static PLATFORM_ENV: std::sync::OnceLock> = std::sync::OnceLock::new(); impl Environment { /// Returns a reference to the shared "platform" Environment pub fn platform_env() -> &'static Self { - PLATFORM_ENV.get_or_init(|| EnvBuilder::new().build_platform_env()) + PLATFORM_ENV.get_or_init(|| Arc::new(EnvBuilder::new().build())) + } + + /// Internal function to get a copy of the platform Environment's Arc ptr + pub(crate) fn platform_env_arc() -> Arc { + PLATFORM_ENV.get_or_init(|| Arc::new(EnvBuilder::new().build())).clone() } /// Returns the Path to the config dir, in an OS-specific location @@ -36,17 +41,23 @@ impl Environment { self.config_dir.as_deref() } + /// Returns the Path to the environment's working_dir + /// + /// NOTE: The Environment's working_dir is not the same as the process working directory, and + /// changing the process's working directory will not affect the environment + pub fn working_dir(&self) -> Option<&Path> { + self.working_dir.as_deref() + } + /// Returns the path to the init.metta file, that is run to initialize a MeTTa runner and customize the MeTTa environment pub fn initialization_metta_file_path(&self) -> Option<&Path> { self.init_metta_path.as_deref() } - /// Returns the search paths to look in for MeTTa modules, in search priority order - /// - /// The working_dir is always returned first - pub fn modules_search_paths<'a>(&'a self) -> impl Iterator + 'a { - [&self.working_dir].into_iter().filter_map(|opt| opt.as_deref()) - .chain(self.extra_include_paths.iter().map(|path| path.borrow())) + /// Returns the extra search paths in the environment, in search priority order. Results do not + /// include the working_dir + pub fn extra_include_paths<'a>(&'a self) -> impl Iterator + 'a { + self.extra_include_paths.iter().map(|path| path.borrow()) } /// Private "default" function @@ -56,6 +67,7 @@ impl Environment { init_metta_path: None, working_dir: None, extra_include_paths: vec![], + is_test: false, } } } @@ -88,6 +100,14 @@ impl EnvBuilder { } } + /// A convenience function to construct an environment suitable for unit tests + /// + /// The `test_env` Environment will not load or create any files. Additionally + /// this method will initialize the logger for the test environment + pub fn test_env() -> Self { + EnvBuilder::new().set_is_test(true).set_no_config_dir() + } + /// Sets (or unsets) the working_dir for the environment pub fn set_working_dir(mut self, working_dir: Option<&Path>) -> Self { self.env.working_dir = working_dir.map(|dir| dir.into()); @@ -113,6 +133,15 @@ impl EnvBuilder { self } + /// Sets the `is_test` flag for the environment, to specify whether the environment is a unit-test + /// + /// NOTE: This currently applies to the logger, but may affect other behaviors in the future. + /// See [env_logger::is_test](https://docs.rs/env_logger/latest/env_logger/struct.Builder.html#method.is_test) + pub fn set_is_test(mut self, is_test: bool) -> Self { + self.env.is_test = is_test; + self + } + /// Adds additional include paths to search for MeTTa modules /// /// NOTE: The most recently added paths will have the highest search priority, save for the `working_dir`, @@ -133,23 +162,19 @@ impl EnvBuilder { /// Initializes the shared platform Environment. Non-panicking version of [init_platform_env] pub fn try_init_platform_env(self) -> Result<(), &'static str> { - PLATFORM_ENV.set(self.build_platform_env()).map_err(|_| "Platform Environment already initialized") - } - - /// Internal function to finalize the building of the shared platform environment. Will always be called - /// from inside the init closure of a OnceLock, so it will only be called once per process. - fn build_platform_env(self) -> Environment { - common::init_logger(false); - self.build() + PLATFORM_ENV.set(Arc::new(self.build())).map_err(|_| "Platform Environment already initialized") } /// Returns a newly created Environment from the builder configuration /// /// NOTE: Creating owned Environments is usually not necessary. It is usually sufficient to use the [platform_env] method. - pub fn build(self) -> Environment { + pub(crate) fn build(self) -> Environment { let mut env = self.env; + //Init the logger. This will have no effect if the logger has already been initialized + let _ = env_logger::builder().is_test(env.is_test).try_init(); + if !self.no_cfg_dir { if env.config_dir.is_none() { match ProjectDirs::from("io", "TrueAGI", "metta") { diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index 439424b7e..33b1328e0 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -2,13 +2,15 @@ use crate::*; use crate::common::shared::Shared; use super::*; +use super::environment::EnvBuilder; use super::space::*; use super::text::{Tokenizer, SExprParser}; use super::types::validate_atom; use std::rc::Rc; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::collections::HashMap; +use std::sync::Arc; use metta::environment::Environment; @@ -48,7 +50,8 @@ pub struct MettaContents { tokenizer: Shared, settings: Shared>, modules: Shared>, - search_paths: Vec, + working_dir: Option, + environment: Arc, } #[derive(Debug, PartialEq, Eq)] @@ -66,23 +69,24 @@ pub struct RunnerState<'a> { impl Metta { - /// A 1-liner to get a MeTTa interpreter using the default configuration - /// + /// A 1-line method to create a fully initialized MeTTa interpreter + /// + /// NOTE: pass `None` for `env_builder` to use the platform environment /// NOTE: This function is appropriate for Rust or C clients, but if other language-specific /// stdlibs are involved then see the documentation for [Metta::init] - pub fn new_rust() -> Metta { + pub fn new_rust(env_builder: Option) -> Metta { let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), - Shared::new(Tokenizer::new())); + Shared::new(Tokenizer::new()), env_builder); metta.load_module(PathBuf::from("stdlib")).expect("Could not load stdlib"); - metta.init_with_platform_env(); + metta.init(); metta } /// Performs initialization of a MeTTa interpreter. Presently this involves running the `init.metta` - /// file from the platform environment. + /// file from the associated environment. /// - /// DISCUSSION: Creating a fully-initialized MeTTa environment should usually be done with with - /// a top-level initialization function, such as [new_metta_rust] or `MeTTa.new()` in Python. + /// DISCUSSION: Creating a fully-initialized MeTTa runner should usually be done with with + /// a top-level initialization function, such as [Metta::new_rust] or `MeTTa()` in Python. /// /// Doing it manually involves several steps: /// 1. Create the MeTTa runner, using [new_with_space]. This provides a working interpreter, but @@ -94,19 +98,31 @@ impl Metta { /// /// TODO: When we are able to load the Rust stdlib before the Python stdlib, which requires value-bridging, /// we can refactor this function to load the appropriate stdlib(s) and simplify the init process - pub fn init_with_platform_env(&self) { - if let Some(init_meta_file) = Environment::platform_env().initialization_metta_file_path() { + pub fn init(&self) { + if let Some(init_meta_file) = self.0.environment.initialization_metta_file_path() { self.load_module(init_meta_file.into()).unwrap(); } } - /// Returns a new MeTTa interpreter, using the provided Space and Tokenizer + /// Returns a new MeTTa interpreter, using the provided Space, Tokenizer /// + /// NOTE: If `env_builder` is `None`, the platform environment will be used /// NOTE: This function does not load any stdlib atoms, nor run the [Environment]'s 'init.metta' - pub fn new_with_space(space: DynSpace, tokenizer: Shared) -> Self { + pub fn new_with_space(space: DynSpace, tokenizer: Shared, env_builder: Option) -> Self { let settings = Shared::new(HashMap::new()); let modules = Shared::new(HashMap::new()); - let contents = MettaContents{ space, tokenizer, settings, modules, search_paths: Environment::platform_env().modules_search_paths().map(|path| path.into()).collect() }; + let environment = match env_builder { + Some(env_builder) => Arc::new(env_builder.build()), + None => Environment::platform_env_arc() + }; + let contents = MettaContents{ + space, + tokenizer, + settings, + modules, + working_dir: environment.working_dir().map(|path| path.into()), + environment, + }; let metta = Self(Rc::new(contents)); register_runner_tokens(&metta); register_common_tokens(&metta); @@ -114,22 +130,17 @@ impl Metta { } /// Returns a new MeTTa interpreter intended for use loading MeTTa modules during import - fn new_loading_runner(metta: &Metta, path: PathBuf) -> Self { + fn new_loading_runner(metta: &Metta, path: &Path) -> Self { let space = DynSpace::new(GroundingSpace::new()); let tokenizer = metta.tokenizer().clone_inner(); + let environment = metta.0.environment.clone(); let settings = metta.0.settings.clone(); let modules = metta.0.modules.clone(); - //Search only the parent directory of the module we're loading - //TODO: I think we want all the same search path behavior as the parent runner, but with - // a different working_dir. Consider using the working_dir from the environment only to - // init the top-level runner, and moving search-path composition logic from the environment - // to the runner. - let mut path = path; - path.pop(); - let search_paths = vec![path]; + //Start search for sub-modules in the parent directory of the module we're loading + let working_dir = path.parent().map(|path| path.into()); - let metta = Self(Rc::new(MettaContents { space, tokenizer, settings, modules, search_paths })); + let metta = Self(Rc::new(MettaContents { space, tokenizer, settings, modules, environment, working_dir })); register_runner_tokens(&metta); metta } @@ -147,7 +158,7 @@ impl Metta { }, None => { // Load the module to the new space - let runner = Metta::new_loading_runner(self, path.clone()); + let runner = Metta::new_loading_runner(self, &path); let program = match path.to_str() { Some("stdlib") => METTA_CODE.to_string(), _ => std::fs::read_to_string(&path).map_err( @@ -188,8 +199,12 @@ impl Metta { &self.0.tokenizer } - pub fn search_paths(&self) -> &Vec { - &self.0.search_paths + /// Returns the search paths to explore for MeTTa modules, in search priority order + /// + /// The runner's working_dir is always returned first + pub fn search_paths<'a>(&'a self) -> impl Iterator + 'a { + [&self.0.working_dir].into_iter().filter_map(|opt| opt.as_deref()) + .chain(self.0.environment.extra_include_paths()) } pub fn modules(&self) -> &Shared> { @@ -366,7 +381,7 @@ mod tests { !(green Fritz) "; - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); let result = metta.run(&mut SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![Atom::sym("T")]])); } @@ -379,7 +394,7 @@ mod tests { (foo b) "; - let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new())); + let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); metta.set_setting("type-check".into(), sym!("auto")); let result = metta.run(&mut SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); @@ -393,7 +408,7 @@ mod tests { !(foo b) "; - let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new())); + let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); metta.set_setting("type-check".into(), sym!("auto")); let result = metta.run(&mut SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); @@ -430,7 +445,7 @@ mod tests { !(foo) "; - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); metta.tokenizer().borrow_mut().register_token(Regex::new("error").unwrap(), |_| Atom::gnd(ErrorOp{})); let result = metta.run(&mut SExprParser::new(program)); @@ -448,7 +463,7 @@ mod tests { !(foo a) "; - let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new())); + let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); metta.set_setting("type-check".into(), sym!("auto")); let result = metta.run(&mut SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); @@ -481,7 +496,7 @@ mod tests { !(empty) "; - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); metta.tokenizer().borrow_mut().register_token(Regex::new("empty").unwrap(), |_| Atom::gnd(ReturnAtomOp(expr!()))); let result = metta.run(&mut SExprParser::new(program)); diff --git a/lib/src/metta/runner/stdlib.rs b/lib/src/metta/runner/stdlib.rs index 5b818cb8a..4409907f7 100644 --- a/lib/src/metta/runner/stdlib.rs +++ b/lib/src/metta/runner/stdlib.rs @@ -16,6 +16,7 @@ use std::cell::RefCell; use std::fmt::Display; use std::collections::HashMap; use std::iter::FromIterator; +use std::path::PathBuf; use regex::Regex; use super::arithmetics::*; @@ -82,8 +83,8 @@ impl Grounded for ImportOp { if let Atom::Symbol(file) = file { //Check each include directory in order, until we find the module we're looking for - for include_dir in self.metta.search_paths().iter() { - let mut path = include_dir.clone(); + for include_dir in self.metta.search_paths() { + let mut path: PathBuf = include_dir.into(); path.push(file.name()); path = path.canonicalize().unwrap_or(path); if path.exists() { @@ -1206,11 +1207,12 @@ pub static METTA_CODE: &'static str = " #[cfg(test)] mod tests { use super::*; + use crate::metta::environment::EnvBuilder; use crate::metta::runner::Metta; use crate::metta::types::validate_atom; fn run_program(program: &str) -> Result>, String> { - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); metta.run(&mut SExprParser::new(program)) } @@ -1423,7 +1425,7 @@ mod tests { #[test] fn superpose_op_multiple_interpretations() { - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); let mut parser = SExprParser::new(" (= (f) A) (= (f) B) @@ -1439,7 +1441,7 @@ mod tests { #[test] fn superpose_op_superposed_with_collapse() { - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); let mut parser = SExprParser::new(" (= (f) A) (= (f) B) @@ -1453,7 +1455,7 @@ mod tests { #[test] fn superpose_op_consumes_interpreter_errors() { - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); let mut parser = SExprParser::new(" (: f (-> A B)) (= (f $x) $x) diff --git a/lib/src/metta/runner/stdlib2.rs b/lib/src/metta/runner/stdlib2.rs index fd11581ba..3756d9272 100644 --- a/lib/src/metta/runner/stdlib2.rs +++ b/lib/src/metta/runner/stdlib2.rs @@ -411,12 +411,13 @@ pub static METTA_CODE: &'static str = include_str!("stdlib.metta"); #[cfg(test)] mod tests { use super::*; + use crate::metta::environment::EnvBuilder; use crate::matcher::atoms_are_equivalent; use std::convert::TryFrom; fn run_program(program: &str) -> Result>, String> { - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); metta.run(&mut SExprParser::new(program)) } @@ -618,7 +619,7 @@ mod tests { #[test] fn metta_assert_equal_op() { - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); let assert = AssertEqualOp::new(metta.space().clone()); let program = " (= (foo $x) $x) @@ -638,7 +639,7 @@ mod tests { #[test] fn metta_assert_equal_to_result_op() { - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); let assert = AssertEqualToResultOp::new(metta.space().clone()); let program = " (= (foo) A) @@ -843,7 +844,7 @@ mod tests { !(eval (interpret (id_a myAtom) %Undefined% &self)) "; - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); metta.tokenizer().borrow_mut().register_token(Regex::new("id_num").unwrap(), |_| Atom::gnd(ID_NUM)); diff --git a/lib/tests/case.rs b/lib/tests/case.rs index d3ad851a7..85c5fa363 100644 --- a/lib/tests/case.rs +++ b/lib/tests/case.rs @@ -1,10 +1,11 @@ use hyperon::assert_eq_metta_results; use hyperon::metta::text::SExprParser; use hyperon::metta::runner::Metta; +use hyperon::metta::environment::EnvBuilder; #[test] fn test_case_operation() { - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); let result = metta.run(&mut SExprParser::new(" ; cases are processed sequentially !(case (+ 1 5) @@ -39,7 +40,7 @@ fn test_case_operation() { ")); assert_eq!(result, expected); - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); let result = metta.run(&mut SExprParser::new(" (Rel-P A B) (Rel-Q A C) diff --git a/lib/tests/metta.rs b/lib/tests/metta.rs index f4b750344..6862f5e17 100644 --- a/lib/tests/metta.rs +++ b/lib/tests/metta.rs @@ -4,6 +4,7 @@ use hyperon::metta::runner::stdlib::UNIT_ATOM; use hyperon::metta::runner::stdlib2::UNIT_ATOM; use hyperon::metta::text::*; use hyperon::metta::runner::Metta; +use hyperon::metta::environment::EnvBuilder; #[test] fn test_reduce_higher_order() { @@ -17,7 +18,7 @@ fn test_reduce_higher_order() { !(assertEqualToResult ((inc) 2) (3)) "; - let metta = Metta::new_rust(); + let metta = Metta::new_rust(Some(EnvBuilder::test_env())); let result = metta.run(&mut SExprParser::new(program)); diff --git a/python/hyperon/runner.py b/python/hyperon/runner.py index a8e7b7afa..888e43bfe 100644 --- a/python/hyperon/runner.py +++ b/python/hyperon/runner.py @@ -5,6 +5,7 @@ import hyperonpy as hp from .atoms import Atom, AtomType, OperationAtom from .base import GroundingSpaceRef, Tokenizer, SExprParser +from hyperonpy import EnvBuilder class RunnerState: def __init__(self, cstate): @@ -32,21 +33,23 @@ def current_results(self, flat=False): class MeTTa: """This class contains the MeTTa program execution utilities""" - def __init__(self, space = None, cmetta = None): + def __init__(self, space = None, cmetta = None, env_builder = None): if cmetta is not None: self.cmetta = cmetta else: if space is None: space = GroundingSpaceRef() tokenizer = Tokenizer() - self.cmetta = hp.metta_new(space.cspace, tokenizer.ctokenizer) + if env_builder is None: + env_builder = hp.env_builder_use_default() + self.cmetta = hp.metta_new(space.cspace, tokenizer.ctokenizer, env_builder) self.load_py_module("hyperon.stdlib") hp.metta_load_module(self.cmetta, "stdlib") self.register_atom('extend-py!', OperationAtom('extend-py!', lambda name: self.load_py_module_from_mod_or_file(name) or [], [AtomType.UNDEFINED, AtomType.ATOM], unwrap=False)) - hp.metta_init_with_platform_env(self.cmetta) + hp.metta_init(self.cmetta) def __del__(self): hp.metta_free(self.cmetta) @@ -110,11 +113,11 @@ def load_py_module_from_mod_or_file(self, mod_name): file_name = mod_name + ".py" # Check each search path directory in order, until we find the module we're looking for - num_search_paths = hp.environment_search_path_cnt() + num_search_paths = hp.metta_search_path_cnt(self.cmetta) search_path_idx = 0 found_path = None while (search_path_idx < num_search_paths): - search_path = hp.environment_nth_search_path(search_path_idx) + search_path = hp.metta_nth_search_path(self.cmetta, search_path_idx) test_path = os.path.join(search_path, file_name) if (os.path.exists(test_path)): found_path = test_path @@ -173,17 +176,32 @@ class Environment: def config_dir(): """Returns the config dir in the platform environment""" - return hp.environment_config_dir() - def init_platform_env(working_dir = None, config_dir = None, disable_config = False, include_paths = []): + path = hp.environment_config_dir() + if (len(path) > 0): + return path + else: + return None + + def init_platform_env(working_dir = None, config_dir = None, disable_config = False, is_test = False, include_paths = []): """Initialize the platform environment with the supplied args""" - builder = hp.environment_init_start() + builder = Environment.custom_env(working_dir, config_dir, disable_config, is_test, include_paths) + return hp.env_builder_init_platform_env(builder) + + def test_env(): + """Returns an EnvBuilder object specifying a unit-test environment, that can be used to init a MeTTa runner""" + return hp.env_builder_use_test_env() + + def custom_env(working_dir = None, config_dir = None, disable_config = False, is_test = False, include_paths = []): + """Returns an EnvBuilder object that can be used to init a MeTTa runner, if you need multiple environments to coexist in the same process""" + builder = hp.env_builder_start() if (working_dir is not None): - hp.environment_init_set_working_dir(builder, working_dir) + hp.env_builder_set_working_dir(builder, working_dir) if (config_dir is not None): - hp.environment_init_set_config_dir(builder, config_dir) + hp.env_builder_set_config_dir(builder, config_dir) if (disable_config): - hp.environment_init_disable_config_dir(builder) + hp.env_builder_disable_config_dir(builder) + if (is_test): + hp.env_builder_set_is_test(True) for path in reversed(include_paths): - hp.environment_init_add_include_path(builder, path) - return hp.environment_init_finish(builder) - + hp.env_builder_add_include_path(builder, path) + return builder diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index 8e07846c5..67247748b 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -37,7 +37,7 @@ using CSyntaxNode = CStruct; using CStepResult = CStruct; using CRunnerState = CStruct; using CMetta = CStruct; -using CEnvBuilder = CStruct; +using EnvBuilder = CStruct; // Returns a string, created by executing a function that writes string data into a buffer typedef size_t (*write_to_buf_func_t)(void*, char*, size_t); @@ -71,6 +71,22 @@ std::string func_to_string_no_arg(write_to_buf_no_arg_func_t func) { } } +// Similar to func_to_string, but for functions that that take two args +typedef size_t (*write_to_buf_two_arg_func_t)(void*, void*, char*, size_t); +std::string func_to_string_two_args(write_to_buf_two_arg_func_t func, void* arg1, void* arg2) { + //First try with a 1K stack buffer, because that will work in the vast majority of cases + char dst_buf[1024]; + size_t len = func(arg1, arg2, dst_buf, 1024); + if (len < 1024) { + return std::string(dst_buf); + } else { + char* data = new char[len+1]; + func(arg1, arg2, data, len+1); + std::string new_string = std::string(data); + return new_string; + } +} + static void copy_atoms(const atom_vec_t* atoms, void* context) { py::list* list = static_cast(context); for (size_t i = 0; i < atom_vec_len(atoms); ++i) { @@ -757,11 +773,15 @@ PYBIND11_MODULE(hyperonpy, m) { ADD_SYMBOL(VOID, "Void"); py::class_(m, "CMetta"); - m.def("metta_new", [](CSpace space, CTokenizer tokenizer) { - return CMetta(metta_new_with_space(space.ptr(), tokenizer.ptr())); + m.def("metta_new", [](CSpace space, CTokenizer tokenizer, EnvBuilder env_builder) { + return CMetta(metta_new_with_space(space.ptr(), tokenizer.ptr(), env_builder.obj)); }, "New MeTTa interpreter instance"); m.def("metta_free", [](CMetta metta) { metta_free(metta.obj); }, "Free MeTTa interpreter"); - m.def("metta_init_with_platform_env", [](CMetta metta) { metta_init_with_platform_env(metta.ptr()); }, "Inits a MeTTa interpreter by running the init.metta file from the environment"); + m.def("metta_init", [](CMetta metta) { metta_init(metta.ptr()); }, "Inits a MeTTa interpreter by running the init.metta file from its environment"); + m.def("metta_search_path_cnt", [](CMetta metta) { return metta_search_path_cnt(metta.ptr()); }, "Returns the number of module search paths in the runner's environment"); + m.def("metta_nth_search_path", [](CMetta metta, size_t idx) { + return func_to_string_two_args((write_to_buf_two_arg_func_t)&metta_nth_search_path, metta.ptr(), (void*)idx); + }, "Returns the module search path at the specified index, in the runner's environment"); m.def("metta_space", [](CMetta metta) { return CSpace(metta_space(metta.ptr())); }, "Get space of MeTTa interpreter"); m.def("metta_tokenizer", [](CMetta metta) { return CTokenizer(metta_tokenizer(metta.ptr())); }, "Get tokenizer of MeTTa interpreter"); m.def("metta_run", [](CMetta metta, CSExprParser& parser) { @@ -789,25 +809,18 @@ PYBIND11_MODULE(hyperonpy, m) { return lists_of_atom; }, "Returns the in-flight results from a runner state"); - py::class_(m, "CEnvBuilder"); + py::class_(m, "EnvBuilder"); m.def("environment_config_dir", []() { return func_to_string_no_arg((write_to_buf_no_arg_func_t)&environment_config_dir); }, "Return the config_dir for the platform environment"); - m.def("environment_search_path_cnt", []() { return environment_search_path_cnt(); }, "Returns the number of module search paths in the environment"); - m.def("environment_nth_search_path", [](size_t idx) { - return func_to_string((write_to_buf_func_t)&environment_nth_search_path, (void*)idx); - }, "Returns the module search path at the specified index, in the environment"); - m.def("environment_init_start", []() { return CEnvBuilder(environment_init_start()); }, "Begin initialization of the platform environment"); - m.def("environment_init_finish", [](CEnvBuilder builder) { return environment_init_finish(builder.obj); }, "Finish initialization of the platform environment"); - m.def("environment_init_set_working_dir", [](CEnvBuilder& builder, std::string path) { environment_init_set_working_dir(builder.ptr(), path.c_str()); }, "Sets the working dir in the platform environment"); - m.def("environment_init_set_config_dir", [](CEnvBuilder& builder, std::string path) { environment_init_set_config_dir(builder.ptr(), path.c_str()); }, "Sets the config dir in the platform environment"); - m.def("environment_init_disable_config_dir", [](CEnvBuilder& builder) { environment_init_disable_config_dir(builder.ptr()); }, "Disables the config dir in the platform environment"); - m.def("environment_init_add_include_path", [](CEnvBuilder& builder, std::string path) { environment_init_add_include_path(builder.ptr(), path.c_str()); }, "Adds an include path to the platform environment"); -} - -__attribute__((constructor)) -static void init_library() { - // TODO: integrate Rust logs with Python logger - init_logger(); + m.def("env_builder_start", []() { return EnvBuilder(env_builder_start()); }, "Begin initialization of the environment"); + m.def("env_builder_use_default", []() { return EnvBuilder(env_builder_use_default()); }, "Use the platform environment"); + m.def("env_builder_use_test_env", []() { return EnvBuilder(env_builder_use_test_env()); }, "Use an environment for unit testing"); + m.def("env_builder_init_platform_env", [](EnvBuilder builder) { return env_builder_init_platform_env(builder.obj); }, "Finish initialization of the platform environment"); + m.def("env_builder_set_working_dir", [](EnvBuilder& builder, std::string path) { env_builder_set_working_dir(builder.ptr(), path.c_str()); }, "Sets the working dir in the platform environment"); + m.def("env_builder_set_config_dir", [](EnvBuilder& builder, std::string path) { env_builder_set_config_dir(builder.ptr(), path.c_str()); }, "Sets the config dir in the platform environment"); + m.def("env_builder_disable_config_dir", [](EnvBuilder& builder) { env_builder_disable_config_dir(builder.ptr()); }, "Disables the config dir in the platform environment"); + m.def("env_builder_set_is_test", [](EnvBuilder& builder, bool is_test) { env_builder_set_is_test(builder.ptr(), is_test); }, "Disables the config dir in the platform environment"); + m.def("env_builder_add_include_path", [](EnvBuilder& builder, std::string path) { env_builder_add_include_path(builder.ptr(), path.c_str()); }, "Adds an include path to the platform environment"); } diff --git a/python/tests/scripts/f1_imports.metta b/python/tests/scripts/f1_imports.metta index 68c05b80d..be40786ac 100644 --- a/python/tests/scripts/f1_imports.metta +++ b/python/tests/scripts/f1_imports.metta @@ -1,19 +1,15 @@ -; NOTE: this behavior can change in the future +; NOTE: This test won't work outside the test environment because it relies on +; specific atoms in a specific order in the space, and loading the default environment's +; init.metta will break the assumptions in this test ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; At the very beginning of the main script, the space -; contains an atom wrapping the space of stdlib. +; Even at the very beginning of the main script `(get-atoms &self)` +; returns one atom, which wraps the space of stdlib. ; The type of this atom is the same as of `&self` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -; QUESTION: Are space implementations really required to preserve ordering of atoms? -; QUESTION / TODO: Is there a better way to find this sub-space than `get-atoms`? -; Now that we load the init.metta file as a module, the previous comment about the -; &self space containing only one atom at the beginning is no longer true !(assertEqual - (let* (($x (collapse (get-atoms &self))) - ($y (car-atom $x))) - (get-type $y)) - (get-type &self)) + ((let $x (get-atoms &self) (get-type $x))) + ((get-type &self))) ; stdlib is already loaded !(assertEqual @@ -26,7 +22,7 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; !(import! &m f1_moduleA.metta) -; Its first atom is a space +; It's first atom is a space !(assertEqual (let* (($x (collapse (get-atoms &m))) ($y (car-atom $x))) @@ -59,17 +55,15 @@ !(assertEqual (g 2) 102) !(assertEqual (f 2) 103) -; `&self` contains 4 atoms-spaces now: +; `&self` contains 3 atoms-spaces now: ; - stdlib -; - init.metta ; - moduleC imported by moduleA and removed from A after its import to &self ; - moduleA itself, which is the same as &m !(assertEqual &m (let* (($a (collapse (get-atoms &self))) ($x (cdr-atom $a)) - ($y (cdr-atom $x)) - ($z (cdr-atom $y))) - (car-atom $z))) + ($y (cdr-atom $x))) + (car-atom $y))) ; NOTE: now the first atom, which was a space, is removed from `&m`, ; because we load modules only once, and we collect atoms-spaces to diff --git a/python/tests/test_custom_space.py b/python/tests/test_custom_space.py index 46b85d97c..838d53c83 100644 --- a/python/tests/test_custom_space.py +++ b/python/tests/test_custom_space.py @@ -107,7 +107,7 @@ def test_query(self): self.assertEqualNoOrder(result, [{"x": S("B")}, {"x": S("E")}]) def test_atom_containing_space(self): - m = MeTTa() + m = MeTTa(env_builder=Environment.test_env()) # Make a little space and add it to the MeTTa interpreter's space little_space = SpaceRef(TestSpace()) @@ -131,7 +131,7 @@ def test_match_nested_custom_space(self): nested.add_atom(E(S("A"), S("B"))) space_atom = G(nested) - runner = MeTTa() + runner = MeTTa(env_builder=Environment.test_env()) runner.tokenizer().register_token("nested", lambda token: space_atom) result = runner.run("!(match nested (A $x) $x)") diff --git a/python/tests/test_examples.py b/python/tests/test_examples.py index 12a662eb8..58d907551 100644 --- a/python/tests/test_examples.py +++ b/python/tests/test_examples.py @@ -6,7 +6,7 @@ class ExamplesTest(HyperonTestCase): def test_grounded_functions(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) obj = SomeObject() # using & as a prefix is not obligatory, but is naming convention metta.register_atom("&obj", ValueAtom(obj)) @@ -14,7 +14,7 @@ def test_grounded_functions(self): target = metta.parse_single('(call:foo &obj)') # interpreting this target in another space still works, # because substitution '&obj' -> obj is done by metta - metta2 = MeTTa() + metta2 = MeTTa(env_builder=Environment.test_env()) result = interpret(metta2.space(), target) self.assertTrue(obj.called) self.assertEqual(result, []) @@ -28,7 +28,7 @@ def test_grounded_functions(self): # interpretation. @unittest.skip("TODO") def test_self_modify(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) metta.run( ''' (= (remove-state $var) @@ -48,7 +48,7 @@ def test_self_modify(self): [[S('Sam')]]) def test_new_object(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) pglob = Global(10) ploc = 10 metta.register_token("pglob", lambda _: ValueAtom(pglob)) @@ -93,7 +93,7 @@ def test_new_object(self): self.assertEqual(ploca.get_object().value, 5) def test_frog_reasoning(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) metta.run(''' (= (Fritz croaks) True) @@ -111,7 +111,7 @@ def test_frog_reasoning(self): metta.run('!(if ($x frog) (= ($x green) True) nop)')) def test_infer_function_application_type(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) metta.run(''' (= (: (apply $f $x) $r) (and (: $f (=> $a $r)) (: $x $a))) @@ -124,7 +124,7 @@ def test_infer_function_application_type(self): self.assertEqualMettaRunnerResults(output, [[S('String')]]) def test_plus_reduces_Z(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) metta.run(''' (= (eq $x $x) True) @@ -152,14 +152,14 @@ def test_multi_space(self): # explicitly from another space should be safe, though) # NOTE: these tests are not indended to remain valid, but are needed to # detect, if something is changes in the interpreter - metta1 = MeTTa() + metta1 = MeTTa(env_builder=Environment.test_env()) metta1.run(''' (eq A B) (= (f-in-s2) failure) (= (how-it-works?) (f-in-s2)) (= (inverse $x) (match &self (eq $y $x) $y)) ''') - metta2 = MeTTa() + metta2 = MeTTa(env_builder=Environment.test_env()) metta2.register_atom("&space1", metta1.run("! &self")[0][0]) metta2.run(''' (eq C B) @@ -180,7 +180,7 @@ def test_multi_space(self): self.assertEqualMettaRunnerResults(metta1.run('!(how-it-works?)'), [[S('failure')]]) def test_custom_deptypes(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) metta.run(''' (= (:? $c) (match &self (:= $c $t) $t)) @@ -256,7 +256,7 @@ def test_custom_deptypes(self): # expression, since HumansAreMortal expects an element of (Human Socrates) - not the type itself # Another syntax - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) metta.run(''' (= (:? $c) (match &self (:: $c $t) $t)) @@ -313,7 +313,7 @@ def test_custom_deptypes(self): def test_visit_kim(self): # legacy test # can be moved to b4_nondeterm.metta or removed - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) metta.run(''' (= (perform (visit $x)) (perform (lunch-order $x))) (= (perform (visit $x)) (perform (health-check $x))) @@ -344,7 +344,7 @@ def test_visit_kim(self): [metta.parse_all('False True')]) def test_char_vs_string(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) self.assertEqual(repr(metta.run("!('A')")), "[[('A')]]") self.assertEqual(repr(metta.run('!("A")')), '[[("A")]]') self.assertEqualMettaRunnerResults(metta.run("!(get-type 'A')"), [[S('Char')]]) diff --git a/python/tests/test_extend.py b/python/tests/test_extend.py index 4f2b0d04b..2d5d3be77 100644 --- a/python/tests/test_extend.py +++ b/python/tests/test_extend.py @@ -8,7 +8,7 @@ def test_extend(self): ''' This test verifies that extend-py! along with @register_atoms and @register_tokens works ''' - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) self.assertEqual( metta.run(''' !(extend-py! extension) @@ -33,7 +33,7 @@ def test_extend_global(self): from extension import g_object # Sanity check self.assertEqual(g_object, None) - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) metta.run(''' !(extend-py! extension) !(set-global! 42) diff --git a/python/tests/test_grounded_type.py b/python/tests/test_grounded_type.py index 546dcb240..24ed6ad3e 100644 --- a/python/tests/test_grounded_type.py +++ b/python/tests/test_grounded_type.py @@ -5,7 +5,7 @@ class GroundedTypeTest(unittest.TestCase): def test_apply_type(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) self.assertEqual( metta.parse_single("+").get_grounded_type(), metta.parse_single("*").get_grounded_type()) @@ -31,7 +31,7 @@ def test_apply_type(self): metta.run("!(+ 1 1)")[0][0].get_grounded_type()) def test_higher_func(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) metta.register_atom( r"curry_num", OperationAtom( @@ -47,7 +47,7 @@ def test_higher_func(self): metta.run("! 3")) def test_meta_types(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) ### Basic functional types metta.register_atom(r"id_num", OperationAtom("id_num", lambda x: x, ['Number', 'Number'])) metta.register_atom(r"as_int", OperationAtom("as_int", lambda x: x, ['Number', 'Int'])) @@ -134,7 +134,7 @@ def test_meta_types(self): # (-> *Undefined Undefined) in the future. @unittest.skip("Behavior to be defined") def test_undefined_operation_type(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) metta.register_atom("untyped", ValueAtom(None)) metta.register_atom("untop", OperationAtom("untop", lambda: None)) self.assertNotEqual(metta.parse_single("untop").get_grounded_type(), diff --git a/python/tests/test_grounding_space.py b/python/tests/test_grounding_space.py index ff2182cfb..a663a9f3a 100644 --- a/python/tests/test_grounding_space.py +++ b/python/tests/test_grounding_space.py @@ -46,7 +46,7 @@ def test_match_nested_grounding_space(self): nested.add_atom(E(S("A"), S("B"))) space_atom = G(nested) - runner = MeTTa() + runner = MeTTa(env_builder=Environment.test_env()) runner.space().add_atom(space_atom) runner.tokenizer().register_token("nested", lambda token: space_atom) diff --git a/python/tests/test_metta.py b/python/tests/test_metta.py index 1e0ee29e6..a40807b5d 100644 --- a/python/tests/test_metta.py +++ b/python/tests/test_metta.py @@ -5,7 +5,7 @@ class MettaTest(unittest.TestCase): def test_adding_tokens_while_parsing(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) atom = metta.parse_single('(A B)') self.assertEqual(atom, E(S('A'), S('B'))) @@ -29,7 +29,7 @@ def test_metta_runner(self): (= (green $x) (frog $x)) !(green Fritz) ''' - runner = MeTTa() + runner = MeTTa(env_builder=Environment.test_env()) result = runner.run(program) self.assertEqual([[S('T')]], result) @@ -38,7 +38,7 @@ def test_incremental_runner(self): program = ''' !(+ 1 (+ 2 (+ 3 4))) ''' - runner = MeTTa() + runner = MeTTa(env_builder=Environment.test_env()) runner_state = runner.start_run(program) step_count = 0 @@ -53,7 +53,7 @@ def test_gnd_type_error(self): program = ''' !(+ 2 "String") ''' - runner = MeTTa() + runner = MeTTa(env_builder=Environment.test_env()) result = runner.run(program) self.assertEqual([[E(S('Error'), ValueAtom('String'), S('BadType'))]], result) diff --git a/python/tests/test_minecraft.py b/python/tests/test_minecraft.py index 279b1bbdf..b692eb5b8 100644 --- a/python/tests/test_minecraft.py +++ b/python/tests/test_minecraft.py @@ -33,7 +33,7 @@ def newMineOp(inventory): class MinecraftTest(unittest.TestCase): def test_minecraft_planning(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) inventory = [S('inventory'), S('hands')] metta.register_token("in-inventory", lambda _: newInInventory(inventory)) metta.register_token("craft", lambda _: newCraftOp(inventory)) @@ -68,7 +68,7 @@ def test_minecraft_planning(self): self.assertTrue(S('wooden-pickaxe') in inventory) def test_minecraft_planning_with_abstractions(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) inventory = [S('inventory'), S('hands'), S('crafting-table'), S('stick'), S('iron-ingot'), S('iron-pickaxe')] diff --git a/python/tests/test_minelogy.py b/python/tests/test_minelogy.py index 810a1a065..3a575c087 100644 --- a/python/tests/test_minelogy.py +++ b/python/tests/test_minelogy.py @@ -10,7 +10,7 @@ def test_minelogy(self): # A nearly direct reimplementation of minelogy as it # was in the minecraft demo. Not optimal representation - # just testing. - mines = MeTTa() + mines = MeTTa(env_builder=Environment.test_env()) mines.run(''' (((: log type) (: $x variant)) (: (stone_axe wooden_axe None) tools) @@ -33,7 +33,7 @@ def test_minelogy(self): ((: stone type) (: $x variant)) ) ''') - crafts = MeTTa() + crafts = MeTTa(env_builder=Environment.test_env()) crafts.run(''' (((: log type) (: $x variant) (: 1 quantity)) ((: planks type) (: $x variant) (: 4 quantity))) @@ -43,7 +43,7 @@ def test_minelogy(self): ((: planks type) (: $y variant) (: 3 quantity)))) ((: wooden_pickaxe type) (: $_ variant) (: 1 quantity))) ''') - utils = MeTTa() + utils = MeTTa(env_builder=Environment.test_env()) utils.register_atom("&mines", mines.run("! &self")[0][0]) utils.register_atom("&crafts", crafts.run("! &self")[0][0]) utils.run(''' @@ -114,11 +114,11 @@ def test_minelogy(self): '[(do-mine ((: stone type) (: stone variant)))]') output = utils.run('!(how-get stick)')[0] self.assertAtomsAreEquivalent(output, - MeTTa().parse_all('(do-craft ((: planks type) (: $x variant) (: 2 quantity)))')) + MeTTa(env_builder=Environment.test_env()).parse_all('(do-craft ((: planks type) (: $x variant) (: 2 quantity)))')) def test_minelogy_wtypes(self): # TODO: revisit this example, when types are automatically checked - kb = MeTTa() + kb = MeTTa(env_builder=Environment.test_env()) kb.run(''' (: BlockT Type) (: log BlockT) @@ -169,7 +169,7 @@ def test_minelogy_wtypes(self): ((CEntityV planks $_) 3))) ((CEntityT wooden_pickaxe) 1)) ''') - utils = MeTTa() + utils = MeTTa(env_builder=Environment.test_env()) utils.register_atom("&kb", kb.run("! &self")[0][0]) utils.run(''' (= (get-mine-block $t) @@ -216,7 +216,7 @@ def test_minelogy_wtypes(self): ) output = utils.run('!(get-ingredients wooden_pickaxe)')[0] self.assertAtomsAreEquivalent(output, - MeTTa().parse_all('(list ((CEntityT stick) 2) ((CEntityV planks $_) 3))')) + MeTTa(env_builder=Environment.test_env()).parse_all('(list ((CEntityT stick) 2) ((CEntityV planks $_) 3))')) if __name__ == "__main__": diff --git a/python/tests/test_pln_tv.py b/python/tests/test_pln_tv.py index dd0539ba3..7f9c147b7 100644 --- a/python/tests/test_pln_tv.py +++ b/python/tests/test_pln_tv.py @@ -6,7 +6,7 @@ class PLNTVTest(HyperonTestCase): def test_fuzzy_conjunction_fn(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) # `stv` as a mix of function and constructor # working through ordinary equalities metta.run(''' diff --git a/python/tests/test_run_metta.py b/python/tests/test_run_metta.py index e3063e183..30af209c1 100644 --- a/python/tests/test_run_metta.py +++ b/python/tests/test_run_metta.py @@ -1,4 +1,4 @@ -from hyperon import MeTTa, E +from hyperon import MeTTa, Environment, E from test_common import HyperonTestCase from pathlib import Path @@ -18,7 +18,7 @@ def test_run_metta(self): !(f) ''' - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) self.assertEqualMettaRunnerResults(metta.run(program), [metta.parse_all('red green blue'), metta.parse_all('5')]) @@ -30,7 +30,7 @@ def test_run_complex_query(self): !(match &self (, (A $x) (C $x)) $x) ''' - result = MeTTa().run(program) + result = MeTTa(env_builder=Environment.test_env()).run(program) self.assertEqual('[[B]]', repr(result)) def test_list_concatenation(self): @@ -46,7 +46,7 @@ def test_list_concatenation(self): !(Concat (lst1) (lst2)) ''' - result = MeTTa().run(program) + result = MeTTa(env_builder=Environment.test_env()).run(program) self.assertEqual('[[(Cons a1 (Cons a2 (Cons b1 (Cons b2 Nil))))]]', repr(result)) def test_comments(self): @@ -56,7 +56,7 @@ def test_comments(self): !(match &self (a $W) $W) ''' - result = MeTTa().run(program) + result = MeTTa(env_builder=Environment.test_env()).run(program) self.assertEqual('[[5]]', repr(result)) program = ''' @@ -65,7 +65,7 @@ def test_comments(self): &self (a $W) $W) ''' - result = MeTTa().run(program) + result = MeTTa(env_builder=Environment.test_env()).run(program) self.assertEqual('[[1]]', repr(result)) def process_exceptions(self, results): @@ -73,25 +73,25 @@ def process_exceptions(self, results): self.assertEqual(result, [E()]) def test_scripts(self): - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/a1_symbols.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/a2_opencoggy.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/a3_twoside.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/b0_chaining_prelim.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/b1_equal_chain.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/b2_backchain.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/b3_direct.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/b4_nondeterm.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/b5_types_prelim.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/c1_grounded_basic.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/c2_spaces.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/c3_pln_stv.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/d1_gadt.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/d2_higherfunc.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/d3_deptypes.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/d4_type_prop.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/d5_auto_types.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/e1_kb_write.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/e2_states.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/e3_match_states.metta")) - self.process_exceptions(MeTTa().import_file(f"{pwd}/scripts/f1_imports.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/a1_symbols.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/a2_opencoggy.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/a3_twoside.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/b0_chaining_prelim.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/b1_equal_chain.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/b2_backchain.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/b3_direct.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/b4_nondeterm.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/b5_types_prelim.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/c1_grounded_basic.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/c2_spaces.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/c3_pln_stv.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/d1_gadt.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/d2_higherfunc.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/d3_deptypes.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/d4_type_prop.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/d5_auto_types.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/e1_kb_write.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/e2_states.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/e3_match_states.metta")) + self.process_exceptions(MeTTa(env_builder=Environment.test_env()).import_file(f"{pwd}/scripts/f1_imports.metta")) diff --git a/python/tests/test_stdlib.py b/python/tests/test_stdlib.py index fc929af1f..599ea39a1 100644 --- a/python/tests/test_stdlib.py +++ b/python/tests/test_stdlib.py @@ -6,7 +6,7 @@ class StdlibTest(HyperonTestCase): def test_text_ops(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) # Check that (repr (my atom)) == "(my atom)" self.assertEqualMettaRunnerResults(metta.run("!(repr (my atom))"), [[ValueAtom("(my atom)")]]) @@ -24,7 +24,7 @@ def test_text_ops(self): [[ValueAtom("ABC")]]) def test_number_parsing(self): - metta = MeTTa() + metta = MeTTa(env_builder=Environment.test_env()) self.assertEqualMettaRunnerResults(metta.run("!(+ 1 2)"), [[ValueAtom(3)]]) self.assertEqualMettaRunnerResults(metta.run("!(+ 5.0 -2.0)"), [[ValueAtom(3.0)]]) self.assertEqualMettaRunnerResults(metta.run("!(+ 1.0e3 2.0e3)"), [[ValueAtom(3e3)]]) diff --git a/repl/src/metta_shim.rs b/repl/src/metta_shim.rs index 213908eb2..dc6be8826 100644 --- a/repl/src/metta_shim.rs +++ b/repl/src/metta_shim.rs @@ -361,7 +361,7 @@ pub mod metta_interface_mod { .init_platform_env(); let new_shim = MettaShim { - metta: Metta::new_rust(), + metta: Metta::new_rust(None), result: vec![], }; new_shim.metta.tokenizer().borrow_mut().register_token_with_regex_str("extend-py!", move |_| { Atom::gnd(ImportPyErr) }); From 457d757ea9cb5b2e46c1feec197b46c8a5072791 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Wed, 11 Oct 2023 13:06:54 -0700 Subject: [PATCH 17/28] =?UTF-8?q?Adding=20`atom=5Fvec=5Ffrom=5Flist`=20to?= =?UTF-8?q?=20hyperonpy,=20in=20order=20to=20create=20an=20atom=5Fvec=5Ft?= =?UTF-8?q?=20with=20a=20Python=20list=20of=20atoms=20Changing=20`sexpr=5F?= =?UTF-8?q?parser=5Ft`=20from=20wrapped=20Shared=20ptr=20to=20wrapped=20Bo?= =?UTF-8?q?x=20Changing=20RunnerState=20API=20to=20take=20ownership=20of?= =?UTF-8?q?=20the=20parser=20and,=20rather=20than=20asking=20the=20caller?= =?UTF-8?q?=20to=20provide=20the=20parser=20each=20step.=20=20There=20is?= =?UTF-8?q?=20no=20good=20reason=20to=20switch=20parsers=20with=20the=20sa?= =?UTF-8?q?me=20RunnerState=20and=20switching=20them=20is=20just=20asking?= =?UTF-8?q?=20for=20bugs=20Adding=20=E2=80=9Cwith=5Fatoms=E2=80=9D=20alter?= =?UTF-8?q?native=20to=20create=20a=20RunnerState?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- c/src/atom.rs | 2 +- c/src/metta.rs | 150 ++++++++++++++++-------- c/tests/check_runner.c | 7 +- c/tests/check_space.c | 3 +- lib/src/metta/interpreter2.rs | 2 + lib/src/metta/runner/mod.rs | 197 ++++++++++++++++++++------------ lib/src/metta/runner/stdlib.rs | 14 +-- lib/src/metta/runner/stdlib2.rs | 22 ++-- lib/src/metta/text.rs | 1 + lib/tests/case.rs | 8 +- lib/tests/metta.rs | 2 +- python/hyperon/base.py | 2 + python/hyperon/runner.py | 34 +++--- python/hyperonpy.cpp | 27 ++++- python/tests/test_metta.py | 2 +- repl/src/metta_shim.rs | 15 ++- 16 files changed, 306 insertions(+), 182 deletions(-) diff --git a/c/src/atom.rs b/c/src/atom.rs index c0aee58d7..f36e73ef2 100644 --- a/c/src/atom.rs +++ b/c/src/atom.rs @@ -737,7 +737,7 @@ impl atom_vec_t { fn new() -> Self { Vec::::new().into() } - fn as_slice(&self) -> &[Atom] { + pub(crate) fn as_slice(&self) -> &[Atom] { unsafe{ core::slice::from_raw_parts(self.ptr.cast(), self.len) } } /// Converts a borrowed vec into an owned vec diff --git a/c/src/metta.rs b/c/src/metta.rs index 7312bf29b..97658c195 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -11,7 +11,6 @@ use crate::util::*; use crate::atom::*; use crate::space::*; -use core::borrow::Borrow; use std::os::raw::*; use regex::Regex; use std::path::PathBuf; @@ -146,43 +145,45 @@ pub extern "C" fn tokenizer_clone(tokenizer: *const tokenizer_t) -> tokenizer_t /// @brief Represents an S-Expression Parser state machine, to parse input text into an Atom /// @ingroup tokenizer_and_parser_group -/// @note `sexpr_parser_t` handles must be freed with `sexpr_parser_free()` +/// @note `sexpr_parser_t` objects must be freed with `sexpr_parser_free()` /// #[repr(C)] pub struct sexpr_parser_t { /// Internal. Should not be accessed directly - parser: *const RustSExprParser, + parser: *mut RustSExprParser, err_string: *mut c_char, } -impl sexpr_parser_t { - fn free_err_string(&mut self) { - if !self.err_string.is_null() { - let string = unsafe{ std::ffi::CString::from_raw(self.err_string) }; - drop(string); - self.err_string = core::ptr::null_mut(); - } - } -} +struct RustSExprParser(SExprParser<'static>); -struct RustSExprParser(std::cell::RefCell>); - -impl From>> for sexpr_parser_t { - fn from(parser: Shared) -> Self { +impl From> for sexpr_parser_t { + fn from(parser: SExprParser<'static>) -> Self { Self{ - parser: std::rc::Rc::into_raw(parser.0).cast(), + parser: Box::into_raw(Box::new(RustSExprParser(parser))), err_string: core::ptr::null_mut() } } } impl sexpr_parser_t { - fn borrow_inner(&self) -> &mut SExprParser<'static> { - let cell = unsafe{ &mut (&mut *self.parser.cast_mut()).0 }; - cell.get_mut() + fn into_inner(self) -> SExprParser<'static> { + unsafe{ (*Box::from_raw(self.parser)).0 } } - fn into_handle(self) -> Shared> { - unsafe{ Shared(std::rc::Rc::from_raw(self.parser.cast())) } + fn borrow(&self) -> &SExprParser<'static> { + &unsafe{ &*self.parser }.0 + } + fn borrow_mut(&mut self) -> &mut SExprParser<'static> { + &mut unsafe{ &mut *self.parser }.0 + } +} + +impl sexpr_parser_t { + fn free_err_string(&mut self) { + if !self.err_string.is_null() { + let string = unsafe{ std::ffi::CString::from_raw(self.err_string) }; + drop(string); + self.err_string = core::ptr::null_mut(); + } } } @@ -191,12 +192,27 @@ impl sexpr_parser_t { /// @param[in] text A C-style string containing the input text to parse /// @return The new `sexpr_parser_t`, ready to parse the text /// @note The returned `sexpr_parser_t` must be freed with `sexpr_parser_free()` -/// @warning The returned `sexpr_parser_t` borrows a reference to the `text` pointer, so the returned +/// @warning The returned `sexpr_parser_t` borrows a reference to the `text`, so the returned /// `sexpr_parser_t` must be freed before the `text` is freed or allowed to go out of scope. /// #[no_mangle] pub extern "C" fn sexpr_parser_new(text: *const c_char) -> sexpr_parser_t { - Shared::new(SExprParser::new(cstr_as_str(text))).into() + SExprParser::new(cstr_as_str(text)).into() +} + +/// @brief Creates a new S-Expression Parser from an existing `sexpr_parser_t` +/// @ingroup tokenizer_and_parser_group +/// @param[in] parser The source `sexpr_parser_t` to clone +/// @return The new `sexpr_parser_t`, ready to parse the text +/// @note The returned `sexpr_parser_t` must be freed with `sexpr_parser_free()` +/// @note A cloned parser can be thought of as an independent read cursor referencing the same source text. +/// @warning The returned `sexpr_parser_t` borrows a reference to the same `text` pointer as the original, +/// so the returned `sexpr_parser_t` must be freed before the `text` is freed or allowed to go out of scope. +/// +#[no_mangle] +pub extern "C" fn sexpr_parser_clone(parser: *const sexpr_parser_t) -> sexpr_parser_t { + let parser = unsafe{ &*parser }.borrow(); + parser.clone().into() } /// @brief Frees an S-Expression Parser @@ -205,7 +221,7 @@ pub extern "C" fn sexpr_parser_new(text: *const c_char) -> sexpr_parser_t { /// #[no_mangle] pub extern "C" fn sexpr_parser_free(parser: sexpr_parser_t) { - let parser = parser.into_handle(); + let parser = parser.into_inner(); drop(parser); } @@ -224,7 +240,7 @@ pub extern "C" fn sexpr_parser_parse( { let parser = unsafe{ &mut *parser }; parser.free_err_string(); - let rust_parser = parser.borrow_inner(); + let rust_parser = parser.borrow_mut(); let tokenizer = unsafe{ &*tokenizer }.borrow_inner(); match rust_parser.parse(tokenizer) { Ok(atom) => atom.into(), @@ -284,7 +300,7 @@ impl syntax_node_t { unsafe{ (*Box::from_raw(self.node)).0 } } fn borrow(&self) -> &SyntaxNode { - &unsafe{ &*(&*self).node }.0 + &unsafe{ &*self.node }.0 } fn is_null(&self) -> bool { self.node == core::ptr::null_mut() @@ -358,7 +374,7 @@ pub extern "C" fn sexpr_parser_parse_to_syntax_tree(parser: *mut sexpr_parser_t) { let parser = unsafe{ &mut *parser }; parser.free_err_string(); - let rust_parser = parser.borrow_inner(); + let rust_parser = parser.borrow_mut(); rust_parser.parse_to_syntax_tree().into() } @@ -581,12 +597,11 @@ pub extern "C" fn get_atom_types(space: *const space_t, atom: *const atom_ref_t, // MeTTa Intperpreter Interface // =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- -//QUESTION: It feels like a runner_state_t and a step_result_t are getting at the same functionality, -// but at different levels. I think it probably makes sense to remove step_result_t from the C -// and Python APIs to cut down on API surface area - /// @brief Contains the state for an in-flight interpreter operation /// @ingroup interpreter_group +/// @note The `step_result_t` API is very low-level; It provides direct access to the interpreter +/// without a an environment, parser, or tokenizer. Usually the `runner_state_t` API is a better +/// choice for executing MeTTa code in most situations. /// @note A `step_result_t` is initially created by `interpret_init()`. Each call to `interpret_step()`, in /// a loop, consumes the `step_result_t` and creates a new one. When the interpreter operation has /// fully resolved, `step_get_result()` can provide the final results. Ownership of the `step_result_t` @@ -838,25 +853,30 @@ pub extern "C" fn metta_tokenizer(metta: *mut metta_t) -> tokenizer_t { /// @brief Runs the MeTTa Interpreter until the input text has been parsed and evaluated /// @ingroup interpreter_group /// @param[in] metta A pointer to the Interpreter handle -/// @param[in] parser A pointer to the S-Expression Parser handle, containing the expression text +/// @param[in] parser An S-Expression Parser containing the MeTTa text /// @param[in] callback A function that will be called to provide a vector of atoms produced by the evaluation /// @param[in] context A pointer to a caller-defined structure to facilitate communication with the `callback` function +/// @warning Ownership of the provided parser will be taken by this function, so it must not be subsequently accessed +/// nor freed. /// #[no_mangle] -pub extern "C" fn metta_run(metta: *mut metta_t, parser: *mut sexpr_parser_t, +pub extern "C" fn metta_run(metta: *mut metta_t, parser: sexpr_parser_t, callback: c_atom_vec_callback_t, context: *mut c_void) { let metta = unsafe{ &*metta }.borrow(); - let mut parser = unsafe{ &*parser }.borrow_inner(); - let results = metta.run(&mut parser); + let parser = parser.into_inner(); + let results = metta.run(parser); // TODO: return erorrs properly after step_get_result() is changed to return errors. for result in results.expect("Returning errors from C API is not implemented yet") { return_atoms(&result, callback, context); } } -/// @brief Represents the state of a MeTTa runner +/// @brief Represents the state of an in-flight MeTTa execution run /// @ingroup interpreter_group -/// @note `runner_state_t` handles must be freed with `runner_state_free()` +/// @note A `runner_state_t` is initially created by `runner_state_new_with_parser()`. Each call to `metta_run_step()`, in +/// a loop, advances the evaluation progress by some amount. When the interpreter operation has +/// fully resolved, `runner_state_is_complete()` will return true. Ownership of the `runner_state_t` +/// must ultimately be released with `runner_state_free()`. /// #[repr(C)] pub struct runner_state_t { @@ -884,16 +904,37 @@ impl runner_state_t { } } -/// @brief Creates a runner_state_t, to use for step-wise execution +/// @brief Creates a runner_state_t, to use for step-wise execution of MeTTa text /// @ingroup interpreter_group -/// @param[in] metta A pointer to the Interpreter handle +/// @param[in] metta A pointer to the Interpreter handle in which to perform the run +/// @param[in] parser An S-Expression Parser containing the MeTTa text +/// @return The newly created `runner_state_t`, which can begin evaluating MeTTa code +/// @warning Ownership of the provided parser will be taken by this function, so it must not be subsequently accessed +/// nor freed. +/// @note The returned `runner_state_t` handle must be freed with `runner_state_free()` +/// +#[no_mangle] +pub extern "C" fn runner_state_new_with_parser(metta: *const metta_t, parser: sexpr_parser_t) -> runner_state_t { + let metta = unsafe{ &*metta }.borrow(); + let parser = parser.into_inner(); + let state = RunnerState::new_with_parser(metta, parser); + state.into() +} + +/// @brief Creates a runner_state_t, to use for step-wise execution of a list of atoms +/// @ingroup interpreter_group +/// @param[in] metta A pointer to the Interpreter handle in which to perform the run +/// @param[in] atoms A pointer to an `atom_vec_t` containing the atoms to run /// @return The newly created `runner_state_t`, which can begin evaluating MeTTa code +/// @warning The referenced `atoms` `atom_vec_t` must not be modified nor freed while the +/// `runner_state_t` remains active /// @note The returned `runner_state_t` handle must be freed with `runner_state_free()` /// #[no_mangle] -pub extern "C" fn metta_start_run(metta: *mut metta_t) -> runner_state_t { +pub extern "C" fn runner_state_new_with_atoms(metta: *const metta_t, atoms: *const atom_vec_t) -> runner_state_t { let metta = unsafe{ &*metta }.borrow(); - let state = metta.start_run(); + let atoms = unsafe{ &*atoms }.as_slice(); + let state = RunnerState::new_with_atoms(metta, atoms); state.into() } @@ -909,16 +950,12 @@ pub extern "C" fn runner_state_free(state: runner_state_t) { /// @brief Runs one step of the interpreter /// @ingroup interpreter_group -/// @param[in] metta A pointer to the Interpreter handle -/// @param[in] parser A pointer to the S-Expression Parser handle, containing the expression text /// @param[in] state A pointer to the in-flight runner state /// #[no_mangle] -pub extern "C" fn metta_run_step(metta: *mut metta_t, parser: *mut sexpr_parser_t, state: *mut runner_state_t) { - let metta = unsafe{ &*metta }.borrow(); - let parser = unsafe{ &*parser }.borrow_inner(); +pub extern "C" fn runner_state_step(state: *mut runner_state_t) { let state = unsafe{ &mut *state }.borrow_mut(); - metta.run_step(parser, state).unwrap_or_else(|err| panic!("Unhandled MeTTa error: {}", err)); + state.run_step().unwrap_or_else(|err| panic!("Unhandled MeTTa error: {}", err)); } /// @brief Returns whether or not the runner_state_t has completed all outstanding work @@ -932,6 +969,21 @@ pub extern "C" fn runner_state_is_complete(state: *const runner_state_t) -> bool state.is_complete() } +/// @brief Renders a text description of a `runner_state_t` into a buffer +/// @ingroup interpreter_group +/// @param[in] state A pointer to a `runner_state_t` to render +/// @param[out] buf A buffer into which the text will be rendered +/// @param[in] buf_len The maximum allocated size of `buf` +/// @return The length of the description string, minus the string terminator character. If +/// `return_value > buf_len + 1`, then the text was not fully rendered and this function should be +/// called again with a larger buffer. +/// +#[no_mangle] +pub extern "C" fn runner_state_to_str(state: *const runner_state_t, buf: *mut c_char, buf_len: usize) -> usize { + let state = unsafe{ &*state }.borrow(); + write_debug_into_buf(state, buf, buf_len) +} + /// @brief Accesses the current in-flight results in the runner_state_t /// @ingroup interpreter_group /// @param[in] state The `runner_state_t` within which to preview results @@ -1156,7 +1208,7 @@ pub extern "C" fn env_builder_add_include_path(builder: *mut env_builder_t, path let builder = if path.is_null() { panic!("Fatal Error: path cannot be NULL"); } else { - builder.add_include_paths(vec![PathBuf::from(cstr_as_str(path).borrow())]) + builder.add_include_paths(vec![PathBuf::from(cstr_as_str(path))]) }; *builder_arg_ref = builder.into(); } diff --git a/c/tests/check_runner.c b/c/tests/check_runner.c index 3fccf792e..2a25d7c7e 100644 --- a/c/tests/check_runner.c +++ b/c/tests/check_runner.c @@ -24,15 +24,14 @@ START_TEST (test_incremental_runner) { metta_t runner = new_test_metta(); - runner_state_t runner_state = metta_start_run(&runner); - sexpr_parser_t parser = sexpr_parser_new("!(+ 1 (+ 2 (+ 3 4)))"); + runner_state_t runner_state = runner_state_new_with_parser(&runner, parser); int step_count = 0; char atom_str_buf[64]; atom_str_buf[0] = 0; while (!runner_state_is_complete(&runner_state)) { - metta_run_step(&runner, &parser, &runner_state); + runner_state_step(&runner_state); atom_vec_t* results = NULL; runner_state_current_results(&runner_state, ©_atom_vec, &results); @@ -50,8 +49,6 @@ START_TEST (test_incremental_runner) } ck_assert_str_eq(atom_str_buf, "10"); - sexpr_parser_free(parser); - runner_state_free(runner_state); metta_free(runner); } diff --git a/c/tests/check_space.c b/c/tests/check_space.c index e45560622..f8a60d5f0 100644 --- a/c/tests/check_space.c +++ b/c/tests/check_space.c @@ -235,7 +235,7 @@ START_TEST (test_space_nested_in_atom) sexpr_parser_t parser = sexpr_parser_new("!(match nested (A $x) $x)"); atom_vec_t results; - metta_run(&runner, &parser, ©_atom_vec, &results); + metta_run(&runner, parser, ©_atom_vec, &results); atom_ref_t result_atom = atom_vec_get(&results, 0); atom_t expected_atom = atom_sym("B"); @@ -243,7 +243,6 @@ START_TEST (test_space_nested_in_atom) atom_free(expected_atom); atom_vec_free(results); - sexpr_parser_free(parser); tokenizer_free(tokenizer); metta_free(runner); diff --git a/lib/src/metta/interpreter2.rs b/lib/src/metta/interpreter2.rs index dd360e653..252605e3b 100644 --- a/lib/src/metta/interpreter2.rs +++ b/lib/src/metta/interpreter2.rs @@ -45,6 +45,7 @@ use std::marker::PhantomData; pub trait SpaceRef<'a> : Space + 'a {} impl<'a, T: Space + 'a> SpaceRef<'a> for T {} +#[derive(Debug)] struct InterpreterContext<'a, T: SpaceRef<'a>> { space: T, phantom: PhantomData<&'a GroundingSpace>, @@ -56,6 +57,7 @@ impl<'a, T: SpaceRef<'a>> InterpreterContext<'a, T> { } } +#[derive(Debug)] pub struct InterpreterState<'a, T: SpaceRef<'a>> { plan: Vec, finished: Vec, diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index 33b1328e0..777884b72 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -63,10 +63,22 @@ enum MettaRunnerMode { pub struct RunnerState<'a> { mode: MettaRunnerMode, + metta: &'a Metta, + parser: Option>, + atoms: Option<&'a [Atom]>, interpreter_state: Option>, results: Vec>, } +impl std::fmt::Debug for RunnerState<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RunnerState") + .field("mode", &self.mode) + .field("interpreter_state", &self.interpreter_state) + .finish() + } +} + impl Metta { /// A 1-line method to create a fully initialized MeTTa interpreter @@ -167,7 +179,7 @@ impl Metta { // Make the imported module be immediately available to itself // to mitigate circular imports self.0.modules.borrow_mut().insert(path.clone(), runner.space().clone()); - runner.run(&mut SExprParser::new(program.as_str())) + runner.run(SExprParser::new(program.as_str())) .map_err(|err| format!("Cannot import module, path: {}, error: {}", path.display(), err))?; // TODO: This is a hack. We need a way to register tokens at module-load-time, for any module @@ -227,28 +239,90 @@ impl Metta { self.0.settings.borrow().get(key.into()).map(|a| a.to_string()) } - pub fn run(&self, parser: &mut SExprParser) -> Result>, String> { - let mut state = self.start_run(); + pub fn run<'p, 'a: 'p>(&'a self, parser: SExprParser<'p>) -> Result>, String> { + let state = RunnerState::new_with_parser(self, parser); + state.run_to_completion() + } + + // TODO: this method is deprecated and should be removed after switching + // to the minimal MeTTa + pub fn evaluate_atom(&self, atom: Atom) -> Result, String> { + #[cfg(feature = "minimal")] + let atom = wrap_atom_by_metta_interpreter(self, atom); + match self.type_check(atom) { + Err(atom) => Ok(vec![atom]), + Ok(atom) => interpret(self.space(), &atom), + } + } + + fn add_atom(&self, atom: Atom) -> Result<(), Atom>{ + let atom = self.type_check(atom)?; + self.0.space.borrow_mut().add(atom); + Ok(()) + } + + fn type_check(&self, atom: Atom) -> Result { + let is_type_check_enabled = self.get_setting_string("type-check").map_or(false, |val| val == "auto"); + if is_type_check_enabled && !validate_atom(self.0.space.borrow().as_space(), &atom) { + Err(Atom::expr([ERROR_SYMBOL, atom, BAD_TYPE_SYMBOL])) + } else { + Ok(atom) + } + } + +} + +#[cfg(feature = "minimal")] +fn wrap_atom_by_metta_interpreter(runner: &Metta, atom: Atom) -> Atom { + let space = Atom::gnd(runner.space().clone()); + let interpret = Atom::expr([Atom::sym("interpret"), atom, ATOM_TYPE_UNDEFINED, space]); + let eval = Atom::expr([EVAL_SYMBOL, interpret]); + eval +} - while !state.is_complete() { - self.run_step(parser, &mut state)?; +impl<'a> RunnerState<'a> { + fn new(metta: &'a Metta) -> Self { + Self { + metta, + mode: MettaRunnerMode::ADD, + interpreter_state: None, + parser: None, + atoms: None, + results: vec![], } - Ok(state.into_results()) + } + /// Returns a new RunnerState, for running code from the [SExprParser] with the specified [Metta] runner + pub fn new_with_parser(metta: &'a Metta, parser: SExprParser<'a>) -> Self { + let mut state = Self::new(metta); + state.parser = Some(parser); + state + } + + /// Returns a new RunnerState, for running code encoded as a slice of [Atom]s with the specified [Metta] runner + pub fn new_with_atoms(metta: &'a Metta, atoms: &'a[Atom]) -> Self { + let mut state = Self::new(metta); + state.atoms = Some(atoms); + state } - pub fn start_run(&self) -> RunnerState { - RunnerState::new() + /// Repeatedly steps a RunnerState until it is complete, and then returns the results + pub fn run_to_completion(mut self) -> Result>, String> { + while !self.is_complete() { + self.run_step()?; + } + Ok(self.into_results()) } - pub fn run_step(&self, parser: &mut SExprParser, state: &mut RunnerState) -> Result<(), String> { + /// Runs one step of the interpreter + pub fn run_step(&mut self) -> Result<(), String> { // If we're in the middle of interpreting an atom... - if let Some(interpreter_state) = core::mem::replace(&mut state.interpreter_state, None) { + if let Some(interpreter_state) = core::mem::take(&mut self.interpreter_state) { if interpreter_state.has_next() { //Take a step with the interpreter, and put it back for next time - state.interpreter_state = Some(interpret_step(interpreter_state)) + self.interpreter_state = Some(interpret_step(interpreter_state)) } else { //This interpreter is finished, process the results @@ -256,9 +330,9 @@ impl Metta { Err(msg) => return Err(msg), Ok(result) => { let error = result.iter().any(|atom| atom_is_error(atom)); - state.results.push(result); + self.results.push(result); if error { - state.mode = MettaRunnerMode::TERMINATE; + self.mode = MettaRunnerMode::TERMINATE; return Ok(()); } } @@ -267,30 +341,45 @@ impl Metta { } else { - // We'll parse the next atom, and start a new intperpreter - if let Some(atom) = parser.parse(&self.0.tokenizer.borrow())? { + // Get the next atom, and start a new intperpreter + let next_atom = if let Some(parser) = self.parser.as_mut() { + parser.parse(&self.metta.0.tokenizer.borrow())? + } else { + if let Some(atoms) = self.atoms.as_mut() { + if let Some((atom, rest)) = atoms.split_first() { + *atoms = rest; + Some(atom.clone()) + } else { + None + } + } else { + None + } + }; + + if let Some(atom) = next_atom { if atom == EXEC_SYMBOL { - state.mode = MettaRunnerMode::INTERPRET; + self.mode = MettaRunnerMode::INTERPRET; return Ok(()); } - match state.mode { + match self.mode { MettaRunnerMode::ADD => { - if let Err(atom) = self.add_atom(atom) { - state.results.push(vec![atom]); - state.mode = MettaRunnerMode::TERMINATE; + if let Err(atom) = self.metta.add_atom(atom) { + self.results.push(vec![atom]); + self.mode = MettaRunnerMode::TERMINATE; return Ok(()); } }, MettaRunnerMode::INTERPRET => { - state.interpreter_state = Some(match self.type_check(atom) { + self.interpreter_state = Some(match self.metta.type_check(atom) { Err(atom) => { - InterpreterState::new_finished(self.space().clone(), vec![atom]) + InterpreterState::new_finished(self.metta.space().clone(), vec![atom]) }, Ok(atom) => { #[cfg(feature = "minimal")] - let atom = wrap_atom_by_metta_interpreter(self, atom); - interpret_init(self.space().clone(), &atom) + let atom = wrap_atom_by_metta_interpreter(&self.metta, atom); + interpret_init(self.metta.space().clone(), &atom) }, }); }, @@ -298,59 +387,15 @@ impl Metta { return Ok(()); }, } - state.mode = MettaRunnerMode::ADD; + self.mode = MettaRunnerMode::ADD; } else { - state.mode = MettaRunnerMode::TERMINATE; + self.mode = MettaRunnerMode::TERMINATE; } } Ok(()) } - // TODO: this method is deprecated and should be removed after switching - // to the minimal MeTTa - pub fn evaluate_atom(&self, atom: Atom) -> Result, String> { - #[cfg(feature = "minimal")] - let atom = wrap_atom_by_metta_interpreter(self, atom); - match self.type_check(atom) { - Err(atom) => Ok(vec![atom]), - Ok(atom) => interpret(self.space(), &atom), - } - } - - fn add_atom(&self, atom: Atom) -> Result<(), Atom>{ - let atom = self.type_check(atom)?; - self.0.space.borrow_mut().add(atom); - Ok(()) - } - - fn type_check(&self, atom: Atom) -> Result { - let is_type_check_enabled = self.get_setting_string("type-check").map_or(false, |val| val == "auto"); - if is_type_check_enabled && !validate_atom(self.0.space.borrow().as_space(), &atom) { - Err(Atom::expr([ERROR_SYMBOL, atom, BAD_TYPE_SYMBOL])) - } else { - Ok(atom) - } - } - -} - -#[cfg(feature = "minimal")] -fn wrap_atom_by_metta_interpreter(runner: &Metta, atom: Atom) -> Atom { - let space = Atom::gnd(runner.space().clone()); - let interpret = Atom::expr([Atom::sym("interpret"), atom, ATOM_TYPE_UNDEFINED, space]); - let eval = Atom::expr([EVAL_SYMBOL, interpret]); - eval -} - -impl<'a> RunnerState<'a> { - fn new() -> Self { - Self { - mode: MettaRunnerMode::ADD, - interpreter_state: None, - results: vec![], - } - } pub fn is_complete(&self) -> bool { self.mode == MettaRunnerMode::TERMINATE } @@ -382,7 +427,7 @@ mod tests { "; let metta = Metta::new_rust(Some(EnvBuilder::test_env())); - let result = metta.run(&mut SExprParser::new(program)); + let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![Atom::sym("T")]])); } @@ -396,7 +441,7 @@ mod tests { let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); metta.set_setting("type-check".into(), sym!("auto")); - let result = metta.run(&mut SExprParser::new(program)); + let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); } @@ -410,7 +455,7 @@ mod tests { let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); metta.set_setting("type-check".into(), sym!("auto")); - let result = metta.run(&mut SExprParser::new(program)); + let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); } @@ -448,7 +493,7 @@ mod tests { let metta = Metta::new_rust(Some(EnvBuilder::test_env())); metta.tokenizer().borrow_mut().register_token(Regex::new("error").unwrap(), |_| Atom::gnd(ErrorOp{})); - let result = metta.run(&mut SExprParser::new(program)); + let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("error") "TestError")]])); } @@ -465,7 +510,7 @@ mod tests { let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); metta.set_setting("type-check".into(), sym!("auto")); - let result = metta.run(&mut SExprParser::new(program)); + let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); } @@ -499,7 +544,7 @@ mod tests { let metta = Metta::new_rust(Some(EnvBuilder::test_env())); metta.tokenizer().borrow_mut().register_token(Regex::new("empty").unwrap(), |_| Atom::gnd(ReturnAtomOp(expr!()))); - let result = metta.run(&mut SExprParser::new(program)); + let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!()]])); } diff --git a/lib/src/metta/runner/stdlib.rs b/lib/src/metta/runner/stdlib.rs index 4409907f7..d2823acc6 100644 --- a/lib/src/metta/runner/stdlib.rs +++ b/lib/src/metta/runner/stdlib.rs @@ -1213,7 +1213,7 @@ mod tests { fn run_program(program: &str) -> Result>, String> { let metta = Metta::new_rust(Some(EnvBuilder::test_env())); - metta.run(&mut SExprParser::new(program)) + metta.run(SExprParser::new(program)) } #[test] @@ -1426,7 +1426,7 @@ mod tests { #[test] fn superpose_op_multiple_interpretations() { let metta = Metta::new_rust(Some(EnvBuilder::test_env())); - let mut parser = SExprParser::new(" + let parser = SExprParser::new(" (= (f) A) (= (f) B) (= (g) C) @@ -1435,28 +1435,28 @@ mod tests { !(superpose ((f) (g))) "); - assert_eq_metta_results!(metta.run(&mut parser), + assert_eq_metta_results!(metta.run(parser), Ok(vec![vec![expr!("A"), expr!("B"), expr!("C"), expr!("D")]])); } #[test] fn superpose_op_superposed_with_collapse() { let metta = Metta::new_rust(Some(EnvBuilder::test_env())); - let mut parser = SExprParser::new(" + let parser = SExprParser::new(" (= (f) A) (= (f) B) !(let $x (collapse (f)) (superpose $x)) "); - assert_eq_metta_results!(metta.run(&mut parser), + assert_eq_metta_results!(metta.run(parser), Ok(vec![vec![expr!("A"), expr!("B")]])); } #[test] fn superpose_op_consumes_interpreter_errors() { let metta = Metta::new_rust(Some(EnvBuilder::test_env())); - let mut parser = SExprParser::new(" + let parser = SExprParser::new(" (: f (-> A B)) (= (f $x) $x) @@ -1466,7 +1466,7 @@ mod tests { !(superpose ((f (superpose ())) (f a) (f b))) "); - assert_eq!(metta.run(&mut parser), Ok(vec![vec![ + assert_eq!(metta.run(parser), Ok(vec![vec![ expr!("Error" ("f" ({SuperposeOp{space:metta.space().clone()}} ())) "NoValidAlternatives"), expr!("a"), expr!("Error" "b" "BadType")]])); } diff --git a/lib/src/metta/runner/stdlib2.rs b/lib/src/metta/runner/stdlib2.rs index 3756d9272..3a4732476 100644 --- a/lib/src/metta/runner/stdlib2.rs +++ b/lib/src/metta/runner/stdlib2.rs @@ -418,7 +418,7 @@ mod tests { fn run_program(program: &str) -> Result>, String> { let metta = Metta::new_rust(Some(EnvBuilder::test_env())); - metta.run(&mut SExprParser::new(program)) + metta.run(SExprParser::new(program)) } #[test] @@ -625,14 +625,14 @@ mod tests { (= (foo $x) $x) (= (bar $x) $x) "; - assert_eq!(metta.run(&mut SExprParser::new(program)), Ok(vec![])); - assert_eq!(metta.run(&mut SExprParser::new("!(assertEqual (foo A) (bar A))")), Ok(vec![ + assert_eq!(metta.run(SExprParser::new(program)), Ok(vec![])); + assert_eq!(metta.run(SExprParser::new("!(assertEqual (foo A) (bar A))")), Ok(vec![ vec![VOID_SYMBOL], ])); - assert_eq!(metta.run(&mut SExprParser::new("!(assertEqual (foo A) (bar B))")), Ok(vec![ + assert_eq!(metta.run(SExprParser::new("!(assertEqual (foo A) (bar B))")), Ok(vec![ vec![expr!("Error" ({assert.clone()} ("foo" "A") ("bar" "B")) "\nExpected: [B]\nGot: [A]\nMissed result: B")], ])); - assert_eq!(metta.run(&mut SExprParser::new("!(assertEqual (foo A) Empty)")), Ok(vec![ + assert_eq!(metta.run(SExprParser::new("!(assertEqual (foo A) Empty)")), Ok(vec![ vec![expr!("Error" ({assert.clone()} ("foo" "A") "Empty") "\nExpected: []\nGot: [A]\nExcessive result: A")] ])); } @@ -648,14 +648,14 @@ mod tests { (= (baz) D) (= (baz) D) "; - assert_eq!(metta.run(&mut SExprParser::new(program)), Ok(vec![])); - assert_eq!(metta.run(&mut SExprParser::new("!(assertEqualToResult (foo) (A B))")), Ok(vec![ + assert_eq!(metta.run(SExprParser::new(program)), Ok(vec![])); + assert_eq!(metta.run(SExprParser::new("!(assertEqualToResult (foo) (A B))")), Ok(vec![ vec![VOID_SYMBOL], ])); - assert_eq!(metta.run(&mut SExprParser::new("!(assertEqualToResult (bar) (A))")), Ok(vec![ + assert_eq!(metta.run(SExprParser::new("!(assertEqualToResult (bar) (A))")), Ok(vec![ vec![expr!("Error" ({assert.clone()} ("bar") ("A")) "\nExpected: [A]\nGot: [C]\nMissed result: A")], ])); - assert_eq!(metta.run(&mut SExprParser::new("!(assertEqualToResult (baz) (D))")), Ok(vec![ + assert_eq!(metta.run(SExprParser::new("!(assertEqualToResult (baz) (D))")), Ok(vec![ vec![expr!("Error" ({assert.clone()} ("baz") ("D")) "\nExpected: [D]\nGot: [D, D]\nExcessive result: D")] ])); } @@ -848,14 +848,14 @@ mod tests { metta.tokenizer().borrow_mut().register_token(Regex::new("id_num").unwrap(), |_| Atom::gnd(ID_NUM)); - assert_eq!(metta.run(&mut SExprParser::new(program1)), + assert_eq!(metta.run(SExprParser::new(program1)), Ok(vec![vec![expr!("Error" "myAtom" "BadType")]])); let program2 = " !(eval (interpret (id_num myAtom) %Undefined% &self)) "; - assert_eq!(metta.run(&mut SExprParser::new(program2)), + assert_eq!(metta.run(SExprParser::new(program2)), Ok(vec![vec![expr!("Error" "myAtom" "BadType")]])); } } diff --git a/lib/src/metta/text.rs b/lib/src/metta/text.rs index 9a806ecc7..6c0f1d007 100644 --- a/lib/src/metta/text.rs +++ b/lib/src/metta/text.rs @@ -217,6 +217,7 @@ impl SyntaxNode { } } +#[derive(Clone)] pub struct SExprParser<'a> { text: &'a str, it: Peekable>, diff --git a/lib/tests/case.rs b/lib/tests/case.rs index 85c5fa363..f3ef250d3 100644 --- a/lib/tests/case.rs +++ b/lib/tests/case.rs @@ -6,7 +6,7 @@ use hyperon::metta::environment::EnvBuilder; #[test] fn test_case_operation() { let metta = Metta::new_rust(Some(EnvBuilder::test_env())); - let result = metta.run(&mut SExprParser::new(" + let result = metta.run(SExprParser::new(" ; cases are processed sequentially !(case (+ 1 5) ((5 Error) @@ -31,7 +31,7 @@ fn test_case_operation() { !(case 5 ((6 OK))) ")); - let expected = metta.run(&mut SExprParser::new(" + let expected = metta.run(SExprParser::new(" ! OK ! 7 ! (superpose (OK-3 OK-4)) @@ -41,7 +41,7 @@ fn test_case_operation() { assert_eq!(result, expected); let metta = Metta::new_rust(Some(EnvBuilder::test_env())); - let result = metta.run(&mut SExprParser::new(" + let result = metta.run(SExprParser::new(" (Rel-P A B) (Rel-Q A C) @@ -65,7 +65,7 @@ fn test_case_operation() { !(maybe-inc Nothing) !(maybe-inc (Just 2)) ")); - let expected = metta.run(&mut SExprParser::new(" + let expected = metta.run(SExprParser::new(" ! (superpose ((Q C) (P B))) ! no-match ! Nothing diff --git a/lib/tests/metta.rs b/lib/tests/metta.rs index 6862f5e17..28ac701a8 100644 --- a/lib/tests/metta.rs +++ b/lib/tests/metta.rs @@ -20,7 +20,7 @@ fn test_reduce_higher_order() { "; let metta = Metta::new_rust(Some(EnvBuilder::test_env())); - let result = metta.run(&mut SExprParser::new(program)); + let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![UNIT_ATOM()]])); } diff --git a/python/hyperon/base.py b/python/hyperon/base.py index 53ee4f48b..42478d357 100644 --- a/python/hyperon/base.py +++ b/python/hyperon/base.py @@ -396,6 +396,8 @@ def parse_to_syntax_tree(self): class Interpreter: """ A wrapper class for the MeTTa interpreter that handles the interpretation of expressions in a given grounding space. + + NOTE: This is a low-level API, and most applications would be better served by a `MeTTa` runner object """ def __init__(self, gnd_space, expr): diff --git a/python/hyperon/runner.py b/python/hyperon/runner.py index 888e43bfe..68263fdbe 100644 --- a/python/hyperon/runner.py +++ b/python/hyperon/runner.py @@ -8,22 +8,38 @@ from hyperonpy import EnvBuilder class RunnerState: - def __init__(self, cstate): - """Initialize a RunnerState""" - self.cstate = cstate + """ + The state for an in-flight MeTTa interpreter handling the interpretation and evaluation of atoms in a given grounding space. + """ + def __init__(self, metta, program): + """Initialize a RunnerState with a MeTTa object and a program to run""" + parser = SExprParser(program) + #WARNING the C parser object has a reference to the text buffer, and hyperonpy's CSExprParser + # copies the buffer into an owned string. So we need to make sure this parser isn't freed + # until the RunnerState is done with it. + self.parser = parser + self.cstate = hp.runner_state_new_with_parser(metta.cmetta, parser.cparser) def __del__(self): """Frees a RunnerState and all associated resources.""" hp.runner_state_free(self.cstate) def run_step(self): - hp.metta_run_step(self.runner.cmetta, self.parser.cparser, self.cstate) + """ + Executes the next step in the interpretation plan, or begins interpretation of the next atom in the stream of MeTTa code. + """ + hp.runner_state_step(self.cstate) def is_complete(self): + """ + Returns True if the runner has concluded, or False if there are more steps remaining to execute + """ return hp.runner_state_is_complete(self.cstate) def current_results(self, flat=False): - """Returns the current in-progress results from an in-flight program evaluation""" + """ + Returns the current in-progress results from an in-flight program evaluation + """ results = hp.runner_state_current_results(self.cstate) if flat: return [Atom._from_catom(catom) for result in results for catom in result] @@ -163,14 +179,6 @@ def run(self, program, flat=False): else: return [[Atom._from_catom(catom) for catom in result] for result in results] - def start_run(self, program): - """Initializes a RunnerState to begin evaluation of MeTTa code""" - parser = SExprParser(program) - state = RunnerState(hp.metta_start_run(self.cmetta)) - state.parser = parser - state.runner = self - return state - class Environment: """This class contains the API for shared platform configuration""" diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index 67247748b..c7ca718b3 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -550,6 +550,15 @@ PYBIND11_MODULE(hyperonpy, m) { }, "Check atom for equivalence"); py::class_(m, "CVecAtom"); + m.def("atom_vec_from_list", [](pybind11::list pylist) { + atom_vec_t new_vec = atom_vec_new(); + for(py::handle pyobj : pylist) { + py::handle atom_pyhandle = pyobj.attr("catom"); + CAtom atom = atom_pyhandle.cast(); + atom_vec_push(&new_vec, atom_clone(atom.ptr())); + } + return CVecAtom(new_vec); + }, "Create a vector of atoms from a Python list"); m.def("atom_vec_new", []() { return CVecAtom(atom_vec_new()); }, "New vector of atoms"); m.def("atom_vec_free", [](CVecAtom& vec) { atom_vec_free(vec.obj); }, "Free vector of atoms"); m.def("atom_vec_len", [](CVecAtom& vec) { return atom_vec_len(vec.ptr()); }, "Return size of the vector"); @@ -786,7 +795,8 @@ PYBIND11_MODULE(hyperonpy, m) { m.def("metta_tokenizer", [](CMetta metta) { return CTokenizer(metta_tokenizer(metta.ptr())); }, "Get tokenizer of MeTTa interpreter"); m.def("metta_run", [](CMetta metta, CSExprParser& parser) { py::list lists_of_atom; - metta_run(metta.ptr(), &parser.parser, copy_lists_of_atom, &lists_of_atom); + sexpr_parser_t cloned_parser = sexpr_parser_clone(&parser.parser); + metta_run(metta.ptr(), cloned_parser, copy_lists_of_atom, &lists_of_atom); return lists_of_atom; }, "Run MeTTa interpreter on an input"); m.def("metta_evaluate_atom", [](CMetta metta, CAtom atom) { @@ -794,13 +804,22 @@ PYBIND11_MODULE(hyperonpy, m) { metta_evaluate_atom(metta.ptr(), atom_clone(atom.ptr()), copy_atoms, &atoms); return atoms; }, "Run MeTTa interpreter on an atom"); - m.def("metta_start_run", [](CMetta& metta) { return CRunnerState(metta_start_run(metta.ptr())); }, "Initializes the MeTTa interpreter for incremental execution"); - m.def("metta_run_step", [](CMetta& metta, CSExprParser& parser, CRunnerState& state) { metta_run_step(metta.ptr(), parser.ptr(), state.ptr()); }, "Runs one incremental step of the MeTTa interpreter"); m.def("metta_load_module", [](CMetta metta, std::string text) { metta_load_module(metta.ptr(), text.c_str()); }, "Load MeTTa module"); - py::class_(m, "CRunnerState"); + py::class_(m, "CRunnerState") + .def("__str__", [](CRunnerState state) { + return func_to_string((write_to_buf_func_t)&runner_state_to_str, state.ptr()); + }, "Render a RunnerState as a human readable string"); + m.def("runner_state_new_with_parser", [](CMetta& metta, CSExprParser& parser) { + sexpr_parser_t cloned_parser = sexpr_parser_clone(&parser.parser); + return CRunnerState(runner_state_new_with_parser(metta.ptr(), cloned_parser)); + }, "Initializes the MeTTa runner state for incremental execution"); + m.def("runner_state_new_with_atoms", [](CMetta& metta, CVecAtom& atoms) { + return CRunnerState(runner_state_new_with_atoms(metta.ptr(), atoms.ptr())); + }, "Initializes the MeTTa runner state for incremental execution"); + m.def("runner_state_step", [](CRunnerState& state) { runner_state_step(state.ptr()); }, "Runs one incremental step of the MeTTa interpreter"); m.def("runner_state_free", [](CRunnerState state) { runner_state_free(state.obj); }, "Frees a Runner State"); m.def("runner_state_is_complete", [](CRunnerState& state) { return runner_state_is_complete(state.ptr()); }, "Returns whether a RunnerState is finished"); m.def("runner_state_current_results", [](CRunnerState& state) { diff --git a/python/tests/test_metta.py b/python/tests/test_metta.py index a40807b5d..a96d2858e 100644 --- a/python/tests/test_metta.py +++ b/python/tests/test_metta.py @@ -39,7 +39,7 @@ def test_incremental_runner(self): !(+ 1 (+ 2 (+ 3 4))) ''' runner = MeTTa(env_builder=Environment.test_env()) - runner_state = runner.start_run(program) + runner_state = RunnerState(runner, program) step_count = 0 while not runner_state.is_complete(): diff --git a/repl/src/metta_shim.rs b/repl/src/metta_shim.rs index dc6be8826..e222f7fe1 100644 --- a/repl/src/metta_shim.rs +++ b/repl/src/metta_shim.rs @@ -126,10 +126,10 @@ pub mod metta_interface_mod { let runner_state = Python::with_gil(|py| -> PyResult> { let line = PyString::new(py, line); let py_metta = self.py_metta.as_ref(py); - let args = PyTuple::new(py, &[py_metta, line]); let module: &PyModule = self.py_mod.as_ref(py); - let func = module.getattr("start_run")?; - let result = func.call1(args)?; + let runner_class = module.getattr("RunnerState")?; + let args = PyTuple::new(py, &[py_metta, line]); + let result = runner_class.call1(args)?; Ok(result.into()) }).unwrap(); @@ -329,7 +329,7 @@ pub mod metta_interface_mod { use hyperon::ExpressionAtom; use hyperon::Atom; use hyperon::metta::environment::Environment; - use hyperon::metta::runner::{Metta, atom_is_error}; + use hyperon::metta::runner::{Metta, RunnerState, atom_is_error}; use super::{strip_quotes, exec_state_prepare, exec_state_should_break}; pub use hyperon::metta::text::SyntaxNodeType as SyntaxNodeType; @@ -413,8 +413,8 @@ pub mod metta_interface_mod { } pub fn exec(&mut self, line: &str) { - let mut parser = SExprParser::new(line); - let mut runner_state = self.metta.start_run(); + let parser = SExprParser::new(line); + let mut runner_state = RunnerState::new_with_parser(&self.metta, parser); exec_state_prepare(); @@ -424,8 +424,7 @@ pub mod metta_interface_mod { } //Run the next step - self.metta.run_step(&mut parser, &mut runner_state) - .unwrap_or_else(|err| panic!("Unhandled MeTTa error: {}", err)); + runner_state.run_step().unwrap_or_else(|err| panic!("Unhandled MeTTa error: {}", err)); self.result = runner_state.current_results().clone(); } } From 3ac2582a92f205b74e3c791c8f504e76eaa83600 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Thu, 12 Oct 2023 12:18:41 -0700 Subject: [PATCH 18/28] Refactoring runner into process with a hook to insert a language-specific stdlib loader --- c/src/metta.rs | 59 +++++++++++++++++++++++++-------- c/tests/util.c | 11 ++---- lib/src/metta/runner/mod.rs | 56 +++++++++++++++---------------- lib/src/metta/runner/stdlib.rs | 8 ++--- lib/src/metta/runner/stdlib2.rs | 8 ++--- lib/tests/case.rs | 4 +-- lib/tests/metta.rs | 2 +- python/hyperon/runner.py | 21 +++++------- python/hyperonpy.cpp | 14 ++++++-- python/tests/test_extend.py | 4 --- repl/src/metta_shim.rs | 2 +- 11 files changed, 108 insertions(+), 81 deletions(-) diff --git a/c/src/metta.rs b/c/src/metta.rs index 97658c195..6bcf0ecf9 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -739,8 +739,39 @@ impl metta_t { /// @note The caller must take ownership responsibility for the returned `metta_t`, and free it with `metta_free()` /// #[no_mangle] -pub extern "C" fn metta_new_rust() -> metta_t { - let metta = Metta::new_rust(None); +pub extern "C" fn metta_new() -> metta_t { + let metta = Metta::new(None); + metta.into() +} + +/// @brief Function signature for a callback to load a language-specific stdlib in a MeTTa runner +/// @ingroup interpreter_group +/// @param[in] metta The `metta_t` into which to load the stdlib. +/// @param[in] context The context state pointer initially passed to the upstream function initiating the callback. +/// +pub type c_stdlib_loader_callback_t = extern "C" fn(metta: *mut metta_t, context: *mut c_void); + +/// @brief Creates a new top-level MeTTa Interpreter, bootstrapped with the a custom stdlib +/// @ingroup interpreter_group +/// @return A `metta_t` handle to the newly created Interpreter +/// @note The caller must take ownership responsibility for the returned `metta_t`, and free it with `metta_free()` +/// @note Most callers can simply call `metta_new`. This function is provided to support languages +/// with their own stdlib, that needs to be loaded before the init.metta file is run +/// +#[no_mangle] +pub extern "C" fn metta_new_with_environment_and_stdlib(env_builder: env_builder_t, + callback: c_stdlib_loader_callback_t, context: *mut c_void) -> metta_t +{ + let env_builder = if env_builder.is_default() { + None + } else { + Some(env_builder.into_inner()) + }; + + let metta = Metta::new_with_stdlib_loader(|metta| { + let mut metta = metta_t{metta: (metta as *const Metta).cast_mut().cast()}; + callback(&mut metta, context); + }, env_builder); metta.into() } @@ -766,6 +797,18 @@ pub extern "C" fn metta_new_with_space(space: *mut space_t, tokenizer: *mut toke metta.into() } +/// @brief Clones a `metta_t` handle +/// @ingroup interpreter_group +/// @param[in] metta The handle to clone +/// @return The newly cloned `metta_t` handle, pointing to the same underlying interpreter +/// @note The caller must take ownership responsibility for the returned `metta_t`, and free it with `metta_free()` +/// +#[no_mangle] +pub extern "C" fn metta_clone_handle(metta: *const metta_t) -> metta_t { + let metta = unsafe{ &*metta }.borrow(); + metta.clone().into() +} + /// @brief Frees a `metta_t` handle /// @ingroup interpreter_group /// @param[in] metta The handle to free @@ -778,18 +821,6 @@ pub extern "C" fn metta_free(metta: metta_t) { drop(metta); } -/// @brief Initializes a MeTTa interpreter, for manually bootstrapping a multi-language interpreter -/// @ingroup interpreter_group -/// @param[in] metta A pointer to the Interpreter handle -/// @note Most callers can simply call `metta_new_rust`. This function is provided to support languages -/// with their own stdlib, that needs to be loaded before the init.metta file is run -/// -#[no_mangle] -pub extern "C" fn metta_init(metta: *mut metta_t) { - let metta = unsafe{ &*metta }.borrow(); - metta.init() -} - /// @brief Returns the number of module search paths that will be searched when importing modules into /// the runner /// @ingroup interpreter_group diff --git a/c/tests/util.c b/c/tests/util.c index 2c7b0a29c..0ea60a959 100644 --- a/c/tests/util.c +++ b/c/tests/util.c @@ -30,14 +30,9 @@ atom_t expr(atom_t atom, ...) { return atom_expr(children, argno); } -metta_t new_test_metta(void) { +void noop_metta_init(metta_t* metta, void* context) {} - space_t space = space_new_grounding_space(); - tokenizer_t tokenizer = tokenizer_new(); - metta_t metta = metta_new_with_space(&space, &tokenizer, env_builder_use_test_env()); - metta_load_module(&metta, "stdlib"); - metta_init(&metta); - space_free(space); - tokenizer_free(tokenizer); +metta_t new_test_metta(void) { + metta_t metta = metta_new_with_environment_and_stdlib(env_builder_use_test_env(), &noop_metta_init, NULL); return metta; } \ No newline at end of file diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index 777884b72..169013195 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -84,36 +84,36 @@ impl Metta { /// A 1-line method to create a fully initialized MeTTa interpreter /// /// NOTE: pass `None` for `env_builder` to use the platform environment - /// NOTE: This function is appropriate for Rust or C clients, but if other language-specific - /// stdlibs are involved then see the documentation for [Metta::init] - pub fn new_rust(env_builder: Option) -> Metta { + pub fn new(env_builder: Option) -> Metta { + Self::new_with_stdlib_loader(|_| {}, env_builder) + } + + /// Create and initialize a MeTTa interpreter with a language-specific stdlib + /// + /// NOTE: pass `None` for `env_builder` to use the platform environment + pub fn new_with_stdlib_loader(loader: F, env_builder: Option) -> Metta + where F: FnOnce(&Self) + { + //Create the raw MeTTa runner let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), env_builder); + + // TODO: Reverse the loading order between the Rust stdlib and user-supplied stdlib loader, + // because user-supplied stdlibs might need to build on top of the Rust stdlib. + // Currently this is problematic because https://github.com/trueagi-io/hyperon-experimental/issues/408, + // and the right fix is value-bridging (https://github.com/trueagi-io/hyperon-experimental/issues/351) + + //Load the custom stdlib + loader(&metta); + + //Load the Rust stdlib metta.load_module(PathBuf::from("stdlib")).expect("Could not load stdlib"); - metta.init(); - metta - } - /// Performs initialization of a MeTTa interpreter. Presently this involves running the `init.metta` - /// file from the associated environment. - /// - /// DISCUSSION: Creating a fully-initialized MeTTa runner should usually be done with with - /// a top-level initialization function, such as [Metta::new_rust] or `MeTTa()` in Python. - /// - /// Doing it manually involves several steps: - /// 1. Create the MeTTa runner, using [new_with_space]. This provides a working interpreter, but - /// doesn't load any stdlibs - /// 2. Load each language-specific stdlib (Currently only Python has an extended stdlib) - /// 3. Load the Rust `stdlib` (TODO: Conceptually I'd like to load Rust's stdlib first, so other - /// stdlibs can utilize the Rust stdlib's atoms, but that depends on value bridging) - /// 4. Run the `init.metta` file by calling this function - /// - /// TODO: When we are able to load the Rust stdlib before the Python stdlib, which requires value-bridging, - /// we can refactor this function to load the appropriate stdlib(s) and simplify the init process - pub fn init(&self) { - if let Some(init_meta_file) = self.0.environment.initialization_metta_file_path() { - self.load_module(init_meta_file.into()).unwrap(); + //Run the `init.metta` file + if let Some(init_meta_file) = metta.0.environment.initialization_metta_file_path() { + metta.load_module(init_meta_file.into()).unwrap(); } + metta } /// Returns a new MeTTa interpreter, using the provided Space, Tokenizer @@ -426,7 +426,7 @@ mod tests { !(green Fritz) "; - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![Atom::sym("T")]])); } @@ -490,7 +490,7 @@ mod tests { !(foo) "; - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); metta.tokenizer().borrow_mut().register_token(Regex::new("error").unwrap(), |_| Atom::gnd(ErrorOp{})); let result = metta.run(SExprParser::new(program)); @@ -541,7 +541,7 @@ mod tests { !(empty) "; - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); metta.tokenizer().borrow_mut().register_token(Regex::new("empty").unwrap(), |_| Atom::gnd(ReturnAtomOp(expr!()))); let result = metta.run(SExprParser::new(program)); diff --git a/lib/src/metta/runner/stdlib.rs b/lib/src/metta/runner/stdlib.rs index d2823acc6..396dac861 100644 --- a/lib/src/metta/runner/stdlib.rs +++ b/lib/src/metta/runner/stdlib.rs @@ -1212,7 +1212,7 @@ mod tests { use crate::metta::types::validate_atom; fn run_program(program: &str) -> Result>, String> { - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); metta.run(SExprParser::new(program)) } @@ -1425,7 +1425,7 @@ mod tests { #[test] fn superpose_op_multiple_interpretations() { - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); let parser = SExprParser::new(" (= (f) A) (= (f) B) @@ -1441,7 +1441,7 @@ mod tests { #[test] fn superpose_op_superposed_with_collapse() { - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); let parser = SExprParser::new(" (= (f) A) (= (f) B) @@ -1455,7 +1455,7 @@ mod tests { #[test] fn superpose_op_consumes_interpreter_errors() { - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); let parser = SExprParser::new(" (: f (-> A B)) (= (f $x) $x) diff --git a/lib/src/metta/runner/stdlib2.rs b/lib/src/metta/runner/stdlib2.rs index 3a4732476..603327db3 100644 --- a/lib/src/metta/runner/stdlib2.rs +++ b/lib/src/metta/runner/stdlib2.rs @@ -417,7 +417,7 @@ mod tests { use std::convert::TryFrom; fn run_program(program: &str) -> Result>, String> { - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); metta.run(SExprParser::new(program)) } @@ -619,7 +619,7 @@ mod tests { #[test] fn metta_assert_equal_op() { - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); let assert = AssertEqualOp::new(metta.space().clone()); let program = " (= (foo $x) $x) @@ -639,7 +639,7 @@ mod tests { #[test] fn metta_assert_equal_to_result_op() { - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); let assert = AssertEqualToResultOp::new(metta.space().clone()); let program = " (= (foo) A) @@ -844,7 +844,7 @@ mod tests { !(eval (interpret (id_a myAtom) %Undefined% &self)) "; - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); metta.tokenizer().borrow_mut().register_token(Regex::new("id_num").unwrap(), |_| Atom::gnd(ID_NUM)); diff --git a/lib/tests/case.rs b/lib/tests/case.rs index f3ef250d3..dd84645b0 100644 --- a/lib/tests/case.rs +++ b/lib/tests/case.rs @@ -5,7 +5,7 @@ use hyperon::metta::environment::EnvBuilder; #[test] fn test_case_operation() { - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); let result = metta.run(SExprParser::new(" ; cases are processed sequentially !(case (+ 1 5) @@ -40,7 +40,7 @@ fn test_case_operation() { ")); assert_eq!(result, expected); - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); let result = metta.run(SExprParser::new(" (Rel-P A B) (Rel-Q A C) diff --git a/lib/tests/metta.rs b/lib/tests/metta.rs index 28ac701a8..72cea2150 100644 --- a/lib/tests/metta.rs +++ b/lib/tests/metta.rs @@ -18,7 +18,7 @@ fn test_reduce_higher_order() { !(assertEqualToResult ((inc) 2) (3)) "; - let metta = Metta::new_rust(Some(EnvBuilder::test_env())); + let metta = Metta::new(Some(EnvBuilder::test_env())); let result = metta.run(SExprParser::new(program)); diff --git a/python/hyperon/runner.py b/python/hyperon/runner.py index 68263fdbe..cdcf87402 100644 --- a/python/hyperon/runner.py +++ b/python/hyperon/runner.py @@ -49,27 +49,24 @@ def current_results(self, flat=False): class MeTTa: """This class contains the MeTTa program execution utilities""" - def __init__(self, space = None, cmetta = None, env_builder = None): + def __init__(self, cmetta = None, env_builder = None): if cmetta is not None: self.cmetta = cmetta else: - if space is None: - space = GroundingSpaceRef() - tokenizer = Tokenizer() if env_builder is None: env_builder = hp.env_builder_use_default() - self.cmetta = hp.metta_new(space.cspace, tokenizer.ctokenizer, env_builder) - self.load_py_module("hyperon.stdlib") - hp.metta_load_module(self.cmetta, "stdlib") - self.register_atom('extend-py!', - OperationAtom('extend-py!', - lambda name: self.load_py_module_from_mod_or_file(name) or [], - [AtomType.UNDEFINED, AtomType.ATOM], unwrap=False)) - hp.metta_init(self.cmetta) + self.cmetta = hp.metta_new(env_builder) def __del__(self): hp.metta_free(self.cmetta) + def _priv_load_metta_py_stdlib(self): + self.load_py_module("hyperon.stdlib") + self.register_atom('extend-py!', + OperationAtom('extend-py!', + lambda name: self.load_py_module_from_mod_or_file(name) or [], + [AtomType.UNDEFINED, AtomType.ATOM], unwrap=False)) + def space(self): """Gets the metta space""" return GroundingSpaceRef._from_cspace(hp.metta_space(self.cmetta)) diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index c7ca718b3..f551e7e2d 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -417,6 +417,15 @@ void syntax_node_copy_to_list_callback(const syntax_node_t* node, void *context) } }; +void run_python_loader_callback(metta_t* metta, void* context) { + py::object runner_mod = py::module_::import("hyperon.runner"); + py::object metta_class = runner_mod.attr("MeTTa"); + py::function load_metta_py_stdlib = metta_class.attr("_priv_load_metta_py_stdlib"); + metta_t cloned_metta = metta_clone_handle(metta); + py::object py_metta = metta_class(CMetta(cloned_metta)); + load_metta_py_stdlib(py_metta); +} + struct CConstr { py::function pyconstr; @@ -782,11 +791,10 @@ PYBIND11_MODULE(hyperonpy, m) { ADD_SYMBOL(VOID, "Void"); py::class_(m, "CMetta"); - m.def("metta_new", [](CSpace space, CTokenizer tokenizer, EnvBuilder env_builder) { - return CMetta(metta_new_with_space(space.ptr(), tokenizer.ptr(), env_builder.obj)); + m.def("metta_new", [](EnvBuilder env_builder) { + return CMetta(metta_new_with_environment_and_stdlib(env_builder.obj, &run_python_loader_callback, NULL)); }, "New MeTTa interpreter instance"); m.def("metta_free", [](CMetta metta) { metta_free(metta.obj); }, "Free MeTTa interpreter"); - m.def("metta_init", [](CMetta metta) { metta_init(metta.ptr()); }, "Inits a MeTTa interpreter by running the init.metta file from its environment"); m.def("metta_search_path_cnt", [](CMetta metta) { return metta_search_path_cnt(metta.ptr()); }, "Returns the number of module search paths in the runner's environment"); m.def("metta_nth_search_path", [](CMetta metta, size_t idx) { return func_to_string_two_args((write_to_buf_two_arg_func_t)&metta_nth_search_path, metta.ptr(), (void*)idx); diff --git a/python/tests/test_extend.py b/python/tests/test_extend.py index 2d5d3be77..d9df51b62 100644 --- a/python/tests/test_extend.py +++ b/python/tests/test_extend.py @@ -18,10 +18,6 @@ def test_extend(self): [[], [ValueAtom(5)], [ValueAtom('B')]]) - self.assertEqual( - metta.run('! &runner')[0][0].get_object().value, - metta) - class ExtendGlobalTest(unittest.TestCase): diff --git a/repl/src/metta_shim.rs b/repl/src/metta_shim.rs index e222f7fe1..06c36828a 100644 --- a/repl/src/metta_shim.rs +++ b/repl/src/metta_shim.rs @@ -361,7 +361,7 @@ pub mod metta_interface_mod { .init_platform_env(); let new_shim = MettaShim { - metta: Metta::new_rust(None), + metta: Metta::new(None), result: vec![], }; new_shim.metta.tokenizer().borrow_mut().register_token_with_regex_str("extend-py!", move |_| { Atom::gnd(ImportPyErr) }); From 205b4d057dd785203787585d91d81833ec513a37 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Thu, 12 Oct 2023 12:37:51 -0700 Subject: [PATCH 19/28] Converting `Atom.is_error()` in atoms.py into `atom_is_error()` in base.py, because error expressions are not part of the core MeTTa spec --- python/hyperon/atoms.py | 3 --- python/hyperon/base.py | 4 ++++ repl/src/py_shim.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/python/hyperon/atoms.py b/python/hyperon/atoms.py index 55df3ad24..6dcd4c0e1 100644 --- a/python/hyperon/atoms.py +++ b/python/hyperon/atoms.py @@ -27,9 +27,6 @@ def __repr__(self): """Renders a human-readable text description of the Atom.""" return hp.atom_to_str(self.catom) - def is_error(self): - return hp.atom_is_error(self.catom) - def get_type(self): """Gets the type of the current Atom instance""" return hp.atom_get_type(self.catom) diff --git a/python/hyperon/base.py b/python/hyperon/base.py index 42478d357..31093614c 100644 --- a/python/hyperon/base.py +++ b/python/hyperon/base.py @@ -485,3 +485,7 @@ def get_atom_types(gnd_space, atom): """Provides all types for the given Atom in the context of the given Space.""" result = hp.get_atom_types(gnd_space.cspace, atom.catom) return [Atom._from_catom(catom) for catom in result] + +def atom_is_error(atom): + """Checks whether an Atom is an error expression""" + return hp.atom_is_error(atom.catom) diff --git a/repl/src/py_shim.py b/repl/src/py_shim.py index d0100bc6d..b678f8239 100644 --- a/repl/src/py_shim.py +++ b/repl/src/py_shim.py @@ -49,7 +49,7 @@ def get_config_atom(metta, config_name): result = metta.run("!(get-state " + config_name + ")") try: atom = result[0][0] - if (atom.is_error()): + if (atom_is_error(atom)): return None else: return atom From 3be64f5d0b94931d657d02c404490c36ac2940a2 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Thu, 12 Oct 2023 12:47:23 -0700 Subject: [PATCH 20/28] Moving Environment sub-module into runner --- c/src/metta.rs | 3 +-- lib/src/metta/mod.rs | 1 - lib/src/metta/{ => runner}/environment.rs | 0 lib/src/metta/{ => runner}/init.default.metta | 0 lib/src/metta/runner/mod.rs | 4 ++-- lib/src/metta/runner/stdlib.rs | 3 +-- lib/src/metta/runner/stdlib2.rs | 2 +- lib/tests/case.rs | 3 +-- lib/tests/metta.rs | 3 +-- 9 files changed, 7 insertions(+), 12 deletions(-) rename lib/src/metta/{ => runner}/environment.rs (100%) rename lib/src/metta/{ => runner}/init.default.metta (100%) diff --git a/c/src/metta.rs b/c/src/metta.rs index 6bcf0ecf9..30c8a097d 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -3,8 +3,7 @@ use hyperon::space::DynSpace; use hyperon::metta::text::*; use hyperon::metta::interpreter; use hyperon::metta::interpreter::InterpreterState; -use hyperon::metta::runner::{Metta, RunnerState}; -use hyperon::metta::environment::{Environment, EnvBuilder}; +use hyperon::metta::runner::{Metta, RunnerState, Environment, EnvBuilder}; use hyperon::rust_type_atom; use crate::util::*; diff --git a/lib/src/metta/mod.rs b/lib/src/metta/mod.rs index 3904bf532..94a82e634 100644 --- a/lib/src/metta/mod.rs +++ b/lib/src/metta/mod.rs @@ -6,7 +6,6 @@ pub mod interpreter; pub mod interpreter2; pub mod types; pub mod runner; -pub mod environment; use text::{SExprParser, Tokenizer}; use regex::Regex; diff --git a/lib/src/metta/environment.rs b/lib/src/metta/runner/environment.rs similarity index 100% rename from lib/src/metta/environment.rs rename to lib/src/metta/runner/environment.rs diff --git a/lib/src/metta/init.default.metta b/lib/src/metta/runner/init.default.metta similarity index 100% rename from lib/src/metta/init.default.metta rename to lib/src/metta/runner/init.default.metta diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index 169013195..fdae2f9b7 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -2,7 +2,6 @@ use crate::*; use crate::common::shared::Shared; use super::*; -use super::environment::EnvBuilder; use super::space::*; use super::text::{Tokenizer, SExprParser}; use super::types::validate_atom; @@ -12,7 +11,8 @@ use std::path::{Path, PathBuf}; use std::collections::HashMap; use std::sync::Arc; -use metta::environment::Environment; +mod environment; +pub use environment::{Environment, EnvBuilder}; #[cfg(not(feature = "minimal"))] pub mod stdlib; diff --git a/lib/src/metta/runner/stdlib.rs b/lib/src/metta/runner/stdlib.rs index 396dac861..3f741443b 100644 --- a/lib/src/metta/runner/stdlib.rs +++ b/lib/src/metta/runner/stdlib.rs @@ -1207,8 +1207,7 @@ pub static METTA_CODE: &'static str = " #[cfg(test)] mod tests { use super::*; - use crate::metta::environment::EnvBuilder; - use crate::metta::runner::Metta; + use crate::metta::runner::{Metta, EnvBuilder}; use crate::metta::types::validate_atom; fn run_program(program: &str) -> Result>, String> { diff --git a/lib/src/metta/runner/stdlib2.rs b/lib/src/metta/runner/stdlib2.rs index 603327db3..53db483c7 100644 --- a/lib/src/metta/runner/stdlib2.rs +++ b/lib/src/metta/runner/stdlib2.rs @@ -411,7 +411,7 @@ pub static METTA_CODE: &'static str = include_str!("stdlib.metta"); #[cfg(test)] mod tests { use super::*; - use crate::metta::environment::EnvBuilder; + use crate::metta::runner::EnvBuilder; use crate::matcher::atoms_are_equivalent; use std::convert::TryFrom; diff --git a/lib/tests/case.rs b/lib/tests/case.rs index dd84645b0..8a5b2adf1 100644 --- a/lib/tests/case.rs +++ b/lib/tests/case.rs @@ -1,7 +1,6 @@ use hyperon::assert_eq_metta_results; use hyperon::metta::text::SExprParser; -use hyperon::metta::runner::Metta; -use hyperon::metta::environment::EnvBuilder; +use hyperon::metta::runner::{Metta, EnvBuilder}; #[test] fn test_case_operation() { diff --git a/lib/tests/metta.rs b/lib/tests/metta.rs index 72cea2150..a4ffe60a1 100644 --- a/lib/tests/metta.rs +++ b/lib/tests/metta.rs @@ -3,8 +3,7 @@ use hyperon::metta::runner::stdlib::UNIT_ATOM; #[cfg(feature = "minimal")] use hyperon::metta::runner::stdlib2::UNIT_ATOM; use hyperon::metta::text::*; -use hyperon::metta::runner::Metta; -use hyperon::metta::environment::EnvBuilder; +use hyperon::metta::runner::{Metta, EnvBuilder}; #[test] fn test_reduce_higher_order() { From 9029f665727eaf5e9da722b79ecfa2323a74d2cf Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Thu, 12 Oct 2023 13:45:44 -0700 Subject: [PATCH 21/28] Renaming "platform" environment to "common" environment --- c/src/metta.rs | 26 ++++++++++----------- lib/src/metta/runner/environment.rs | 36 ++++++++++++++--------------- lib/src/metta/runner/mod.rs | 8 +++---- python/hyperon/runner.py | 10 ++++---- python/hyperonpy.cpp | 16 ++++++------- python/tests/test_environment.py | 4 ++-- repl/src/metta_shim.rs | 16 ++++++------- repl/src/py_shim.py | 2 +- 8 files changed, 58 insertions(+), 60 deletions(-) diff --git a/c/src/metta.rs b/c/src/metta.rs index 30c8a097d..625394cdc 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -1067,7 +1067,7 @@ pub extern "C" fn metta_load_module(metta: *mut metta_t, name: *const c_char) { // Environment Interface // =-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- -/// @brief Renders the config_dir path from the platform environment into a text buffer +/// @brief Renders the config_dir path from the common environment into a text buffer /// @ingroup environment_group /// @param[out] buf A buffer into which the text will be written /// @param[in] buf_len The maximum allocated size of `buf` @@ -1077,7 +1077,7 @@ pub extern "C" fn metta_load_module(metta: *mut metta_t, name: *const c_char) { /// #[no_mangle] pub extern "C" fn environment_config_dir(buf: *mut c_char, buf_len: usize) -> usize { - match Environment::platform_env().config_dir() { + match Environment::common_env().config_dir() { Some(path) => write_into_buf(path.display(), buf, buf_len), None => 0 } @@ -1119,7 +1119,7 @@ impl env_builder_t { /// @brief Begins initialization of an environment /// @ingroup environment_group /// @return The `env_builder_t` object representing the in-process environment initialization -/// @note The `env_builder_t` must be passed to either `env_builder_init_platform_env` +/// @note The `env_builder_t` must be passed to either `env_builder_init_common_env` /// or `metta_new_with_space` in order to properly deallocate it /// #[no_mangle] @@ -1127,9 +1127,9 @@ pub extern "C" fn env_builder_start() -> env_builder_t { EnvBuilder::new().into() } -/// @brief Creates an `env_builder_t` to specify that the default platform environment should be used +/// @brief Creates an `env_builder_t` to specify that the default common environment should be used /// @ingroup environment_group -/// @return The `env_builder_t` object specifying the default platform environment +/// @return The `env_builder_t` object specifying the common environment /// @note This function exists to supply an argument to `metta_new_with_space` when no special /// behavior is desired /// @note The `env_builder_t` must be passed to `metta_new_with_space` @@ -1150,19 +1150,19 @@ pub extern "C" fn env_builder_use_test_env() -> env_builder_t { EnvBuilder::test_env().into() } -/// @brief Finishes initialization of the platform environment +/// @brief Finishes initialization of the common environment /// @ingroup environment_group -/// @param[in] builder The in-process environment builder state to install as the platform environment +/// @param[in] builder The in-process environment builder state to install as the common environment /// @return True if the environment was sucessfully initialized with the provided builder state. False /// if the environment had already been initialized by a prior call /// #[no_mangle] -pub extern "C" fn env_builder_init_platform_env(builder: env_builder_t) -> bool { +pub extern "C" fn env_builder_init_common_env(builder: env_builder_t) -> bool { let builder = builder.into_inner(); - builder.try_init_platform_env().is_ok() + builder.try_init_common_env().is_ok() } -/// @brief Sets the working directory for the platform environment +/// @brief Sets the working directory for the environment /// @ingroup environment_group /// @param[in] builder A pointer to the in-process environment builder state /// @param[in] path A C-style string specifying a path to a working directory, to search for modules to load. @@ -1182,7 +1182,7 @@ pub extern "C" fn env_builder_set_working_dir(builder: *mut env_builder_t, path: *builder_arg_ref = builder.into(); } -/// @brief Sets the config directory for the platform environment. A directory at the specified path +/// @brief Sets the config directory for the environment. A directory at the specified path /// will be created, and its contents populated with default values, if one does not already exist /// @ingroup environment_group /// @param[in] builder A pointer to the in-process environment builder state @@ -1200,7 +1200,7 @@ pub extern "C" fn env_builder_set_config_dir(builder: *mut env_builder_t, path: *builder_arg_ref = builder.into(); } -/// @brief Configures the platform environment so that no config directory will be read nor created +/// @brief Configures the environment so that no config directory will be read nor created /// @ingroup environment_group /// @param[in] builder A pointer to the in-process environment builder state /// @@ -1212,7 +1212,7 @@ pub extern "C" fn env_builder_disable_config_dir(builder: *mut env_builder_t) { *builder_arg_ref = builder.into(); } -/// @brief Configures the platform environment for use in unit testing +/// @brief Configures the environment for use in unit testing /// @ingroup environment_group /// @param[in] builder A pointer to the in-process environment builder state /// @param[in] is_test True if the environment is a unit-test environment, False otherwise diff --git a/lib/src/metta/runner/environment.rs b/lib/src/metta/runner/environment.rs index 52f480329..01773fe6f 100644 --- a/lib/src/metta/runner/environment.rs +++ b/lib/src/metta/runner/environment.rs @@ -7,10 +7,10 @@ use std::sync::Arc; use directories::ProjectDirs; -/// Contains state and platform interfaces shared by all MeTTa runners. This includes config settings +/// Contains state and host platform interfaces shared by all MeTTa runners. This includes config settings /// and logger /// -/// Generally there will be only one environment object needed, and it can be accessed by calling the [platform_env] method +/// Generally there will be only one environment object needed, and it can be accessed by calling the [common_env] method #[derive(Debug)] pub struct Environment { config_dir: Option, @@ -22,18 +22,18 @@ pub struct Environment { const DEFAULT_INIT_METTA: &[u8] = include_bytes!("init.default.metta"); -static PLATFORM_ENV: std::sync::OnceLock> = std::sync::OnceLock::new(); +static COMMON_ENV: std::sync::OnceLock> = std::sync::OnceLock::new(); impl Environment { - /// Returns a reference to the shared "platform" Environment - pub fn platform_env() -> &'static Self { - PLATFORM_ENV.get_or_init(|| Arc::new(EnvBuilder::new().build())) + /// Returns a reference to the shared common Environment + pub fn common_env() -> &'static Self { + COMMON_ENV.get_or_init(|| Arc::new(EnvBuilder::new().build())) } - /// Internal function to get a copy of the platform Environment's Arc ptr - pub(crate) fn platform_env_arc() -> Arc { - PLATFORM_ENV.get_or_init(|| Arc::new(EnvBuilder::new().build())).clone() + /// Internal function to get a copy of the common Environment's Arc ptr + pub(crate) fn common_env_arc() -> Arc { + COMMON_ENV.get_or_init(|| Arc::new(EnvBuilder::new().build())).clone() } /// Returns the Path to the config dir, in an OS-specific location @@ -85,7 +85,7 @@ impl EnvBuilder { /// Returns a new EnvBuilder, to set the parameters for the MeTTa Environment /// /// NOTE: Unless otherwise specified by calling either [set_no_config_dir] or [set_config_dir], the - /// [Environment] will be configured using the OS-Specific platform configuration files. + /// [Environment] will be configured using files in the OS-Specific configuration file locations. /// /// Depending on the host OS, the config directory locations will be: /// * Linux: ~/.config/metta/ @@ -153,21 +153,21 @@ impl EnvBuilder { self } - /// Initializes the shared platform Environment, accessible with [platform_env] + /// Initializes the shared common Environment, accessible with [common_env] /// - /// NOTE: This method will panic if the platform Environment has already been initialized - pub fn init_platform_env(self) { - self.try_init_platform_env().expect("Fatal Error: Platform Environment already initialized"); + /// NOTE: This method will panic if the common Environment has already been initialized + pub fn init_common_env(self) { + self.try_init_common_env().expect("Fatal Error: Common Environment already initialized"); } - /// Initializes the shared platform Environment. Non-panicking version of [init_platform_env] - pub fn try_init_platform_env(self) -> Result<(), &'static str> { - PLATFORM_ENV.set(Arc::new(self.build())).map_err(|_| "Platform Environment already initialized") + /// Initializes the shared common Environment. Non-panicking version of [init_common_env] + pub fn try_init_common_env(self) -> Result<(), &'static str> { + COMMON_ENV.set(Arc::new(self.build())).map_err(|_| "Common Environment already initialized") } /// Returns a newly created Environment from the builder configuration /// - /// NOTE: Creating owned Environments is usually not necessary. It is usually sufficient to use the [platform_env] method. + /// NOTE: Creating owned Environments is usually not necessary. It is usually sufficient to use the [common_env] method. pub(crate) fn build(self) -> Environment { let mut env = self.env; diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index fdae2f9b7..6dc571d01 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -83,14 +83,14 @@ impl Metta { /// A 1-line method to create a fully initialized MeTTa interpreter /// - /// NOTE: pass `None` for `env_builder` to use the platform environment + /// NOTE: pass `None` for `env_builder` to use the common environment pub fn new(env_builder: Option) -> Metta { Self::new_with_stdlib_loader(|_| {}, env_builder) } /// Create and initialize a MeTTa interpreter with a language-specific stdlib /// - /// NOTE: pass `None` for `env_builder` to use the platform environment + /// NOTE: pass `None` for `env_builder` to use the common environment pub fn new_with_stdlib_loader(loader: F, env_builder: Option) -> Metta where F: FnOnce(&Self) { @@ -118,14 +118,14 @@ impl Metta { /// Returns a new MeTTa interpreter, using the provided Space, Tokenizer /// - /// NOTE: If `env_builder` is `None`, the platform environment will be used + /// NOTE: If `env_builder` is `None`, the common environment will be used /// NOTE: This function does not load any stdlib atoms, nor run the [Environment]'s 'init.metta' pub fn new_with_space(space: DynSpace, tokenizer: Shared, env_builder: Option) -> Self { let settings = Shared::new(HashMap::new()); let modules = Shared::new(HashMap::new()); let environment = match env_builder { Some(env_builder) => Arc::new(env_builder.build()), - None => Environment::platform_env_arc() + None => Environment::common_env_arc() }; let contents = MettaContents{ space, diff --git a/python/hyperon/runner.py b/python/hyperon/runner.py index cdcf87402..83e56d864 100644 --- a/python/hyperon/runner.py +++ b/python/hyperon/runner.py @@ -177,20 +177,20 @@ def run(self, program, flat=False): return [[Atom._from_catom(catom) for catom in result] for result in results] class Environment: - """This class contains the API for shared platform configuration""" + """This class contains the API for configuring the host platform interface used by MeTTa""" def config_dir(): - """Returns the config dir in the platform environment""" + """Returns the config dir in the common environment""" path = hp.environment_config_dir() if (len(path) > 0): return path else: return None - def init_platform_env(working_dir = None, config_dir = None, disable_config = False, is_test = False, include_paths = []): - """Initialize the platform environment with the supplied args""" + def init_common_env(working_dir = None, config_dir = None, disable_config = False, is_test = False, include_paths = []): + """Initialize the common environment with the supplied args""" builder = Environment.custom_env(working_dir, config_dir, disable_config, is_test, include_paths) - return hp.env_builder_init_platform_env(builder) + return hp.env_builder_init_common_env(builder) def test_env(): """Returns an EnvBuilder object specifying a unit-test environment, that can be used to init a MeTTa runner""" diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index f551e7e2d..992d14f29 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -839,15 +839,15 @@ PYBIND11_MODULE(hyperonpy, m) { py::class_(m, "EnvBuilder"); m.def("environment_config_dir", []() { return func_to_string_no_arg((write_to_buf_no_arg_func_t)&environment_config_dir); - }, "Return the config_dir for the platform environment"); + }, "Return the config_dir for the common environment"); m.def("env_builder_start", []() { return EnvBuilder(env_builder_start()); }, "Begin initialization of the environment"); - m.def("env_builder_use_default", []() { return EnvBuilder(env_builder_use_default()); }, "Use the platform environment"); + m.def("env_builder_use_default", []() { return EnvBuilder(env_builder_use_default()); }, "Use the common environment"); m.def("env_builder_use_test_env", []() { return EnvBuilder(env_builder_use_test_env()); }, "Use an environment for unit testing"); - m.def("env_builder_init_platform_env", [](EnvBuilder builder) { return env_builder_init_platform_env(builder.obj); }, "Finish initialization of the platform environment"); - m.def("env_builder_set_working_dir", [](EnvBuilder& builder, std::string path) { env_builder_set_working_dir(builder.ptr(), path.c_str()); }, "Sets the working dir in the platform environment"); - m.def("env_builder_set_config_dir", [](EnvBuilder& builder, std::string path) { env_builder_set_config_dir(builder.ptr(), path.c_str()); }, "Sets the config dir in the platform environment"); - m.def("env_builder_disable_config_dir", [](EnvBuilder& builder) { env_builder_disable_config_dir(builder.ptr()); }, "Disables the config dir in the platform environment"); - m.def("env_builder_set_is_test", [](EnvBuilder& builder, bool is_test) { env_builder_set_is_test(builder.ptr(), is_test); }, "Disables the config dir in the platform environment"); - m.def("env_builder_add_include_path", [](EnvBuilder& builder, std::string path) { env_builder_add_include_path(builder.ptr(), path.c_str()); }, "Adds an include path to the platform environment"); + m.def("env_builder_init_common_env", [](EnvBuilder builder) { return env_builder_init_common_env(builder.obj); }, "Finish initialization of the common environment"); + m.def("env_builder_set_working_dir", [](EnvBuilder& builder, std::string path) { env_builder_set_working_dir(builder.ptr(), path.c_str()); }, "Sets the working dir in the environment"); + m.def("env_builder_set_config_dir", [](EnvBuilder& builder, std::string path) { env_builder_set_config_dir(builder.ptr(), path.c_str()); }, "Sets the config dir in the environment"); + m.def("env_builder_disable_config_dir", [](EnvBuilder& builder) { env_builder_disable_config_dir(builder.ptr()); }, "Disables the config dir in the environment"); + m.def("env_builder_set_is_test", [](EnvBuilder& builder, bool is_test) { env_builder_set_is_test(builder.ptr(), is_test); }, "Disables the config dir in the environment"); + m.def("env_builder_add_include_path", [](EnvBuilder& builder, std::string path) { env_builder_add_include_path(builder.ptr(), path.c_str()); }, "Adds an include path to the environment"); } diff --git a/python/tests/test_environment.py b/python/tests/test_environment.py index 4bcdb18aa..3ef64be7a 100644 --- a/python/tests/test_environment.py +++ b/python/tests/test_environment.py @@ -8,7 +8,7 @@ def __init__(self, methodName): super().__init__(methodName) def testEnvironment(self): - self.assertTrue(Environment.init_platform_env(config_dir = "/tmp/test_dir")) + self.assertTrue(Environment.init_common_env(config_dir = "/tmp/test_dir")) self.assertEqual(Environment.config_dir(), "/tmp/test_dir") - self.assertFalse(Environment.init_platform_env(disable_config = True)) + self.assertFalse(Environment.init_common_env(disable_config = True)) diff --git a/repl/src/metta_shim.rs b/repl/src/metta_shim.rs index 06c36828a..a790a70c3 100644 --- a/repl/src/metta_shim.rs +++ b/repl/src/metta_shim.rs @@ -81,7 +81,7 @@ pub mod metta_interface_mod { confirm_hyperonpy_version(">=0.1.0, <0.2.0")?; //Initialize the Hyperon environment - let new_shim = MettaShim::init_platform_env(working_dir, include_paths)?; + let new_shim = MettaShim::init_common_env(working_dir, include_paths)?; Ok(new_shim) }() { @@ -93,7 +93,7 @@ pub mod metta_interface_mod { } } - pub fn init_platform_env(working_dir: PathBuf, include_paths: Vec) -> Result { + pub fn init_common_env(working_dir: PathBuf, include_paths: Vec) -> Result { match Python::with_gil(|py| -> PyResult<(Py, Py)> { let py_mod = PyModule::from_code(py, Self::PY_CODE, "", "")?; let init_func = py_mod.getattr("init_metta")?; @@ -324,12 +324,10 @@ pub mod metta_interface_mod { use hyperon::atom::{Grounded, ExecError, match_by_equality}; use hyperon::matcher::MatchResultIter; use hyperon::metta::*; - use hyperon::metta::environment::EnvBuilder; use hyperon::metta::text::SExprParser; use hyperon::ExpressionAtom; use hyperon::Atom; - use hyperon::metta::environment::Environment; - use hyperon::metta::runner::{Metta, RunnerState, atom_is_error}; + use hyperon::metta::runner::{Metta, RunnerState, atom_is_error, Environment, EnvBuilder}; use super::{strip_quotes, exec_state_prepare, exec_state_should_break}; pub use hyperon::metta::text::SyntaxNodeType as SyntaxNodeType; @@ -343,7 +341,7 @@ pub mod metta_interface_mod { pub fn new(working_dir: PathBuf, include_paths: Vec) -> Self { match || -> Result<_, String> { - let new_shim = MettaShim::init_platform_env(working_dir, include_paths)?; + let new_shim = MettaShim::init_common_env(working_dir, include_paths)?; Ok(new_shim) }() { Ok(shim) => shim, @@ -354,11 +352,11 @@ pub mod metta_interface_mod { } } - pub fn init_platform_env(working_dir: PathBuf, include_paths: Vec) -> Result { + pub fn init_common_env(working_dir: PathBuf, include_paths: Vec) -> Result { EnvBuilder::new() .set_working_dir(Some(&working_dir)) .add_include_paths(include_paths) - .init_platform_env(); + .init_common_env(); let new_shim = MettaShim { metta: Metta::new(None), @@ -436,7 +434,7 @@ pub mod metta_interface_mod { } pub fn config_dir(&self) -> Option<&Path> { - Environment::platform_env().config_dir() + Environment::common_env().config_dir() } pub fn get_config_atom(&mut self, config_name: &str) -> Option { diff --git a/repl/src/py_shim.py b/repl/src/py_shim.py index b678f8239..62064f022 100644 --- a/repl/src/py_shim.py +++ b/repl/src/py_shim.py @@ -2,7 +2,7 @@ from hyperon import * def init_metta(working_dir, include_paths): - Environment.init_platform_env(working_dir = working_dir, include_paths = include_paths) + Environment.init_common_env(working_dir = working_dir, include_paths = include_paths) return MeTTa() def load_metta_module(metta, mod_path): From a3808f196ae4fdc212da537709de1520308414e0 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Thu, 12 Oct 2023 17:58:02 -0700 Subject: [PATCH 22/28] Updating the way parse errors are returned from the C & Python parse APIs, and adding some infrastructure around error expression atoms --- c/src/metta.rs | 76 +++++++++++++--------------------- lib/src/metta/mod.rs | 50 ++++++++++++++++++++++ lib/src/metta/runner/mod.rs | 9 ---- python/hyperon/base.py | 12 +++--- python/hyperonpy.cpp | 11 ++--- python/tests/test_sexparser.py | 15 +++---- repl/src/py_shim.py | 10 ++--- 7 files changed, 100 insertions(+), 83 deletions(-) diff --git a/c/src/metta.rs b/c/src/metta.rs index 625394cdc..625fe7cce 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -1,4 +1,5 @@ use hyperon::common::shared::Shared; +use hyperon::metta::error_atom; use hyperon::space::DynSpace; use hyperon::metta::text::*; use hyperon::metta::interpreter; @@ -149,18 +150,14 @@ pub extern "C" fn tokenizer_clone(tokenizer: *const tokenizer_t) -> tokenizer_t #[repr(C)] pub struct sexpr_parser_t { /// Internal. Should not be accessed directly - parser: *mut RustSExprParser, - err_string: *mut c_char, + parser: *mut RustSExprParser } struct RustSExprParser(SExprParser<'static>); impl From> for sexpr_parser_t { fn from(parser: SExprParser<'static>) -> Self { - Self{ - parser: Box::into_raw(Box::new(RustSExprParser(parser))), - err_string: core::ptr::null_mut() - } + Self{parser: Box::into_raw(Box::new(RustSExprParser(parser)))} } } @@ -176,16 +173,6 @@ impl sexpr_parser_t { } } -impl sexpr_parser_t { - fn free_err_string(&mut self) { - if !self.err_string.is_null() { - let string = unsafe{ std::ffi::CString::from_raw(self.err_string) }; - drop(string); - self.err_string = core::ptr::null_mut(); - } - } -} - /// @brief Creates a new S-Expression Parser /// @ingroup tokenizer_and_parser_group /// @param[in] text A C-style string containing the input text to parse @@ -228,45 +215,24 @@ pub extern "C" fn sexpr_parser_free(parser: sexpr_parser_t) { /// @ingroup tokenizer_and_parser_group /// @param[in] parser A pointer to the Parser, which is associated with the text to parse /// @param[in] tokenizer A pointer to the Tokenizer, to use to interpret atoms within the expression -/// @return The new `atom_t`, which may be an Expression atom with many child atoms +/// @return The new `atom_t`, which may be an Expression atom with many child atoms. Returns a `none` +/// atom if parsing is finished, or an error expression atom if a parse error occurred. /// @note The caller must take ownership responsibility for the returned `atom_t`, and ultimately free -/// it with `atom_free()` or pass it to another function that takes ownership responsibility +/// it with `atom_free()` or pass it to another function that takes ownership responsibility /// #[no_mangle] pub extern "C" fn sexpr_parser_parse( parser: *mut sexpr_parser_t, tokenizer: *const tokenizer_t) -> atom_t { - let parser = unsafe{ &mut *parser }; - parser.free_err_string(); - let rust_parser = parser.borrow_mut(); + let parser = unsafe{ &mut *parser }.borrow_mut(); let tokenizer = unsafe{ &*tokenizer }.borrow_inner(); - match rust_parser.parse(tokenizer) { + match parser.parse(tokenizer) { Ok(atom) => atom.into(), - Err(err) => { - let err_cstring = std::ffi::CString::new(err).unwrap(); - parser.err_string = err_cstring.into_raw(); - atom_t::null() - } + Err(err) => error_atom(None, None, err).into() } } -/// @brief Returns the error string associated with the last `sexpr_parser_parse` call -/// @ingroup tokenizer_and_parser_group -/// @param[in] parser A pointer to the Parser, which is associated with the text to parse -/// @return A pointer to the C-string containing the parse error that occurred, or NULL if no -/// parse error occurred -/// @warning The returned pointer should NOT be freed. It must never be accessed after the -/// sexpr_parser_t has been freed, or any subsequent `sexpr_parser_parse` or -/// `sexpr_parser_parse_to_syntax_tree` has been made. -/// -#[no_mangle] -pub extern "C" fn sexpr_parser_err_str( - parser: *mut sexpr_parser_t) -> *const c_char { - let parser = unsafe{ &*parser }; - parser.err_string -} - /// @brief Represents a component in a syntax tree created by parsing MeTTa code /// @ingroup tokenizer_and_parser_group /// @note `syntax_node_t` objects must be freed with `syntax_node_free()` @@ -371,10 +337,8 @@ pub type c_syntax_node_callback_t = extern "C" fn(node: *const syntax_node_t, co #[no_mangle] pub extern "C" fn sexpr_parser_parse_to_syntax_tree(parser: *mut sexpr_parser_t) -> syntax_node_t { - let parser = unsafe{ &mut *parser }; - parser.free_err_string(); - let rust_parser = parser.borrow_mut(); - rust_parser.parse_to_syntax_tree().into() + let parser = unsafe{ &mut *parser }.borrow_mut(); + parser.parse_to_syntax_tree().into() } /// @brief Frees a syntax_node_t @@ -477,7 +441,23 @@ pub extern "C" fn syntax_node_src_range(node: *const syntax_node_t, range_start: #[no_mangle] pub extern "C" fn atom_is_error(atom: *const atom_ref_t) -> bool { let atom = unsafe{ &*atom }.borrow(); - hyperon::metta::runner::atom_is_error(atom) + hyperon::metta::atom_is_error(atom) +} + +/// @brief Renders the text message from an error expression atom into a buffer +/// @ingroup metta_language_group +/// @param[in] atom The error expression atom from which to extract the error message +/// @param[out] buf A buffer into which the text will be rendered +/// @param[in] buf_len The maximum allocated size of `buf` +/// @return The length of the message string, minus the string terminator character. If +/// `return_value > buf_len + 1`, then the text was not fully rendered and this function should be +/// called again with a larger buffer. +/// @warning The `atom` argument must be an error expression, otherwise this function will panic +/// +#[no_mangle] +pub extern "C" fn atom_error_message(atom: *const atom_ref_t, buf: *mut c_char, buf_len: usize) -> usize { + let atom = unsafe{ &*atom }.borrow(); + write_into_buf(hyperon::metta::atom_error_message(atom), buf, buf_len) } /// @brief Creates a Symbol atom for the special MeTTa symbol: "%Undefined%" diff --git a/lib/src/metta/mod.rs b/lib/src/metta/mod.rs index 94a82e634..7db8df5a4 100644 --- a/lib/src/metta/mod.rs +++ b/lib/src/metta/mod.rs @@ -41,6 +41,56 @@ pub const UNIFY_SYMBOL : Atom = sym!("unify"); pub const DECONS_SYMBOL : Atom = sym!("decons"); pub const CONS_SYMBOL : Atom = sym!("cons"); +/// Initializes an error expression atom +pub fn error_atom(err_atom: Option, err_code: Option, message: String) -> Atom { + let err_atom = match err_atom { + Some(err_atom) => err_atom, + None => EMPTY_SYMBOL, + }; + if let Some(err_code) = err_code { + Atom::expr([ERROR_SYMBOL, err_atom, err_code, Atom::sym(message)]) + } else { + Atom::expr([ERROR_SYMBOL, err_atom, Atom::sym(message)]) + } +} + +/// Tests whether or not an atom is an error expression +pub fn atom_is_error(atom: &Atom) -> bool { + match atom { + Atom::Expression(expr) => { + expr.children().len() > 0 && expr.children()[0] == ERROR_SYMBOL + }, + _ => false, + } +} + +/// Returns a message string from an error expression +/// +/// NOTE: this function will panic if the suppoed atom is not a valid error expression +pub fn atom_error_message(atom: &Atom) -> &str { + const PANIC_STR: &str = "Atom is not error expression"; + match atom { + Atom::Expression(expr) => { + let sym_atom = match expr.children().len() { + 3 => expr.children().get(2).unwrap(), + 4 => expr.children().get(3).unwrap(), + _ => panic!("{}", PANIC_STR) + }; + let sym_atom = <&SymbolAtom>::try_from(sym_atom).expect(PANIC_STR); + sym_atom.name() + }, + _ => panic!("{}", PANIC_STR) + } +} + +//QUESTION: These functions below seem to only be used in tests. If that's the case, should we +// move them into a test-only module so they aren't exposed as part of the public API? +//Alternatively rewrite the tests to use a runner? +// +//Also, a "Metta::parse_atoms() -> Vec" method +// might be useful to parse atoms with the context of a runner's tokenizer as a compliment to +// RunnerState::new_with_atoms + // TODO: use stdlib to parse input text pub fn metta_space(text: &str) -> GroundingSpace { let tokenizer = common_tokenizer(); diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index 6dc571d01..8bb98290d 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -32,15 +32,6 @@ mod arithmetics; const EXEC_SYMBOL : Atom = sym!("!"); -pub fn atom_is_error(atom: &Atom) -> bool { - match atom { - Atom::Expression(expr) => { - expr.children().len() > 0 && expr.children()[0] == ERROR_SYMBOL - }, - _ => false, - } -} - #[derive(Clone, Debug)] pub struct Metta(Rc); diff --git a/python/hyperon/base.py b/python/hyperon/base.py index 31093614c..c46d79db6 100644 --- a/python/hyperon/base.py +++ b/python/hyperon/base.py @@ -378,13 +378,11 @@ def parse(self, tokenizer): Parses the S-expression using the provided Tokenizer. """ catom = self.cparser.parse(tokenizer.ctokenizer) - return Atom._from_catom(catom) if catom is not None else None - - def parse_err(self): - """ - Returns the parse error string from the previous parse, or None. - """ - return self.cparser.sexpr_parser_err_str() + if (catom is None): + return None + if (hp.atom_is_error(catom)): + raise SyntaxError(hp.atom_error_message(catom)) + return Atom._from_catom(catom) def parse_to_syntax_tree(self): """ diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index 992d14f29..24c8c976d 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -466,11 +466,6 @@ struct CSExprParser { return !atom_is_null(&atom) ? py::cast(CAtom(atom)) : py::none(); } - py::object err_str() { - const char* err_str = sexpr_parser_err_str(&this->parser); - return err_str != NULL ? py::cast(std::string(err_str)) : py::none(); - } - py::object parse_to_syntax_tree() { syntax_node_t root_node = sexpr_parser_parse_to_syntax_tree(&this->parser); return !syntax_node_is_null(&root_node) ? py::cast(CSyntaxNode(root_node)) : py::none(); @@ -525,6 +520,9 @@ PYBIND11_MODULE(hyperonpy, m) { m.def("atom_eq", [](CAtom& a, CAtom& b) -> bool { return atom_eq(a.ptr(), b.ptr()); }, "Test if two atoms are equal"); m.def("atom_is_error", [](CAtom& atom) -> bool { return atom_is_error(atom.ptr()); }, "Returns True if an atom is a MeTTa error expression"); + m.def("atom_error_message", [](CAtom& atom) { + return func_to_string((write_to_buf_func_t)&atom_error_message, atom.ptr()); + }, "Renders the error message from an error expression atom"); m.def("atom_to_str", [](CAtom& atom) { return func_to_string((write_to_buf_func_t)&atom_to_str, atom.ptr()); }, "Convert atom to human readable string"); @@ -739,8 +737,7 @@ PYBIND11_MODULE(hyperonpy, m) { py::class_(m, "CSExprParser") .def(py::init()) - .def("parse", &CSExprParser::parse, "Return next parser atom or None") - .def("sexpr_parser_err_str", &CSExprParser::err_str, "Return the parse error from the previous parse operation or None") + .def("parse", &CSExprParser::parse, "Return next parsed atom, None, or an error expression") .def("parse_to_syntax_tree", &CSExprParser::parse_to_syntax_tree, "Return next parser atom or None, as a syntax node at the root of a syntax tree"); py::class_(m, "CStepResult") diff --git a/python/tests/test_sexparser.py b/python/tests/test_sexparser.py index e916377a5..4d8cad319 100644 --- a/python/tests/test_sexparser.py +++ b/python/tests/test_sexparser.py @@ -28,16 +28,17 @@ def testParseToSyntaxNodes(self): def testParseErr(self): tokenizer = Tokenizer() parser = SExprParser("(+ one \"one") - parsed_atom = parser.parse(tokenizer) - self.assertEqual(parsed_atom, None) - self.assertEqual(parser.parse_err(), "Unclosed String Literal") + try: + parsed_atom = parser.parse(tokenizer) + except SyntaxError as e: + self.assertEqual(e.args[0], 'Unclosed String Literal') parser = SExprParser("(+ one \"one\"") - parsed_atom = parser.parse(tokenizer) - self.assertEqual(parsed_atom, None) - self.assertEqual(parser.parse_err(), "Unexpected end of expression") + try: + parsed_atom = parser.parse(tokenizer) + except SyntaxError as e: + self.assertEqual(e.args[0], 'Unexpected end of expression') parser = SExprParser("(+ one \"one\")") parsed_atom = parser.parse(tokenizer) self.assertTrue(parsed_atom is not None) - self.assertEqual(parser.parse_err(), None) diff --git a/repl/src/py_shim.py b/repl/src/py_shim.py index 62064f022..a0ef111bb 100644 --- a/repl/src/py_shim.py +++ b/repl/src/py_shim.py @@ -22,12 +22,12 @@ def parse_line(metta, line): tokenizer = metta.tokenizer() parser = SExprParser(line) while True: - parsed_atom = parser.parse(tokenizer) - if (parsed_atom is None): - if (parser.parse_err() is None): + try: + parsed_atom = parser.parse(tokenizer) + if (parsed_atom is None): return None - else: - return parser.parse_err() + except SyntaxError as e: + return e.args[0] def parse_line_to_syntax_tree(line): leaf_node_types = []; From 790ad1e0869685f6dfc0ef600b45e9b744580919 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Mon, 16 Oct 2023 18:01:23 -0700 Subject: [PATCH 23/28] Adding back the ability to create a fully-initialized top-level runner with a custom space --- c/src/metta.rs | 20 +++++++++++++------- c/tests/util.c | 4 +++- lib/src/metta/runner/mod.rs | 25 +++++++++++++++---------- python/hyperon/runner.py | 6 ++++-- python/hyperonpy.cpp | 4 ++-- python/tests/test_custom_space.py | 15 +++++++++++++++ 6 files changed, 52 insertions(+), 22 deletions(-) diff --git a/c/src/metta.rs b/c/src/metta.rs index 625fe7cce..b78871a58 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -732,15 +732,21 @@ pub type c_stdlib_loader_callback_t = extern "C" fn(metta: *mut metta_t, context /// @brief Creates a new top-level MeTTa Interpreter, bootstrapped with the a custom stdlib /// @ingroup interpreter_group +/// @param[in] space A pointer to a handle for the Space for use by the Interpreter +/// @param[in] environment An `env_builder_t` handle to configure the environment to use +/// @param[in] callback The c_stdlib_loader_callback_t function to load the stdlib +/// @param[in] context A pointer to a caller-defined structure to facilitate communication with the `callback` function /// @return A `metta_t` handle to the newly created Interpreter /// @note The caller must take ownership responsibility for the returned `metta_t`, and free it with `metta_free()` /// @note Most callers can simply call `metta_new`. This function is provided to support languages /// with their own stdlib, that needs to be loaded before the init.metta file is run /// #[no_mangle] -pub extern "C" fn metta_new_with_environment_and_stdlib(env_builder: env_builder_t, - callback: c_stdlib_loader_callback_t, context: *mut c_void) -> metta_t +pub extern "C" fn metta_new_with_space_environment_and_stdlib(space: *mut space_t, + env_builder: env_builder_t, callback: c_stdlib_loader_callback_t, context: *mut c_void) -> metta_t { + let dyn_space = unsafe{ &*space }.borrow(); + let env_builder = if env_builder.is_default() { None } else { @@ -750,21 +756,21 @@ pub extern "C" fn metta_new_with_environment_and_stdlib(env_builder: env_builder let metta = Metta::new_with_stdlib_loader(|metta| { let mut metta = metta_t{metta: (metta as *const Metta).cast_mut().cast()}; callback(&mut metta, context); - }, env_builder); + }, Some(dyn_space.clone()), env_builder); metta.into() } -/// @brief Creates a new MeTTa Interpreter with a provided Space, Tokenizer and Environment +/// @brief Creates a new core MeTTa Interpreter, with no loaded stdlib nor initialization /// @ingroup interpreter_group /// @param[in] space A pointer to a handle for the Space for use by the Interpreter /// @param[in] tokenizer A pointer to a handle for the Tokenizer for use by the Interpreter -/// @param[in] environment An `environment_t` handle representing the environment to use +/// @param[in] environment An `env_builder_t` handle to configure the environment to use /// @return A `metta_t` handle to the newly created Interpreter /// @note The caller must take ownership responsibility for the returned `metta_t`, and free it with `metta_free()` /// @note This function does not load any stdlib, nor does it run the `init.metta` file from the environment /// #[no_mangle] -pub extern "C" fn metta_new_with_space(space: *mut space_t, tokenizer: *mut tokenizer_t, env_builder: env_builder_t) -> metta_t { +pub extern "C" fn metta_new_core(space: *mut space_t, tokenizer: *mut tokenizer_t, env_builder: env_builder_t) -> metta_t { let dyn_space = unsafe{ &*space }.borrow(); let tokenizer = unsafe{ &*tokenizer }.clone_handle(); let env_builder = if env_builder.is_default() { @@ -772,7 +778,7 @@ pub extern "C" fn metta_new_with_space(space: *mut space_t, tokenizer: *mut toke } else { Some(env_builder.into_inner()) }; - let metta = Metta::new_with_space(dyn_space.clone(), tokenizer, env_builder); + let metta = Metta::new_core(dyn_space.clone(), tokenizer, env_builder); metta.into() } diff --git a/c/tests/util.c b/c/tests/util.c index 0ea60a959..6378cf1d5 100644 --- a/c/tests/util.c +++ b/c/tests/util.c @@ -33,6 +33,8 @@ atom_t expr(atom_t atom, ...) { void noop_metta_init(metta_t* metta, void* context) {} metta_t new_test_metta(void) { - metta_t metta = metta_new_with_environment_and_stdlib(env_builder_use_test_env(), &noop_metta_init, NULL); + space_t space = space_new_grounding_space(); + metta_t metta = metta_new_with_space_environment_and_stdlib(&space, env_builder_use_test_env(), &noop_metta_init, NULL); + space_free(space); return metta; } \ No newline at end of file diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index 8bb98290d..d5a0d01da 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -76,18 +76,23 @@ impl Metta { /// /// NOTE: pass `None` for `env_builder` to use the common environment pub fn new(env_builder: Option) -> Metta { - Self::new_with_stdlib_loader(|_| {}, env_builder) + Self::new_with_stdlib_loader(|_| {}, None, env_builder) } /// Create and initialize a MeTTa interpreter with a language-specific stdlib /// - /// NOTE: pass `None` for `env_builder` to use the common environment - pub fn new_with_stdlib_loader(loader: F, env_builder: Option) -> Metta + /// NOTE: pass `None` for space to create a new [GroundingSpace] + /// pass `None` for `env_builder` to use the common environment + pub fn new_with_stdlib_loader(loader: F, space: Option, env_builder: Option) -> Metta where F: FnOnce(&Self) { + let space = match space { + Some(space) => space, + None => DynSpace::new(GroundingSpace::new()) + }; + //Create the raw MeTTa runner - let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), - Shared::new(Tokenizer::new()), env_builder); + let metta = Metta::new_core(space, Shared::new(Tokenizer::new()), env_builder); // TODO: Reverse the loading order between the Rust stdlib and user-supplied stdlib loader, // because user-supplied stdlibs might need to build on top of the Rust stdlib. @@ -107,11 +112,11 @@ impl Metta { metta } - /// Returns a new MeTTa interpreter, using the provided Space, Tokenizer + /// Returns a new core MeTTa interpreter without any stdlib or initialization /// /// NOTE: If `env_builder` is `None`, the common environment will be used /// NOTE: This function does not load any stdlib atoms, nor run the [Environment]'s 'init.metta' - pub fn new_with_space(space: DynSpace, tokenizer: Shared, env_builder: Option) -> Self { + pub fn new_core(space: DynSpace, tokenizer: Shared, env_builder: Option) -> Self { let settings = Shared::new(HashMap::new()); let modules = Shared::new(HashMap::new()); let environment = match env_builder { @@ -430,7 +435,7 @@ mod tests { (foo b) "; - let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); + let metta = Metta::new_core(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); metta.set_setting("type-check".into(), sym!("auto")); let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); @@ -444,7 +449,7 @@ mod tests { !(foo b) "; - let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); + let metta = Metta::new_core(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); metta.set_setting("type-check".into(), sym!("auto")); let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); @@ -499,7 +504,7 @@ mod tests { !(foo a) "; - let metta = Metta::new_with_space(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); + let metta = Metta::new_core(DynSpace::new(GroundingSpace::new()), Shared::new(Tokenizer::new()), Some(EnvBuilder::test_env())); metta.set_setting("type-check".into(), sym!("auto")); let result = metta.run(SExprParser::new(program)); assert_eq!(result, Ok(vec![vec![expr!("Error" ("foo" "b") "BadType")]])); diff --git a/python/hyperon/runner.py b/python/hyperon/runner.py index 83e56d864..3ba062ac2 100644 --- a/python/hyperon/runner.py +++ b/python/hyperon/runner.py @@ -49,13 +49,15 @@ def current_results(self, flat=False): class MeTTa: """This class contains the MeTTa program execution utilities""" - def __init__(self, cmetta = None, env_builder = None): + def __init__(self, cmetta = None, space = None, env_builder = None): if cmetta is not None: self.cmetta = cmetta else: + if space is None: + space = GroundingSpaceRef() if env_builder is None: env_builder = hp.env_builder_use_default() - self.cmetta = hp.metta_new(env_builder) + self.cmetta = hp.metta_new(space.cspace, env_builder) def __del__(self): hp.metta_free(self.cmetta) diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index 24c8c976d..9459f874f 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -788,8 +788,8 @@ PYBIND11_MODULE(hyperonpy, m) { ADD_SYMBOL(VOID, "Void"); py::class_(m, "CMetta"); - m.def("metta_new", [](EnvBuilder env_builder) { - return CMetta(metta_new_with_environment_and_stdlib(env_builder.obj, &run_python_loader_callback, NULL)); + m.def("metta_new", [](CSpace space, EnvBuilder env_builder) { + return CMetta(metta_new_with_space_environment_and_stdlib(space.ptr(), env_builder.obj, &run_python_loader_callback, NULL)); }, "New MeTTa interpreter instance"); m.def("metta_free", [](CMetta metta) { metta_free(metta.obj); }, "Free MeTTa interpreter"); m.def("metta_search_path_cnt", [](CMetta metta) { return metta_search_path_cnt(metta.ptr()); }, "Returns the number of module search paths in the runner's environment"); diff --git a/python/tests/test_custom_space.py b/python/tests/test_custom_space.py index 838d53c83..85cf2e132 100644 --- a/python/tests/test_custom_space.py +++ b/python/tests/test_custom_space.py @@ -137,5 +137,20 @@ def test_match_nested_custom_space(self): result = runner.run("!(match nested (A $x) $x)") self.assertEqual([[S("B")]], result) + def test_runner_with_custom_space(self): + + test_space = TestSpace() + test_space.test_attrib = "Test Space Payload Attrib" + + space = SpaceRef(test_space) + space.add_atom(E(S("="), S("key"), S("val"))) + + metta = MeTTa(space=space) + + self.assertEqual(metta.space().get_payload().test_attrib, "Test Space Payload Attrib") + self.assertEqualMettaRunnerResults(metta.run("!(match &self (= key $val) $val)"), + [[S("val")]] + ) + if __name__ == "__main__": unittest.main() From 8254d3f88b3f042daeba9d8bf65728d6a9e9342d Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Tue, 17 Oct 2023 11:29:07 -0700 Subject: [PATCH 24/28] Adding runner_eq() function to compare the runners referenced by runner handles Re-adding runner-equivalence test --- c/src/metta.rs | 13 +++++++++++++ lib/src/metta/runner/mod.rs | 6 ++++++ python/hyperon/runner.py | 4 ++++ python/hyperonpy.cpp | 15 ++++++++------- python/tests/test_extend.py | 2 ++ 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/c/src/metta.rs b/c/src/metta.rs index b78871a58..eee54b7ed 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -806,6 +806,19 @@ pub extern "C" fn metta_free(metta: metta_t) { drop(metta); } +/// @brief Compares two `metta_t` handles to test whether the referenced MeTTa runner is the same +/// @ingroup interpreter_group +/// @param[in] a A pointer to the first Interpreter handle +/// @param[in] b A pointer to the first Interpreter handle +/// @return True if the two handles reference the same runner, otherwise False +/// +#[no_mangle] +pub extern "C" fn metta_eq(a: *const metta_t, b: *const metta_t) -> bool { + let a = unsafe{ &*a }.borrow(); + let b = unsafe{ &*b }.borrow(); + *a == *b +} + /// @brief Returns the number of module search paths that will be searched when importing modules into /// the runner /// @ingroup interpreter_group diff --git a/lib/src/metta/runner/mod.rs b/lib/src/metta/runner/mod.rs index d5a0d01da..72b414d99 100644 --- a/lib/src/metta/runner/mod.rs +++ b/lib/src/metta/runner/mod.rs @@ -35,6 +35,12 @@ const EXEC_SYMBOL : Atom = sym!("!"); #[derive(Clone, Debug)] pub struct Metta(Rc); +impl PartialEq for Metta { + fn eq(&self, other: &Self) -> bool { + Rc::ptr_eq(&self.0, &other.0) + } +} + #[derive(Debug)] pub struct MettaContents { space: DynSpace, diff --git a/python/hyperon/runner.py b/python/hyperon/runner.py index 3ba062ac2..7e62d24cd 100644 --- a/python/hyperon/runner.py +++ b/python/hyperon/runner.py @@ -62,6 +62,10 @@ def __init__(self, cmetta = None, space = None, env_builder = None): def __del__(self): hp.metta_free(self.cmetta) + def __eq__(self, other): + """Checks if two MeTTa runner handles point to the same runner.""" + return (hp.metta_eq(self.cmetta, other.cmetta)) + def _priv_load_metta_py_stdlib(self): self.load_py_module("hyperon.stdlib") self.register_atom('extend-py!', diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index 9459f874f..e14ad9294 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -792,24 +792,25 @@ PYBIND11_MODULE(hyperonpy, m) { return CMetta(metta_new_with_space_environment_and_stdlib(space.ptr(), env_builder.obj, &run_python_loader_callback, NULL)); }, "New MeTTa interpreter instance"); m.def("metta_free", [](CMetta metta) { metta_free(metta.obj); }, "Free MeTTa interpreter"); - m.def("metta_search_path_cnt", [](CMetta metta) { return metta_search_path_cnt(metta.ptr()); }, "Returns the number of module search paths in the runner's environment"); - m.def("metta_nth_search_path", [](CMetta metta, size_t idx) { + m.def("metta_eq", [](CMetta& a, CMetta& b) { return metta_eq(a.ptr(), b.ptr()); }, "Compares two MeTTa handles"); + m.def("metta_search_path_cnt", [](CMetta& metta) { return metta_search_path_cnt(metta.ptr()); }, "Returns the number of module search paths in the runner's environment"); + m.def("metta_nth_search_path", [](CMetta& metta, size_t idx) { return func_to_string_two_args((write_to_buf_two_arg_func_t)&metta_nth_search_path, metta.ptr(), (void*)idx); }, "Returns the module search path at the specified index, in the runner's environment"); - m.def("metta_space", [](CMetta metta) { return CSpace(metta_space(metta.ptr())); }, "Get space of MeTTa interpreter"); - m.def("metta_tokenizer", [](CMetta metta) { return CTokenizer(metta_tokenizer(metta.ptr())); }, "Get tokenizer of MeTTa interpreter"); - m.def("metta_run", [](CMetta metta, CSExprParser& parser) { + m.def("metta_space", [](CMetta& metta) { return CSpace(metta_space(metta.ptr())); }, "Get space of MeTTa interpreter"); + m.def("metta_tokenizer", [](CMetta& metta) { return CTokenizer(metta_tokenizer(metta.ptr())); }, "Get tokenizer of MeTTa interpreter"); + m.def("metta_run", [](CMetta& metta, CSExprParser& parser) { py::list lists_of_atom; sexpr_parser_t cloned_parser = sexpr_parser_clone(&parser.parser); metta_run(metta.ptr(), cloned_parser, copy_lists_of_atom, &lists_of_atom); return lists_of_atom; }, "Run MeTTa interpreter on an input"); - m.def("metta_evaluate_atom", [](CMetta metta, CAtom atom) { + m.def("metta_evaluate_atom", [](CMetta& metta, CAtom atom) { py::list atoms; metta_evaluate_atom(metta.ptr(), atom_clone(atom.ptr()), copy_atoms, &atoms); return atoms; }, "Run MeTTa interpreter on an atom"); - m.def("metta_load_module", [](CMetta metta, std::string text) { + m.def("metta_load_module", [](CMetta& metta, std::string text) { metta_load_module(metta.ptr(), text.c_str()); }, "Load MeTTa module"); diff --git a/python/tests/test_extend.py b/python/tests/test_extend.py index d9df51b62..162b6e2ac 100644 --- a/python/tests/test_extend.py +++ b/python/tests/test_extend.py @@ -18,6 +18,8 @@ def test_extend(self): [[], [ValueAtom(5)], [ValueAtom('B')]]) + self.assertEqual( + metta.run('! &runner')[0][0].get_object().value, metta) class ExtendGlobalTest(unittest.TestCase): From d5253d706fc592943151425fb09492622fcf4701 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Tue, 17 Oct 2023 14:38:45 -0700 Subject: [PATCH 25/28] Re-Adding side-band error accessor to parser, so parsing parse errors doesn't appear to be the same as encountering parse errors --- c/src/metta.rs | 56 ++++++++++++++++++++++++++++------ python/hyperon/base.py | 11 ++++--- python/hyperonpy.cpp | 6 ++++ python/tests/test_sexparser.py | 2 ++ 4 files changed, 62 insertions(+), 13 deletions(-) diff --git a/c/src/metta.rs b/c/src/metta.rs index eee54b7ed..30934a163 100644 --- a/c/src/metta.rs +++ b/c/src/metta.rs @@ -1,5 +1,4 @@ use hyperon::common::shared::Shared; -use hyperon::metta::error_atom; use hyperon::space::DynSpace; use hyperon::metta::text::*; use hyperon::metta::interpreter; @@ -150,19 +149,34 @@ pub extern "C" fn tokenizer_clone(tokenizer: *const tokenizer_t) -> tokenizer_t #[repr(C)] pub struct sexpr_parser_t { /// Internal. Should not be accessed directly - parser: *mut RustSExprParser + parser: *mut RustSExprParser, + err_string: *mut c_char, +} + +impl sexpr_parser_t { + fn free_err_string(&mut self) { + if !self.err_string.is_null() { + let string = unsafe{ std::ffi::CString::from_raw(self.err_string) }; + drop(string); + self.err_string = core::ptr::null_mut(); + } + } } struct RustSExprParser(SExprParser<'static>); impl From> for sexpr_parser_t { fn from(parser: SExprParser<'static>) -> Self { - Self{parser: Box::into_raw(Box::new(RustSExprParser(parser)))} + Self{ + parser: Box::into_raw(Box::new(RustSExprParser(parser))), + err_string: core::ptr::null_mut(), + } } } impl sexpr_parser_t { - fn into_inner(self) -> SExprParser<'static> { + fn into_inner(mut self) -> SExprParser<'static> { + self.free_err_string(); unsafe{ (*Box::from_raw(self.parser)).0 } } fn borrow(&self) -> &SExprParser<'static> { @@ -225,14 +239,36 @@ pub extern "C" fn sexpr_parser_parse( parser: *mut sexpr_parser_t, tokenizer: *const tokenizer_t) -> atom_t { - let parser = unsafe{ &mut *parser }.borrow_mut(); + let parser = unsafe{ &mut *parser }; + parser.free_err_string(); + let rust_parser = parser.borrow_mut(); let tokenizer = unsafe{ &*tokenizer }.borrow_inner(); - match parser.parse(tokenizer) { + match rust_parser.parse(tokenizer) { Ok(atom) => atom.into(), - Err(err) => error_atom(None, None, err).into() + Err(err) => { + let err_cstring = std::ffi::CString::new(err).unwrap(); + parser.err_string = err_cstring.into_raw(); + atom_t::null() + } } } +/// @brief Returns the error string associated with the last `sexpr_parser_parse` call +/// @ingroup tokenizer_and_parser_group +/// @param[in] parser A pointer to the Parser, which is associated with the text to parse +/// @return A pointer to the C-string containing the parse error that occurred, or NULL if no +/// parse error occurred +/// @warning The returned pointer should NOT be freed. It must never be accessed after the +/// sexpr_parser_t has been freed, or any subsequent `sexpr_parser_parse` or +/// `sexpr_parser_parse_to_syntax_tree` has been made. +/// +#[no_mangle] +pub extern "C" fn sexpr_parser_err_str( + parser: *mut sexpr_parser_t) -> *const c_char { + let parser = unsafe{ &*parser }; + parser.err_string +} + /// @brief Represents a component in a syntax tree created by parsing MeTTa code /// @ingroup tokenizer_and_parser_group /// @note `syntax_node_t` objects must be freed with `syntax_node_free()` @@ -337,8 +373,10 @@ pub type c_syntax_node_callback_t = extern "C" fn(node: *const syntax_node_t, co #[no_mangle] pub extern "C" fn sexpr_parser_parse_to_syntax_tree(parser: *mut sexpr_parser_t) -> syntax_node_t { - let parser = unsafe{ &mut *parser }.borrow_mut(); - parser.parse_to_syntax_tree().into() + let parser = unsafe{ &mut *parser }; + parser.free_err_string(); + let rust_parser = parser.borrow_mut(); + rust_parser.parse_to_syntax_tree().into() } /// @brief Frees a syntax_node_t diff --git a/python/hyperon/base.py b/python/hyperon/base.py index c46d79db6..3461c39c6 100644 --- a/python/hyperon/base.py +++ b/python/hyperon/base.py @@ -379,10 +379,13 @@ def parse(self, tokenizer): """ catom = self.cparser.parse(tokenizer.ctokenizer) if (catom is None): - return None - if (hp.atom_is_error(catom)): - raise SyntaxError(hp.atom_error_message(catom)) - return Atom._from_catom(catom) + err_str = self.cparser.sexpr_parser_err_str() + if (err_str is None): + return None + else: + raise SyntaxError(err_str) + else: + return Atom._from_catom(catom) def parse_to_syntax_tree(self): """ diff --git a/python/hyperonpy.cpp b/python/hyperonpy.cpp index e14ad9294..5466496b0 100644 --- a/python/hyperonpy.cpp +++ b/python/hyperonpy.cpp @@ -466,6 +466,11 @@ struct CSExprParser { return !atom_is_null(&atom) ? py::cast(CAtom(atom)) : py::none(); } + py::object err_str() { + const char* err_str = sexpr_parser_err_str(&this->parser); + return err_str != NULL ? py::cast(std::string(err_str)) : py::none(); + } + py::object parse_to_syntax_tree() { syntax_node_t root_node = sexpr_parser_parse_to_syntax_tree(&this->parser); return !syntax_node_is_null(&root_node) ? py::cast(CSyntaxNode(root_node)) : py::none(); @@ -738,6 +743,7 @@ PYBIND11_MODULE(hyperonpy, m) { py::class_(m, "CSExprParser") .def(py::init()) .def("parse", &CSExprParser::parse, "Return next parsed atom, None, or an error expression") + .def("sexpr_parser_err_str", &CSExprParser::err_str, "Return the parse error from the previous parse operation or None") .def("parse_to_syntax_tree", &CSExprParser::parse_to_syntax_tree, "Return next parser atom or None, as a syntax node at the root of a syntax tree"); py::class_(m, "CStepResult") diff --git a/python/tests/test_sexparser.py b/python/tests/test_sexparser.py index 4d8cad319..996deedf5 100644 --- a/python/tests/test_sexparser.py +++ b/python/tests/test_sexparser.py @@ -30,12 +30,14 @@ def testParseErr(self): parser = SExprParser("(+ one \"one") try: parsed_atom = parser.parse(tokenizer) + self.assertTrue(False, "Parse error expected") except SyntaxError as e: self.assertEqual(e.args[0], 'Unclosed String Literal') parser = SExprParser("(+ one \"one\"") try: parsed_atom = parser.parse(tokenizer) + self.assertTrue(False, "Parse error expected") except SyntaxError as e: self.assertEqual(e.args[0], 'Unexpected end of expression') From d649ba19eb4c74865f4dfbb6037a358fcdd6dde6 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Wed, 18 Oct 2023 09:17:24 -0700 Subject: [PATCH 26/28] Fixing benchmarks and tests to compile with all the API changes --- lib/benches/interpreter2.rs | 4 ++- lib/benches/states.rs | 52 +++++++++++++++---------------- python/tests/test_custom_space.py | 2 +- 3 files changed, 30 insertions(+), 28 deletions(-) diff --git a/lib/benches/interpreter2.rs b/lib/benches/interpreter2.rs index 140a84138..3730d7a23 100644 --- a/lib/benches/interpreter2.rs +++ b/lib/benches/interpreter2.rs @@ -1,5 +1,6 @@ #![feature(test)] - +#[cfg(feature = "minimal")] +mod interpreter2_bench { extern crate test; use test::Bencher; @@ -27,3 +28,4 @@ fn chain_x100(bencher: &mut Bencher) { assert_eq!(res, expected); }) } +} \ No newline at end of file diff --git a/lib/benches/states.rs b/lib/benches/states.rs index 63266fb37..ce4bc9d58 100644 --- a/lib/benches/states.rs +++ b/lib/benches/states.rs @@ -20,7 +20,7 @@ keep track of such possible changes. */ fn metta_state(size: isize) -> Metta { - let mut metta = new_metta_rust(); + let metta = Metta::new(None); let program = " ! (bind! &data (new-space)) (= (new-entry! $key $value) @@ -28,24 +28,24 @@ fn metta_state(size: isize) -> Metta { (add-atom &data (= (get-data $key) $new-state)) )) "; - metta.run(&mut SExprParser::new(program)); + metta.run(SExprParser::new(program)).unwrap(); for i in (0..size).step_by(1) { - metta.run(&mut SExprParser::new((format!("! (new-entry! k-{:X} v-{:X})", i, i)).as_str())); + metta.run(SExprParser::new((format!("! (new-entry! k-{:X} v-{:X})", i, i)).as_str())).unwrap(); } metta } fn metta_atoms(size: isize) -> Metta { - let mut metta = new_metta_rust(); + let metta = Metta::new(None); let program = " ! (bind! &data (new-space)) (= (new-entry! $key $value) (add-atom &data (= (get-data $key) $value) )) "; - metta.run(&mut SExprParser::new(program)); + metta.run(SExprParser::new(program)).unwrap(); for i in (0..size).step_by(1) { - metta.run(&mut SExprParser::new((format!("! (new-entry! k-{:X} v-{:X})", i, i)).as_str())); + metta.run(SExprParser::new((format!("! (new-entry! k-{:X} v-{:X})", i, i)).as_str())).unwrap(); } metta } @@ -57,7 +57,7 @@ fn query_state_x10(bencher: &mut Bencher) { bencher.iter(|| { let i = size / 2; // Retrieval of states by value - let res = metta.run(&mut SExprParser::new( + let res = metta.run(SExprParser::new( (format!("! (let $v (new-state v-{:X}) (match &data (= (get-data $x) $v) $x))", i)).as_str() )); @@ -72,7 +72,7 @@ fn query_state_x50(bencher: &mut Bencher) { bencher.iter(|| { let i = size / 2; // It is expected to be dependent on the space size - let res = metta.run(&mut SExprParser::new( + let res = metta.run(SExprParser::new( (format!("! (let $v (new-state v-{:X}) (match &data (= (get-data $x) $v) $x))", i)).as_str() )); @@ -87,11 +87,11 @@ fn change_state_x10(bencher: &mut Bencher) { // States are changed using on keys, so value-based retrieval is not involved bencher.iter(|| { let i = size / 2; - metta.run(&mut SExprParser::new( + metta.run(SExprParser::new( (format!("! (change-state! (match &data (= (get-data k-{:X}) $x) $x) v-{:X})", i, i*2)).as_str() - )); + )).unwrap(); }); - let res = metta.run(&mut SExprParser::new( + let res = metta.run(SExprParser::new( (format!("! (get-state (match &data (= (get-data k-{:X}) $x) $x))", size / 2)).as_str() )); assert_eq!(res, Ok(vec![vec![Atom::sym(format!("v-{:X}", size))]])) @@ -104,11 +104,11 @@ fn change_state_x50(bencher: &mut Bencher) { bencher.iter(|| { let i = size / 2; // Changing states is expected to be almost independent on the space size - metta.run(&mut SExprParser::new( + metta.run(SExprParser::new( (format!("! (change-state! (match &data (= (get-data k-{:X}) $x) $x) v-{:X})", i, i*2)).as_str() - )); + )).unwrap(); }); - let res = metta.run(&mut SExprParser::new( + let res = metta.run(SExprParser::new( (format!("! (get-state (match &data (= (get-data k-{:X}) $x) $x))", size / 2)).as_str() )); assert_eq!(res, Ok(vec![vec![Atom::sym(format!("v-{:X}", size))]])) @@ -121,7 +121,7 @@ fn query_atom_x10(bencher: &mut Bencher) { bencher.iter(|| { let i = size / 2; // Querying atoms by value should be as fast as by key - let res = metta.run(&mut SExprParser::new( + let res = metta.run(SExprParser::new( (format!("!(match &data (= (get-data $x) v-{:X}) $x)", i)).as_str() )); assert_eq!(res, Ok(vec![vec![Atom::sym(format!("k-{:X}", i))]])) @@ -135,7 +135,7 @@ fn query_atom_x50(bencher: &mut Bencher) { bencher.iter(|| { let i = size / 2; // It should be almost independent on the space size - let res = metta.run(&mut SExprParser::new( + let res = metta.run(SExprParser::new( (format!("!(match &data (= (get-data $x) v-{:X}) $x)", i)).as_str() )); assert_eq!(res, Ok(vec![vec![Atom::sym(format!("k-{:X}", i))]])) @@ -149,15 +149,15 @@ fn change_atom_x10(bencher: &mut Bencher) { bencher.iter(|| { let i = size / 2; // Replacing atoms has some overheads - metta.run(&mut SExprParser::new( + metta.run(SExprParser::new( (format!("! (match &data (= (get-data k-{:X}) $x) (remove-atom &data (= (get-data k-{:X}) $x)))", i, i)).as_str() - )); - metta.run(&mut SExprParser::new( + )).unwrap(); + metta.run(SExprParser::new( (format!("! (new-entry! k-{:X} v-{:X})", i, i*2)).as_str() - )); + )).unwrap(); }); - let res = metta.run(&mut SExprParser::new( + let res = metta.run(SExprParser::new( (format!("! (match &data (= (get-data k-{:X}) $x) $x)", size / 2)).as_str() )); assert_eq!(res, Ok(vec![vec![Atom::sym(format!("v-{:X}", size))]])) @@ -171,15 +171,15 @@ fn change_atom_x50(bencher: &mut Bencher) { let i = size / 2; // It may be not too dependent on the space size, but maybe // some shuffling of atoms under replacement is needed - metta.run(&mut SExprParser::new( + metta.run(SExprParser::new( (format!("! (match &data (= (get-data k-{:X}) $x) (remove-atom &data (= (get-data k-{:X}) $x)))", i, i)).as_str() - )); - metta.run(&mut SExprParser::new( + )).unwrap(); + metta.run(SExprParser::new( (format!("! (new-entry! k-{:X} v-{:X})", i, i*2)).as_str() - )); + )).unwrap(); }); - let res = metta.run(&mut SExprParser::new( + let res = metta.run(SExprParser::new( (format!("! (match &data (= (get-data k-{:X}) $x) $x)", size / 2)).as_str() )); assert_eq!(res, Ok(vec![vec![Atom::sym(format!("v-{:X}", size))]])) diff --git a/python/tests/test_custom_space.py b/python/tests/test_custom_space.py index 85cf2e132..cef927c05 100644 --- a/python/tests/test_custom_space.py +++ b/python/tests/test_custom_space.py @@ -145,7 +145,7 @@ def test_runner_with_custom_space(self): space = SpaceRef(test_space) space.add_atom(E(S("="), S("key"), S("val"))) - metta = MeTTa(space=space) + metta = MeTTa(space=space, env_builder=Environment.test_env()) self.assertEqual(metta.space().get_payload().test_attrib, "Test Space Payload Attrib") self.assertEqualMettaRunnerResults(metta.run("!(match &self (= key $val) $val)"), From fe6c15d3571995dc098875c0a7668c34351dee30 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Wed, 18 Oct 2023 09:47:38 -0700 Subject: [PATCH 27/28] one last fix so repl's "no_python" feature uses new API --- repl/src/metta_shim.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repl/src/metta_shim.rs b/repl/src/metta_shim.rs index a790a70c3..0d499ca28 100644 --- a/repl/src/metta_shim.rs +++ b/repl/src/metta_shim.rs @@ -327,7 +327,7 @@ pub mod metta_interface_mod { use hyperon::metta::text::SExprParser; use hyperon::ExpressionAtom; use hyperon::Atom; - use hyperon::metta::runner::{Metta, RunnerState, atom_is_error, Environment, EnvBuilder}; + use hyperon::metta::runner::{Metta, RunnerState, Environment, EnvBuilder}; use super::{strip_quotes, exec_state_prepare, exec_state_should_break}; pub use hyperon::metta::text::SyntaxNodeType as SyntaxNodeType; From f31df5618b40defaf81b856de2fe1c2075930564 Mon Sep 17 00:00:00 2001 From: Luke Peterson Date: Thu, 19 Oct 2023 09:29:51 -0600 Subject: [PATCH 28/28] GitHub.com editor messed up indentation in last merge. Fixing. --- python/hyperon/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/hyperon/runner.py b/python/hyperon/runner.py index e698d7162..664063a26 100644 --- a/python/hyperon/runner.py +++ b/python/hyperon/runner.py @@ -49,10 +49,10 @@ def current_results(self, flat=False): class MeTTa: """This class contains the MeTTa program execution utilities""" - def __init__(self, space = None, env_builder = None, cmetta = None): + def __init__(self, cmetta = None, space = None, env_builder = None): self.pymods = {} - if cmetta is not None: + if cmetta is not None: self.cmetta = cmetta else: if space is None: