feat : Add the ability to generate sourcemaps from CLI

kclvm/api/src/service/into.rs

@@ -49,6 +49,7 @@ impl IntoLoadSettingsFiles for SettingsFile {
                 show_hidden: config.show_hidden.unwrap_or_default(),
                 fast_eval: config.fast_eval.unwrap_or_default(),
                 include_schema_type_path: config.include_schema_type_path.unwrap_or_default(),
+                sourcemap: config.sourcemap.unwrap_or_default(),
             }),
             kcl_options: match self.kcl_options {
                 Some(opts) => opts

kclvm/cmd/src/lib.rs

@@ -57,7 +57,8 @@ pub fn app() -> Command {
             .arg(arg!(overrides: -O --overrides <overrides> ... "Specify the configuration override path and value").num_args(1..))
             .arg(arg!(target: --target <target> "Specify the target type"))
             .arg(arg!(recursive: -R --recursive "Compile the files directory recursively"))
-            .arg(arg!(package_map: -E --external <package_map> ... "Mapping of package name and path where the package is located").num_args(1..)),
+            .arg(arg!(package_map: -E --external <package_map> ... "Mapping of package name and path where the package is located").num_args(1..))
+            .arg(arg!(sourcemap: --sourcemap <sourcemap> "Generate a sourcemap")),
         )
     .subcommand(Command::new("server").about("Start a rpc server for APIs"))
     .subcommand(Command::new("version").about("Show the KCL version"))

kclvm/cmd/src/run.rs

@@ -13,6 +13,7 @@ pub fn run_command<W: Write>(matches: &ArgMatches, writer: &mut W) -> Result<()>
     // Config settings building
     let settings = must_build_settings(matches);
     let output = settings.output();
+    let sourcemap = settings.sourcemap();
     let sess = Arc::new(ParseSession::default());
     match exec_program(sess.clone(), &settings.try_into()?) {
         Ok(result) => {
@@ -34,6 +35,10 @@ pub fn run_command<W: Write>(matches: &ArgMatches, writer: &mut W) -> Result<()>
                     // using [`writeln`] can be better to redirect the output.
                     None => writeln!(writer, "{}", result.yaml_result)?,
                 }
+                match sourcemap {
+                    Some(s) => std::fs::write(s, result.sourcemap_result)?,
+                    None => {}
+                }
             }
         }
         // Other error message

kclvm/cmd/src/settings.rs

@@ -55,6 +55,7 @@ pub(crate) fn build_settings(matches: &ArgMatches) -> Result<SettingsPathBuf> {
                 show_hidden: bool_from_matches(matches, "show_hidden"),
                 fast_eval: bool_from_matches(matches, "fast_eval"),
                 package_maps,
+                sourcemap : matches.get_one::<String>("output").map(|v| v.to_string()),
                 ..Default::default()
             }),
             kcl_options: if arguments.is_some() {

kclvm/config/src/settings.rs

@@ -1,5 +1,7 @@
 //! Copyright The KCL Authors. All rights reserved.
 use anyhow::{Context, Result};
+use indexmap::set;
+use kclvm_ast::token::LitKind::Str;
 use serde::{
     de::{DeserializeSeed, Error, MapAccess, SeqAccess, Unexpected, Visitor},
     Deserialize, Serialize,
@@ -29,6 +31,15 @@ impl SettingsPathBuf {
         }
     }
 
+    /// Get the SourceMap setting.
+    #[inline]
+    pub fn sourcemap(&self) -> Option<String> {
+        match &self.1.kcl_cli_configs {
+            Some(c) => c.sourcemap.clone(),
+            None => None,
+        }
+    }
+
     /// Get the path.
     #[inline]
     pub fn path(&self) -> &Option<PathBuf> {
@@ -68,6 +79,7 @@ pub struct Config {
     pub package_maps: Option<HashMap<String, String>>,
     /// Use the evaluator to execute the AST program instead of AOT.
     pub fast_eval: Option<bool>,
+    pub sourcemap: Option<String>,
 }
 
 impl SettingsFile {
@@ -88,6 +100,7 @@ impl SettingsFile {
                 fast_eval: Some(false),
                 include_schema_type_path: Some(false),
                 package_maps: Some(HashMap::default()),
+                sourcemap: None,
             }),
             kcl_options: Some(vec![]),
         }
@@ -388,6 +401,7 @@ pub fn merge_settings(settings: &[SettingsFile]) -> SettingsFile {
                 set_if!(result_kcl_cli_configs, sort_keys, kcl_cli_configs);
                 set_if!(result_kcl_cli_configs, show_hidden, kcl_cli_configs);
                 set_if!(result_kcl_cli_configs, fast_eval, kcl_cli_configs);
+                set_if!(result_kcl_cli_configs, sourcemap, kcl_cli_configs);
                 set_if!(
                     result_kcl_cli_configs,
                     include_schema_type_path,

kclvm/evaluator/src/lib.rs

@@ -27,6 +27,7 @@ use generational_arena::{Arena, Index};
 use indexmap::IndexMap;
 use kclvm_runtime::val_plan::KCL_PRIVATE_VAR_PREFIX;
 use lazy::{BacktrackMeta, LazyEvalScope};
+use node::KCLSourceMap;
 use proxy::{Frame, Proxy};
 use rule::RuleEvalContextRef;
 use schema::SchemaEvalContextRef;
@@ -88,7 +89,13 @@ pub struct Evaluator<'ctx> {
     /// Schema attr backtrack meta.
     pub backtrack_meta: RefCell<Vec<BacktrackMeta>>,
     /// Current AST id for the evaluator walker.
-    pub ast_id: RefCell<AstIndex>,
+    pub ast_id: RefCell<AstIndex>, 
+    // Source map for code generated
+    pub source_map: std::option::Option<KCLSourceMap>,
+    // Current source position
+    pub current_source_pos: RefCell<u32>,
+    //Current YAML line
+    pub yaml_line_counter: RefCell<u32>,
 }
 
 #[derive(Clone)]
@@ -147,11 +154,14 @@ impl<'ctx> Evaluator<'ctx> {
             local_vars: RefCell::new(Default::default()),
             backtrack_meta: RefCell::new(Default::default()),
             ast_id: RefCell::new(AstIndex::default()),
+            source_map: Some(KCLSourceMap::new()),
+            current_source_pos: RefCell::new(0),
+            yaml_line_counter: RefCell::new(0),
         }
     }
 
     /// Evaluate the program and return the JSON and YAML result.
-    pub fn run(self: &Evaluator<'ctx>) -> Result<(String, String)> {
+    pub fn run(self: &Evaluator<'ctx> ) -> Result<(String, String)> {
         let modules = self.program.get_modules_for_pkg(kclvm_ast::MAIN_PKG);
         self.init_scope(kclvm_ast::MAIN_PKG);
         self.compile_ast_modules(&modules);

kclvm/evaluator/src/node.rs

@@ -6,6 +6,7 @@ use std::sync::{Arc, RwLock};
 
 use anyhow::Ok;
 use generational_arena::Index;
+use crate::HashMap;
 use kclvm_ast::ast::{self, CallExpr, ConfigEntry, Module, NodeRef};
 use kclvm_ast::walker::TypedResultWalker;
 use kclvm_runtime::{
@@ -27,6 +28,33 @@ use crate::union::union_entry;
 use crate::{backtrack_break_here, backtrack_update_break};
 use crate::{error as kcl_error, GLOBAL_LEVEL, INNER_LEVEL};
 use crate::{EvalResult, Evaluator};
+#[derive(Clone, Debug)]
+pub struct KCLSourceMap {
+    version: u8,
+    sources: Vec<String>,
+    mappings: HashMap<String, Vec<Mapping>>,
+}
+
+#[derive(Clone , Debug)]
+pub struct Mapping {
+    generated_line: (u32, u32),
+    original_line: u32,
+}
+
+impl KCLSourceMap {
+    pub fn new() -> Self {
+        Self {
+            version: 1,
+            sources: Vec::new(),
+            mappings: HashMap::new(),
+        }
+    }
+
+    pub fn add_mapping(&mut self, source: String, mapping: Mapping) {
+        self.mappings.entry(source).or_default().push(mapping);
+    }
+}
+
 
 /// Impl TypedResultWalker for Evaluator to visit AST nodes to evaluate the result.
 impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> {
@@ -37,9 +65,9 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> {
      */
 
     fn walk_stmt(&self, stmt: &'ctx ast::Node<ast::Stmt>) -> Self::Result {
-        backtrack_break_here!(self, stmt);
-        self.update_ctx_panic_info(stmt);
-        self.update_ast_id(stmt);
+        // let current_source_pos = self.get_current_source_position();
+        *self.current_source_pos.borrow_mut() = (stmt.pos().1 as u32).into();
+        let yaml_start_line = ValueRef::get_yaml_line_count();
         let value = match &stmt.node {
             ast::Stmt::TypeAlias(type_alias) => self.walk_type_alias_stmt(type_alias),
             ast::Stmt::Expr(expr_stmt) => self.walk_expr_stmt(expr_stmt),
@@ -55,7 +83,17 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> {
             ast::Stmt::Schema(schema_stmt) => self.walk_schema_stmt(schema_stmt),
             ast::Stmt::Rule(rule_stmt) => self.walk_rule_stmt(rule_stmt),
         };
-        backtrack_update_break!(self, stmt);
+        let yaml_end = ValueRef::get_yaml_line_count();
+
+        // Store mapping after YAML generation
+        if let Some(mut source_map) = self.get_source_map() {
+            // let current_source_pos = current_source_pos.borrow();
+            let mapping = Mapping {
+                original_line: self.current_source_pos.borrow().clone(),
+                generated_line: (yaml_start_line , yaml_end),
+            };
+            source_map.add_mapping( stmt.filename.clone() , mapping);
+        }
         value
     }
 
@@ -1129,6 +1167,12 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> {
 }
 
 impl<'ctx> Evaluator<'ctx> {
+
+    fn get_source_map(&self) -> Option<KCLSourceMap> {
+        self.source_map.clone()
+    }
+
+
     pub fn walk_stmts_except_import(&self, stmts: &'ctx [Box<ast::Node<ast::Stmt>>]) -> EvalResult {
         let mut result = self.ok_result();
         for stmt in stmts {
@@ -1242,7 +1286,7 @@ impl<'ctx> Evaluator<'ctx> {
                     }
                 } else {
                     // If variable exists in the scope and update it, if not, add it to the scope.
-                    if !self.store_variable_in_current_scope(name, value.clone()) {
+                    if (!self.store_variable_in_current_scope(name, value.clone())) {
                         self.add_variable(name, self.undefined_value());
                         self.store_variable(name, value);
                     }
@@ -1325,7 +1369,7 @@ impl<'ctx> Evaluator<'ctx> {
                             }
                         } else {
                             // If variable exists in the scope and update it, if not, add it to the scope.
-                            if !self.store_variable_in_current_scope(name, value.clone()) {
+                            if (!self.store_variable_in_current_scope(name, value.clone())) {
                                 self.add_variable(name, self.undefined_value());
                                 self.store_variable(name, value);
                             }

kclvm/runner/src/runner.rs

@@ -1,5 +1,7 @@
 use anyhow::{anyhow, Result};
+use kclvm_ast::token::LitKind::Str;
 use kclvm_evaluator::Evaluator;
+use kclvm_sema::eval;
 use std::collections::HashMap;
 use std::{cell::RefCell, rc::Rc};
 
@@ -32,6 +34,7 @@ pub type kclvm_context_t = std::ffi::c_void;
 #[allow(non_camel_case_types)]
 pub type kclvm_value_ref_t = std::ffi::c_void;
 
+
 /// ExecProgramArgs denotes the configuration required to execute the KCL program.
 #[derive(Serialize, Deserialize, Debug, Default, Clone)]
 pub struct ExecProgramArgs {
@@ -72,6 +75,9 @@ pub struct ExecProgramArgs {
     /// the result without any form of compilation.
     #[serde(skip)]
     pub fast_eval: bool,
+    /// sourcemap denotes whether to generate a source map.
+    /// This is used for debugging purposes.
+    pub sourcemap: String,
 }
 
 impl ExecProgramArgs {
@@ -103,6 +109,7 @@ pub struct ExecProgramResult {
     pub yaml_result: String,
     pub log_message: String,
     pub err_message: String,
+    pub sourcemap_result: String,
 }
 
 pub trait MapErrorResult {
@@ -187,6 +194,7 @@ impl TryFrom<SettingsFile> for ExecProgramArgs {
             args.sort_keys = cli_configs.sort_keys.unwrap_or_default();
             args.show_hidden = cli_configs.show_hidden.unwrap_or_default();
             args.fast_eval = cli_configs.fast_eval.unwrap_or_default();
+            args.sourcemap = cli_configs.sourcemap.unwrap_or_default();
             args.include_schema_type_path =
                 cli_configs.include_schema_type_path.unwrap_or_default();
             for override_str in cli_configs.overrides.unwrap_or_default() {
@@ -502,7 +510,7 @@ impl FastRunner {
     }
 
     /// Run kcl library with exec arguments.
-    pub fn run(&self, program: &ast::Program, args: &ExecProgramArgs) -> Result<ExecProgramResult> {
+    pub fn run(&mut self, program: &ast::Program, args: &ExecProgramArgs) -> Result<ExecProgramResult> {
         let ctx = Rc::new(RefCell::new(args_to_ctx(program, args)));
         let evaluator = Evaluator::new_with_runtime_ctx(program, ctx.clone());
         #[cfg(target_arch = "wasm32")]
@@ -532,20 +540,26 @@ impl FastRunner {
                 }
             })
         }));
-        let evaluator_result = std::panic::catch_unwind(|| {
-            if self.opts.plugin_agent_ptr > 0 {
-                #[cfg(not(target_arch = "wasm32"))]
-                unsafe {
-                    let plugin_method: extern "C" fn(
-                        method: *const c_char,
-                        args: *const c_char,
-                        kwargs: *const c_char,
-                    ) -> *const c_char = std::mem::transmute(self.opts.plugin_agent_ptr);
-                    kclvm_plugin_init(plugin_method);
-                }
+
+        // During evaluation, track locations
+        let evaluator_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+            //boolean value to check if the sourcemap is enabled
+            let sourcemap_enabled = !args.sourcemap.is_empty();
+            let result = evaluator.run();
+            let sourcemap = evaluator.source_map;
+            if sourcemap_enabled {
+            result.map(|r| {
+                let (json, yaml) = r;
+                (json, yaml, sourcemap)
+            })
+            } else {
+            result.map(|r| {
+                let (json, yaml) = r;
+                (json, yaml, None) 
+            })
             }
-            evaluator.run()
-        });
+        }));
+
         #[cfg(not(target_arch = "wasm32"))]
         std::panic::set_hook(prev_hook);
         KCL_RUNTIME_PANIC_RECORD.with(|record| {
@@ -559,9 +573,12 @@ impl FastRunner {
         let is_err = evaluator_result.is_err();
         match evaluator_result {
             Ok(r) => match r {
-                Ok((json, yaml)) => {
+                Ok((json, yaml , sourcemap)) => {
                     result.json_result = json;
                     result.yaml_result = yaml;
+                    if let Some(sourcemap) = sourcemap {
+                        result.sourcemap_result = format!("{:?}", sourcemap);
+                    }
                 }
                 Err(err) => {
                     result.err_message = err.to_string();

kclvm/runtime/src/value/val_yaml.rs

@@ -6,6 +6,7 @@ extern crate serde_yaml;
 use crate::*;
 
 use serde::{Deserialize, Serialize};
+use std::cell::RefCell;
 
 /// YAML encode options.
 /// - sort_keys: Sort the encode result by keys (defaults to false).
@@ -51,6 +52,10 @@ impl ValueRef {
         Ok(Self::from_json(ctx, serde_json::to_string(&json_value).unwrap().as_ref()).unwrap())
     }
 
+    pub fn get_yaml_line_count() -> u32 {
+        YAML_LINE_COUNTER.with(|counter| *counter.borrow())
+    }
+
     /// Decode yaml stream string that contains `---` to a ValueRef.
     /// Returns [serde_yaml::Error] when decoding fails.
     pub fn from_yaml_stream(ctx: &mut Context, s: &str) -> Result<Self, serde_yaml::Error> {
@@ -97,6 +102,11 @@ impl ValueRef {
         match serde_yaml::to_string(&yaml_value) {
             Ok(s) => {
                 let s = s.strip_prefix("---\n").unwrap_or_else(|| s.as_ref());
+                // Count newlines in generated YAML
+                let line_count = s.chars().filter(|&c| c == '\n').count();
+                YAML_LINE_COUNTER.with(|counter| {
+                    *counter.borrow_mut() += line_count as u32;
+                });
                 s.to_string()
             }
             Err(err) => panic!("{}", err),
@@ -117,13 +127,22 @@ impl ValueRef {
         match serde_yaml::to_string(&yaml_value) {
             Ok(s) => {
                 let s = s.strip_prefix("---\n").unwrap_or_else(|| s.as_ref());
+                // Count newlines in generated YAML with options
+                let line_count = s.chars().filter(|&c| c == '\n').count();
+                YAML_LINE_COUNTER.with(|counter| {
+                    *counter.borrow_mut() += line_count as u32;
+                });
                 s.to_string()
             }
             Err(err) => panic!("{}", err),
         }
     }
 }
 
+thread_local! {
+    static YAML_LINE_COUNTER: RefCell<u32> = RefCell::new(0);
+}
+
 #[cfg(test)]
 mod test_value_yaml {
     use crate::*;