From b4c347750aff0ea07c2a982df887a4f70535cdc6 Mon Sep 17 00:00:00 2001 From: Akash Date: Wed, 15 Jan 2025 23:45:49 +0530 Subject: [PATCH 01/11] feat: add sourcemap option to configuration and command line arguments Signed-off-by: Akash --- kclvm/api/src/service/into.rs | 1 + kclvm/cmd/src/lib.rs | 3 ++- kclvm/config/src/settings.rs | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/kclvm/api/src/service/into.rs b/kclvm/api/src/service/into.rs index fb179a43a..42c9c6951 100644 --- a/kclvm/api/src/service/into.rs +++ b/kclvm/api/src/service/into.rs @@ -49,6 +49,7 @@ impl IntoLoadSettingsFiles for SettingsFile { show_hidden: config.show_hidden.unwrap_or_default(), fast_eval: config.fast_eval.unwrap_or_default(), include_schema_type_path: config.include_schema_type_path.unwrap_or_default(), + sourcemap: config.sourcemap.unwrap_or_default(), }), kcl_options: match self.kcl_options { Some(opts) => opts diff --git a/kclvm/cmd/src/lib.rs b/kclvm/cmd/src/lib.rs index e85ade250..4b2223a1d 100644 --- a/kclvm/cmd/src/lib.rs +++ b/kclvm/cmd/src/lib.rs @@ -57,7 +57,8 @@ pub fn app() -> Command { .arg(arg!(overrides: -O --overrides ... "Specify the configuration override path and value").num_args(1..)) .arg(arg!(target: --target "Specify the target type")) .arg(arg!(recursive: -R --recursive "Compile the files directory recursively")) - .arg(arg!(package_map: -E --external ... "Mapping of package name and path where the package is located").num_args(1..)), + .arg(arg!(package_map: -E --external ... "Mapping of package name and path where the package is located").num_args(1..)) + .arg(arg!(sourcemap: --sourcemap "Generate a sourcemap")), ) .subcommand(Command::new("server").about("Start a rpc server for APIs")) .subcommand(Command::new("version").about("Show the KCL version")) diff --git a/kclvm/config/src/settings.rs b/kclvm/config/src/settings.rs index 30108e965..c2074850e 100644 --- a/kclvm/config/src/settings.rs +++ b/kclvm/config/src/settings.rs @@ -68,6 +68,7 @@ pub struct Config { pub package_maps: Option>, /// Use the evaluator to execute the AST program instead of AOT. pub fast_eval: Option, + pub sourcemap: Option, } impl SettingsFile { @@ -88,6 +89,7 @@ impl SettingsFile { fast_eval: Some(false), include_schema_type_path: Some(false), package_maps: Some(HashMap::default()), + sourcemap: Some(false), }), kcl_options: Some(vec![]), } From 170e25052ebde92f071e25e50f971633258d19ea Mon Sep 17 00:00:00 2001 From: Akash Date: Wed, 15 Jan 2025 23:57:19 +0530 Subject: [PATCH 02/11] feat: add sourcemap option to ExecProgramArgs and merge_settings Signed-off-by: Akash --- kclvm/config/src/settings.rs | 2 ++ kclvm/runner/src/runner.rs | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/kclvm/config/src/settings.rs b/kclvm/config/src/settings.rs index c2074850e..bfb85fe19 100644 --- a/kclvm/config/src/settings.rs +++ b/kclvm/config/src/settings.rs @@ -1,5 +1,6 @@ //! Copyright The KCL Authors. All rights reserved. use anyhow::{Context, Result}; +use indexmap::set; use serde::{ de::{DeserializeSeed, Error, MapAccess, SeqAccess, Unexpected, Visitor}, Deserialize, Serialize, @@ -390,6 +391,7 @@ pub fn merge_settings(settings: &[SettingsFile]) -> SettingsFile { set_if!(result_kcl_cli_configs, sort_keys, kcl_cli_configs); set_if!(result_kcl_cli_configs, show_hidden, kcl_cli_configs); set_if!(result_kcl_cli_configs, fast_eval, kcl_cli_configs); + set_if!(result_kcl_cli_configs, sourcemap, kcl_cli_configs); set_if!( result_kcl_cli_configs, include_schema_type_path, diff --git a/kclvm/runner/src/runner.rs b/kclvm/runner/src/runner.rs index b22825e77..74167c2ed 100644 --- a/kclvm/runner/src/runner.rs +++ b/kclvm/runner/src/runner.rs @@ -72,6 +72,9 @@ pub struct ExecProgramArgs { /// the result without any form of compilation. #[serde(skip)] pub fast_eval: bool, + /// sourcemap denotes whether to generate a source map. + /// This is used for debugging purposes. + pub sourcemap: bool, } impl ExecProgramArgs { @@ -187,6 +190,7 @@ impl TryFrom for ExecProgramArgs { args.sort_keys = cli_configs.sort_keys.unwrap_or_default(); args.show_hidden = cli_configs.show_hidden.unwrap_or_default(); args.fast_eval = cli_configs.fast_eval.unwrap_or_default(); + args.sourcemap = cli_configs.sourcemap.unwrap_or_default(); args.include_schema_type_path = cli_configs.include_schema_type_path.unwrap_or_default(); for override_str in cli_configs.overrides.unwrap_or_default() { From 0f2e2b89347cb9294fbd65cff8ea319d4eb24afc Mon Sep 17 00:00:00 2001 From: Akash Date: Thu, 16 Jan 2025 21:31:19 +0530 Subject: [PATCH 03/11] feat: update sourcemap option to accept string values in settings and command line arguments Signed-off-by: Akash --- kclvm/cmd/src/lib.rs | 2 +- kclvm/cmd/src/run.rs | 1 + kclvm/cmd/src/settings.rs | 1 + kclvm/config/src/settings.rs | 14 ++++++++++++-- kclvm/runner/src/runner.rs | 3 ++- 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/kclvm/cmd/src/lib.rs b/kclvm/cmd/src/lib.rs index 4b2223a1d..e31899c8f 100644 --- a/kclvm/cmd/src/lib.rs +++ b/kclvm/cmd/src/lib.rs @@ -58,7 +58,7 @@ pub fn app() -> Command { .arg(arg!(target: --target "Specify the target type")) .arg(arg!(recursive: -R --recursive "Compile the files directory recursively")) .arg(arg!(package_map: -E --external ... "Mapping of package name and path where the package is located").num_args(1..)) - .arg(arg!(sourcemap: --sourcemap "Generate a sourcemap")), + .arg(arg!(sourcemap: --sourcemap "Generate a sourcemap")), ) .subcommand(Command::new("server").about("Start a rpc server for APIs")) .subcommand(Command::new("version").about("Show the KCL version")) diff --git a/kclvm/cmd/src/run.rs b/kclvm/cmd/src/run.rs index 9ab1fd469..985ce829a 100644 --- a/kclvm/cmd/src/run.rs +++ b/kclvm/cmd/src/run.rs @@ -13,6 +13,7 @@ pub fn run_command(matches: &ArgMatches, writer: &mut W) -> Result<()> // Config settings building let settings = must_build_settings(matches); let output = settings.output(); + let sourcemap = settings.sourcemap(); let sess = Arc::new(ParseSession::default()); match exec_program(sess.clone(), &settings.try_into()?) { Ok(result) => { diff --git a/kclvm/cmd/src/settings.rs b/kclvm/cmd/src/settings.rs index cbdd6aa73..09639d9cf 100644 --- a/kclvm/cmd/src/settings.rs +++ b/kclvm/cmd/src/settings.rs @@ -55,6 +55,7 @@ pub(crate) fn build_settings(matches: &ArgMatches) -> Result { show_hidden: bool_from_matches(matches, "show_hidden"), fast_eval: bool_from_matches(matches, "fast_eval"), package_maps, + sourcemap : matches.get_one::("output").map(|v| v.to_string()), ..Default::default() }), kcl_options: if arguments.is_some() { diff --git a/kclvm/config/src/settings.rs b/kclvm/config/src/settings.rs index bfb85fe19..34900b4e4 100644 --- a/kclvm/config/src/settings.rs +++ b/kclvm/config/src/settings.rs @@ -1,6 +1,7 @@ //! Copyright The KCL Authors. All rights reserved. use anyhow::{Context, Result}; use indexmap::set; +use kclvm_ast::token::LitKind::Str; use serde::{ de::{DeserializeSeed, Error, MapAccess, SeqAccess, Unexpected, Visitor}, Deserialize, Serialize, @@ -30,6 +31,15 @@ impl SettingsPathBuf { } } + /// Get the SourceMap setting. + #[inline] + pub fn sourcemap(&self) -> Option { + match &self.1.kcl_cli_configs { + Some(c) => c.sourcemap.clone(), + None => None, + } + } + /// Get the path. #[inline] pub fn path(&self) -> &Option { @@ -69,7 +79,7 @@ pub struct Config { pub package_maps: Option>, /// Use the evaluator to execute the AST program instead of AOT. pub fast_eval: Option, - pub sourcemap: Option, + pub sourcemap: Option, } impl SettingsFile { @@ -90,7 +100,7 @@ impl SettingsFile { fast_eval: Some(false), include_schema_type_path: Some(false), package_maps: Some(HashMap::default()), - sourcemap: Some(false), + sourcemap: None, }), kcl_options: Some(vec![]), } diff --git a/kclvm/runner/src/runner.rs b/kclvm/runner/src/runner.rs index 74167c2ed..8aafe67da 100644 --- a/kclvm/runner/src/runner.rs +++ b/kclvm/runner/src/runner.rs @@ -1,4 +1,5 @@ use anyhow::{anyhow, Result}; +use kclvm_ast::token::LitKind::Str; use kclvm_evaluator::Evaluator; use std::collections::HashMap; use std::{cell::RefCell, rc::Rc}; @@ -74,7 +75,7 @@ pub struct ExecProgramArgs { pub fast_eval: bool, /// sourcemap denotes whether to generate a source map. /// This is used for debugging purposes. - pub sourcemap: bool, + pub sourcemap: String, } impl ExecProgramArgs { From 4f76c3834a352138c385dd35182025d556a5f4f2 Mon Sep 17 00:00:00 2001 From: Akash Date: Thu, 16 Jan 2025 21:59:41 +0530 Subject: [PATCH 04/11] feat: implement sourcemap support in FastRunner and add mapping functionality Signed-off-by: Akash --- kclvm/cmd/src/run.rs | 4 ++ kclvm/runner/src/runner.rs | 79 +++++++++++++++++++++++++++++++------- 2 files changed, 69 insertions(+), 14 deletions(-) diff --git a/kclvm/cmd/src/run.rs b/kclvm/cmd/src/run.rs index 985ce829a..44d0a3061 100644 --- a/kclvm/cmd/src/run.rs +++ b/kclvm/cmd/src/run.rs @@ -35,6 +35,10 @@ pub fn run_command(matches: &ArgMatches, writer: &mut W) -> Result<()> // using [`writeln`] can be better to redirect the output. None => writeln!(writer, "{}", result.yaml_result)?, } + match sourcemap { + Some(s) => std::fs::write(s, result.sourcemap_result)?, + None => {} + } } } // Other error message diff --git a/kclvm/runner/src/runner.rs b/kclvm/runner/src/runner.rs index 8aafe67da..231827f6d 100644 --- a/kclvm/runner/src/runner.rs +++ b/kclvm/runner/src/runner.rs @@ -33,6 +33,37 @@ pub type kclvm_context_t = std::ffi::c_void; #[allow(non_camel_case_types)] pub type kclvm_value_ref_t = std::ffi::c_void; +#[derive(Debug, Serialize, Deserialize)] +pub struct KCLSourceMap { + version: u8, + sources: Vec, + mappings: HashMap>, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Mapping { + generated_line: u32, + generated_column: u32, + original_line: u32, + original_column: u32, + source_index: usize, +} + +impl KCLSourceMap { + pub fn new() -> Self { + Self { + version: 1, + sources: Vec::new(), + mappings: HashMap::new(), + } + } + + pub fn add_mapping(&mut self, source: String, mapping: Mapping) { + self.mappings.entry(source).or_default().push(mapping); + } +} + + /// ExecProgramArgs denotes the configuration required to execute the KCL program. #[derive(Serialize, Deserialize, Debug, Default, Clone)] pub struct ExecProgramArgs { @@ -107,6 +138,7 @@ pub struct ExecProgramResult { pub yaml_result: String, pub log_message: String, pub err_message: String, + pub sourcemap_result: String, } pub trait MapErrorResult { @@ -496,6 +528,7 @@ static ONCE_PANIC_HOOK: Lazy<()> = Lazy::new(|| { pub struct FastRunner { opts: RunnerOptions, + sourcemap: Option, } impl FastRunner { @@ -503,11 +536,12 @@ impl FastRunner { pub fn new(opts: Option) -> Self { Self { opts: opts.unwrap_or_default(), + sourcemap: None, } } /// Run kcl library with exec arguments. - pub fn run(&self, program: &ast::Program, args: &ExecProgramArgs) -> Result { + pub fn run(&mut self, program: &ast::Program, args: &ExecProgramArgs) -> Result { let ctx = Rc::new(RefCell::new(args_to_ctx(program, args))); let evaluator = Evaluator::new_with_runtime_ctx(program, ctx.clone()); #[cfg(target_arch = "wasm32")] @@ -537,20 +571,37 @@ impl FastRunner { } }) })); - let evaluator_result = std::panic::catch_unwind(|| { - if self.opts.plugin_agent_ptr > 0 { - #[cfg(not(target_arch = "wasm32"))] - unsafe { - let plugin_method: extern "C" fn( - method: *const c_char, - args: *const c_char, - kwargs: *const c_char, - ) -> *const c_char = std::mem::transmute(self.opts.plugin_agent_ptr); - kclvm_plugin_init(plugin_method); - } + + // Before evaluation, initialize sourcemap if enabled + if args.sourcemap == "true" { + self.sourcemap = Some(KCLSourceMap::new()); + } + + // During evaluation, track locations + let evaluator_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let result = evaluator.run(); + + // If sourcemap enabled, collect mappings + if let Some(sourcemap) = &mut self.sourcemap { + // Get source location from result + let source_loc = result.get_source_location(); + + // Create mapping + let mapping = Mapping { + generated_line: result.yaml_line, + generated_column: result.yaml_column, + original_line: source_loc.line, + original_column: source_loc.column, + source_index: source_loc.file_index, + }; + + // Add to sourcemap + sourcemap.add_mapping(source_loc.filename, mapping); } - evaluator.run() - }); + + result + })); + #[cfg(not(target_arch = "wasm32"))] std::panic::set_hook(prev_hook); KCL_RUNTIME_PANIC_RECORD.with(|record| { From 9b68de96cd41c62c523167e9f9fb77f8d0e92020 Mon Sep 17 00:00:00 2001 From: Akash Date: Thu, 16 Jan 2025 22:41:40 +0530 Subject: [PATCH 05/11] feat: add KCLSourceMap struct and integrate into Evaluator for source mapping support Signed-off-by: Akash --- kclvm/evaluator/src/lib.rs | 8 +++-- kclvm/evaluator/src/node.rs | 32 +++++++++++++++++ kclvm/runner/src/runner.rs | 70 +++++++++---------------------------- 3 files changed, 55 insertions(+), 55 deletions(-) diff --git a/kclvm/evaluator/src/lib.rs b/kclvm/evaluator/src/lib.rs index 983f02615..dcfa55e9a 100644 --- a/kclvm/evaluator/src/lib.rs +++ b/kclvm/evaluator/src/lib.rs @@ -27,6 +27,7 @@ use generational_arena::{Arena, Index}; use indexmap::IndexMap; use kclvm_runtime::val_plan::KCL_PRIVATE_VAR_PREFIX; use lazy::{BacktrackMeta, LazyEvalScope}; +use node::KCLSourceMap; use proxy::{Frame, Proxy}; use rule::RuleEvalContextRef; use schema::SchemaEvalContextRef; @@ -88,7 +89,9 @@ pub struct Evaluator<'ctx> { /// Schema attr backtrack meta. pub backtrack_meta: RefCell>, /// Current AST id for the evaluator walker. - pub ast_id: RefCell, + pub ast_id: RefCell, + // Source map for code generated + pub source_map: std::option::Option, } #[derive(Clone)] @@ -147,11 +150,12 @@ impl<'ctx> Evaluator<'ctx> { local_vars: RefCell::new(Default::default()), backtrack_meta: RefCell::new(Default::default()), ast_id: RefCell::new(AstIndex::default()), + source_map: Some(KCLSourceMap::new()), } } /// Evaluate the program and return the JSON and YAML result. - pub fn run(self: &Evaluator<'ctx>) -> Result<(String, String)> { + pub fn run(self: &Evaluator<'ctx> ) -> Result<(String, String)> { let modules = self.program.get_modules_for_pkg(kclvm_ast::MAIN_PKG); self.init_scope(kclvm_ast::MAIN_PKG); self.compile_ast_modules(&modules); diff --git a/kclvm/evaluator/src/node.rs b/kclvm/evaluator/src/node.rs index 3490e5c62..2559826ba 100644 --- a/kclvm/evaluator/src/node.rs +++ b/kclvm/evaluator/src/node.rs @@ -6,6 +6,8 @@ use std::sync::{Arc, RwLock}; use anyhow::Ok; use generational_arena::Index; +// use serde::{Serialize, Deserialize}; +use crate::HashMap; use kclvm_ast::ast::{self, CallExpr, ConfigEntry, Module, NodeRef}; use kclvm_ast::walker::TypedResultWalker; use kclvm_runtime::{ @@ -28,6 +30,36 @@ use crate::{backtrack_break_here, backtrack_update_break}; use crate::{error as kcl_error, GLOBAL_LEVEL, INNER_LEVEL}; use crate::{EvalResult, Evaluator}; + +pub struct KCLSourceMap { + version: u8, + sources: Vec, + mappings: HashMap>, +} + +pub struct Mapping { + generated_line: u32, + generated_column: u32, + original_line: u32, + original_column: u32, + source_index: usize, +} + +impl KCLSourceMap { + pub fn new() -> Self { + Self { + version: 1, + sources: Vec::new(), + mappings: HashMap::new(), + } + } + + pub fn add_mapping(&mut self, source: String, mapping: Mapping) { + self.mappings.entry(source).or_default().push(mapping); + } +} + + /// Impl TypedResultWalker for Evaluator to visit AST nodes to evaluate the result. impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> { type Result = EvalResult; diff --git a/kclvm/runner/src/runner.rs b/kclvm/runner/src/runner.rs index 231827f6d..8fce5b65b 100644 --- a/kclvm/runner/src/runner.rs +++ b/kclvm/runner/src/runner.rs @@ -33,36 +33,6 @@ pub type kclvm_context_t = std::ffi::c_void; #[allow(non_camel_case_types)] pub type kclvm_value_ref_t = std::ffi::c_void; -#[derive(Debug, Serialize, Deserialize)] -pub struct KCLSourceMap { - version: u8, - sources: Vec, - mappings: HashMap>, -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct Mapping { - generated_line: u32, - generated_column: u32, - original_line: u32, - original_column: u32, - source_index: usize, -} - -impl KCLSourceMap { - pub fn new() -> Self { - Self { - version: 1, - sources: Vec::new(), - mappings: HashMap::new(), - } - } - - pub fn add_mapping(&mut self, source: String, mapping: Mapping) { - self.mappings.entry(source).or_default().push(mapping); - } -} - /// ExecProgramArgs denotes the configuration required to execute the KCL program. #[derive(Serialize, Deserialize, Debug, Default, Clone)] @@ -528,7 +498,6 @@ static ONCE_PANIC_HOOK: Lazy<()> = Lazy::new(|| { pub struct FastRunner { opts: RunnerOptions, - sourcemap: Option, } impl FastRunner { @@ -536,7 +505,6 @@ impl FastRunner { pub fn new(opts: Option) -> Self { Self { opts: opts.unwrap_or_default(), - sourcemap: None, } } @@ -571,33 +539,29 @@ impl FastRunner { } }) })); - - // Before evaluation, initialize sourcemap if enabled - if args.sourcemap == "true" { - self.sourcemap = Some(KCLSourceMap::new()); - } // During evaluation, track locations let evaluator_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + //boolean value to check if the sourcemap is enabled let result = evaluator.run(); - // If sourcemap enabled, collect mappings - if let Some(sourcemap) = &mut self.sourcemap { - // Get source location from result - let source_loc = result.get_source_location(); + // // If sourcemap enabled, collect mappings + // if let Some(sourcemap) = &mut self.sourcemap { + // // Get source location from result + // // let source_loc = result.get_source_location(); - // Create mapping - let mapping = Mapping { - generated_line: result.yaml_line, - generated_column: result.yaml_column, - original_line: source_loc.line, - original_column: source_loc.column, - source_index: source_loc.file_index, - }; - - // Add to sourcemap - sourcemap.add_mapping(source_loc.filename, mapping); - } + // // // Create mapping + // // let mapping = Mapping { + // // generated_line: result.yaml_line, + // // generated_column: result.yaml_column, + // // original_line: source_loc.line, + // // original_column: source_loc.column, + // // source_index: source_loc.file_index, + // // }; + + // // Add to sourcemap + // sourcemap.add_mapping(source_loc.filename, mapping); + // } result })); From c223a495437e0ebf2920d31d1c63aecada41558c Mon Sep 17 00:00:00 2001 From: Akash Date: Fri, 17 Jan 2025 22:39:54 +0530 Subject: [PATCH 06/11] feat: add current source position tracking to Evaluator and update sourcemap generation Signed-off-by: Akash --- kclvm/evaluator/src/lib.rs | 23 ++++++++++++++++++++++- kclvm/evaluator/src/node.rs | 30 ++++++++++++++++++++++-------- 2 files changed, 44 insertions(+), 9 deletions(-) diff --git a/kclvm/evaluator/src/lib.rs b/kclvm/evaluator/src/lib.rs index dcfa55e9a..917d179cf 100644 --- a/kclvm/evaluator/src/lib.rs +++ b/kclvm/evaluator/src/lib.rs @@ -27,7 +27,7 @@ use generational_arena::{Arena, Index}; use indexmap::IndexMap; use kclvm_runtime::val_plan::KCL_PRIVATE_VAR_PREFIX; use lazy::{BacktrackMeta, LazyEvalScope}; -use node::KCLSourceMap; +use node::{KCLSourceMap , SourcePos}; use proxy::{Frame, Proxy}; use rule::RuleEvalContextRef; use schema::SchemaEvalContextRef; @@ -92,6 +92,8 @@ pub struct Evaluator<'ctx> { pub ast_id: RefCell, // Source map for code generated pub source_map: std::option::Option, + // Current source position + pub current_source_pos: RefCell>, } #[derive(Clone)] @@ -151,11 +153,30 @@ impl<'ctx> Evaluator<'ctx> { backtrack_meta: RefCell::new(Default::default()), ast_id: RefCell::new(AstIndex::default()), source_map: Some(KCLSourceMap::new()), + current_source_pos: RefCell::new(None), } } /// Evaluate the program and return the JSON and YAML result. pub fn run(self: &Evaluator<'ctx> ) -> Result<(String, String)> { + //ab yaha pe kuch circus karna h +// The modules variable represents the AST modules from the main package of the KCL program. It's retrieved via self.program.get_modules_for_pkg(kclvm_ast::MAIN_PKG). + +// Looking at the code structure: + +// Type: Vec + +// Contains AST nodes for each KCL source file in the main package +// Source: kclvm_ast::ast::Program + +// Each module represents a single KCL source file +// Contains declarations, statements, and other AST nodes +// Usage: + +// Passed to compile_ast_modules() for compilation +// Contains source locations needed for sourcemap generation +// Each module has file path, line numbers, and column information +// This is where we need to start tracking source locations for the sourcemap, as these modules contain the original source position information. let modules = self.program.get_modules_for_pkg(kclvm_ast::MAIN_PKG); self.init_scope(kclvm_ast::MAIN_PKG); self.compile_ast_modules(&modules); diff --git a/kclvm/evaluator/src/node.rs b/kclvm/evaluator/src/node.rs index 2559826ba..f0d8d88ba 100644 --- a/kclvm/evaluator/src/node.rs +++ b/kclvm/evaluator/src/node.rs @@ -29,14 +29,17 @@ use crate::union::union_entry; use crate::{backtrack_break_here, backtrack_update_break}; use crate::{error as kcl_error, GLOBAL_LEVEL, INNER_LEVEL}; use crate::{EvalResult, Evaluator}; - - pub struct KCLSourceMap { version: u8, sources: Vec, mappings: HashMap>, } +pub struct SourcePos { + line: u32, + column: u32, +} + pub struct Mapping { generated_line: u32, generated_column: u32, @@ -69,9 +72,12 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> { */ fn walk_stmt(&self, stmt: &'ctx ast::Node) -> Self::Result { - backtrack_break_here!(self, stmt); - self.update_ctx_panic_info(stmt); - self.update_ast_id(stmt); + self.current_source_pos = Some(SourcePos { + line: stmt.pos().1 as u32, + column: stmt.pos().2 as u32, + }).into(); + + let yaml_start_line = self.get_current_yaml_line(); let value = match &stmt.node { ast::Stmt::TypeAlias(type_alias) => self.walk_type_alias_stmt(type_alias), ast::Stmt::Expr(expr_stmt) => self.walk_expr_stmt(expr_stmt), @@ -87,7 +93,15 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> { ast::Stmt::Schema(schema_stmt) => self.walk_schema_stmt(schema_stmt), ast::Stmt::Rule(rule_stmt) => self.walk_rule_stmt(rule_stmt), }; - backtrack_update_break!(self, stmt); + + // Store mapping after YAML generation + if let Some(pos) = *self.current_source_pos.borrow() { + if let Some(ref mut sm) = self.source_map { + let vec = sm.mappings.entry(stmt.filename.clone()).or_insert_with(Vec::new); + vec.push((pos.line, yaml_start_line)); + } + } + value } @@ -1274,7 +1288,7 @@ impl<'ctx> Evaluator<'ctx> { } } else { // If variable exists in the scope and update it, if not, add it to the scope. - if !self.store_variable_in_current_scope(name, value.clone()) { + if (!self.store_variable_in_current_scope(name, value.clone())) { self.add_variable(name, self.undefined_value()); self.store_variable(name, value); } @@ -1357,7 +1371,7 @@ impl<'ctx> Evaluator<'ctx> { } } else { // If variable exists in the scope and update it, if not, add it to the scope. - if !self.store_variable_in_current_scope(name, value.clone()) { + if (!self.store_variable_in_current_scope(name, value.clone())) { self.add_variable(name, self.undefined_value()); self.store_variable(name, value); } From 5b20f833e87337e7e43a3e43ec1da5932a3ced05 Mon Sep 17 00:00:00 2001 From: Akash Date: Sat, 18 Jan 2025 15:35:22 +0530 Subject: [PATCH 07/11] feat: enhance Evaluator with YAML line tracking and refactor source position handling Signed-off-by: Akash --- kclvm/evaluator/src/lib.rs | 9 ++++-- kclvm/evaluator/src/node.rs | 58 ++++++++++++++++++++++++------------- 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/kclvm/evaluator/src/lib.rs b/kclvm/evaluator/src/lib.rs index 917d179cf..7ba390206 100644 --- a/kclvm/evaluator/src/lib.rs +++ b/kclvm/evaluator/src/lib.rs @@ -27,7 +27,7 @@ use generational_arena::{Arena, Index}; use indexmap::IndexMap; use kclvm_runtime::val_plan::KCL_PRIVATE_VAR_PREFIX; use lazy::{BacktrackMeta, LazyEvalScope}; -use node::{KCLSourceMap , SourcePos}; +use node::KCLSourceMap; use proxy::{Frame, Proxy}; use rule::RuleEvalContextRef; use schema::SchemaEvalContextRef; @@ -93,7 +93,9 @@ pub struct Evaluator<'ctx> { // Source map for code generated pub source_map: std::option::Option, // Current source position - pub current_source_pos: RefCell>, + pub current_source_pos: RefCell, + //Current YAML line + pub yaml_line_counter: RefCell, } #[derive(Clone)] @@ -153,7 +155,8 @@ impl<'ctx> Evaluator<'ctx> { backtrack_meta: RefCell::new(Default::default()), ast_id: RefCell::new(AstIndex::default()), source_map: Some(KCLSourceMap::new()), - current_source_pos: RefCell::new(None), + current_source_pos: RefCell::new(0), + yaml_line_counter: RefCell::new(0), } } diff --git a/kclvm/evaluator/src/node.rs b/kclvm/evaluator/src/node.rs index f0d8d88ba..e476db249 100644 --- a/kclvm/evaluator/src/node.rs +++ b/kclvm/evaluator/src/node.rs @@ -6,7 +6,6 @@ use std::sync::{Arc, RwLock}; use anyhow::Ok; use generational_arena::Index; -// use serde::{Serialize, Deserialize}; use crate::HashMap; use kclvm_ast::ast::{self, CallExpr, ConfigEntry, Module, NodeRef}; use kclvm_ast::walker::TypedResultWalker; @@ -29,23 +28,18 @@ use crate::union::union_entry; use crate::{backtrack_break_here, backtrack_update_break}; use crate::{error as kcl_error, GLOBAL_LEVEL, INNER_LEVEL}; use crate::{EvalResult, Evaluator}; + +#[derive(Clone)] pub struct KCLSourceMap { version: u8, sources: Vec, mappings: HashMap>, } -pub struct SourcePos { - line: u32, - column: u32, -} - +#[derive(Clone)] pub struct Mapping { generated_line: u32, - generated_column: u32, original_line: u32, - original_column: u32, - source_index: usize, } impl KCLSourceMap { @@ -72,11 +66,7 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> { */ fn walk_stmt(&self, stmt: &'ctx ast::Node) -> Self::Result { - self.current_source_pos = Some(SourcePos { - line: stmt.pos().1 as u32, - column: stmt.pos().2 as u32, - }).into(); - + let current_source_pos = self.get_current_source_position(); let yaml_start_line = self.get_current_yaml_line(); let value = match &stmt.node { ast::Stmt::TypeAlias(type_alias) => self.walk_type_alias_stmt(type_alias), @@ -95,13 +85,18 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> { }; // Store mapping after YAML generation - if let Some(pos) = *self.current_source_pos.borrow() { - if let Some(ref mut sm) = self.source_map { - let vec = sm.mappings.entry(stmt.filename.clone()).or_insert_with(Vec::new); - vec.push((pos.line, yaml_start_line)); - } + if let Some(source_map) = self.get_source_map() { + let current_source_pos = current_source_pos.borrow(); + let yaml_end_line = self.get_current_yaml_line(); + let mapping = Mapping { + generated_line: yaml_end_line, + original_line: yaml_start_line, + }; + source_map.borrow_mut().add_mapping( + current_source_pos.get_filename().to_string(), + mapping, + ); } - value } @@ -1175,6 +1170,29 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> { } impl<'ctx> Evaluator<'ctx> { + + fn get_source_map(&self) -> Option { + self.source_map.clone() + } + + fn get_current_source_position(&self) -> u32 { + *self.current_source_pos.borrow() + } + + fn get_current_yaml_line(&self) -> u32 { + *self.yaml_line_counter.borrow() + } + + // Increment line counter during YAML generation + fn increment_yaml_line(&self) { + *self.yaml_line_counter.borrow_mut() += 1; + } + + // Reset counter when starting new evaluation + fn reset_yaml_line(&self) { + *self.yaml_line_counter.borrow_mut() = 0; + } + pub fn walk_stmts_except_import(&self, stmts: &'ctx [Box>]) -> EvalResult { let mut result = self.ok_result(); for stmt in stmts { From a4e03028e19da88b6a9f226c6d0121070e542c01 Mon Sep 17 00:00:00 2001 From: Akash Date: Sat, 18 Jan 2025 15:38:58 +0530 Subject: [PATCH 08/11] refactor: simplify KCLSourceMap structure and update mapping logic Signed-off-by: Akash --- kclvm/evaluator/src/node.rs | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/kclvm/evaluator/src/node.rs b/kclvm/evaluator/src/node.rs index e476db249..9f1a70eb5 100644 --- a/kclvm/evaluator/src/node.rs +++ b/kclvm/evaluator/src/node.rs @@ -31,8 +31,8 @@ use crate::{EvalResult, Evaluator}; #[derive(Clone)] pub struct KCLSourceMap { - version: u8, - sources: Vec, + // version: u8, + // sources: Vec, mappings: HashMap>, } @@ -45,14 +45,19 @@ pub struct Mapping { impl KCLSourceMap { pub fn new() -> Self { Self { - version: 1, - sources: Vec::new(), + // version: 1, + // sources: Vec::new(), mappings: HashMap::new(), } } - pub fn add_mapping(&mut self, source: String, mapping: Mapping) { - self.mappings.entry(source).or_default().push(mapping); + pub fn add_mapping(&mut self, mapping: Mapping) { + let key = mapping.original_line.to_string(); + if let Some(mappings) = self.mappings.get_mut(&key) { + mappings.push(mapping); + } else { + self.mappings.insert(key, vec![mapping]); + } } } @@ -85,17 +90,14 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> { }; // Store mapping after YAML generation - if let Some(source_map) = self.get_source_map() { - let current_source_pos = current_source_pos.borrow(); + if let Some(mut source_map) = self.get_source_map() { + // let current_source_pos = current_source_pos.borrow(); let yaml_end_line = self.get_current_yaml_line(); let mapping = Mapping { generated_line: yaml_end_line, original_line: yaml_start_line, }; - source_map.borrow_mut().add_mapping( - current_source_pos.get_filename().to_string(), - mapping, - ); + source_map.add_mapping(mapping); } value } From d792538056dc03fabee77734492fc52b4ce99013 Mon Sep 17 00:00:00 2001 From: Akash Date: Sat, 18 Jan 2025 15:57:47 +0530 Subject: [PATCH 09/11] fix: finished source position handling in walk_stmt Signed-off-by: Akash --- kclvm/evaluator/src/node.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kclvm/evaluator/src/node.rs b/kclvm/evaluator/src/node.rs index 9f1a70eb5..4db1d3a7c 100644 --- a/kclvm/evaluator/src/node.rs +++ b/kclvm/evaluator/src/node.rs @@ -71,7 +71,8 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> { */ fn walk_stmt(&self, stmt: &'ctx ast::Node) -> Self::Result { - let current_source_pos = self.get_current_source_position(); + // let current_source_pos = self.get_current_source_position(); + *self.current_source_pos.borrow_mut() = (stmt.pos().1 as u32).into(); let yaml_start_line = self.get_current_yaml_line(); let value = match &stmt.node { ast::Stmt::TypeAlias(type_alias) => self.walk_type_alias_stmt(type_alias), @@ -92,10 +93,9 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> { // Store mapping after YAML generation if let Some(mut source_map) = self.get_source_map() { // let current_source_pos = current_source_pos.borrow(); - let yaml_end_line = self.get_current_yaml_line(); let mapping = Mapping { - generated_line: yaml_end_line, - original_line: yaml_start_line, + original_line: self.current_source_pos.borrow().clone(), + generated_line: yaml_start_line, }; source_map.add_mapping(mapping); } From fdf93c99ca00825b00614aaf1edbb37aab6ad997 Mon Sep 17 00:00:00 2001 From: Akash Date: Sat, 18 Jan 2025 17:27:49 +0530 Subject: [PATCH 10/11] feat: enhance KCLSourceMap and Evaluator with improved mapping logic and YAML line counting Signed-off-by: Akash --- kclvm/evaluator/src/node.rs | 26 +++++++++++--------------- kclvm/runtime/src/value/val_yaml.rs | 19 +++++++++++++++++++ 2 files changed, 30 insertions(+), 15 deletions(-) diff --git a/kclvm/evaluator/src/node.rs b/kclvm/evaluator/src/node.rs index 4db1d3a7c..2efb31371 100644 --- a/kclvm/evaluator/src/node.rs +++ b/kclvm/evaluator/src/node.rs @@ -31,33 +31,28 @@ use crate::{EvalResult, Evaluator}; #[derive(Clone)] pub struct KCLSourceMap { - // version: u8, - // sources: Vec, + version: u8, + sources: Vec, mappings: HashMap>, } #[derive(Clone)] pub struct Mapping { - generated_line: u32, + generated_line: (u32, u32), original_line: u32, } impl KCLSourceMap { pub fn new() -> Self { Self { - // version: 1, - // sources: Vec::new(), + version: 1, + sources: Vec::new(), mappings: HashMap::new(), } } - pub fn add_mapping(&mut self, mapping: Mapping) { - let key = mapping.original_line.to_string(); - if let Some(mappings) = self.mappings.get_mut(&key) { - mappings.push(mapping); - } else { - self.mappings.insert(key, vec![mapping]); - } + pub fn add_mapping(&mut self, source: String, mapping: Mapping) { + self.mappings.entry(source).or_default().push(mapping); } } @@ -73,7 +68,7 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> { fn walk_stmt(&self, stmt: &'ctx ast::Node) -> Self::Result { // let current_source_pos = self.get_current_source_position(); *self.current_source_pos.borrow_mut() = (stmt.pos().1 as u32).into(); - let yaml_start_line = self.get_current_yaml_line(); + let yaml_start_line = ValueRef::get_yaml_line_count(); let value = match &stmt.node { ast::Stmt::TypeAlias(type_alias) => self.walk_type_alias_stmt(type_alias), ast::Stmt::Expr(expr_stmt) => self.walk_expr_stmt(expr_stmt), @@ -89,15 +84,16 @@ impl<'ctx> TypedResultWalker<'ctx> for Evaluator<'ctx> { ast::Stmt::Schema(schema_stmt) => self.walk_schema_stmt(schema_stmt), ast::Stmt::Rule(rule_stmt) => self.walk_rule_stmt(rule_stmt), }; + let yaml_end = ValueRef::get_yaml_line_count(); // Store mapping after YAML generation if let Some(mut source_map) = self.get_source_map() { // let current_source_pos = current_source_pos.borrow(); let mapping = Mapping { original_line: self.current_source_pos.borrow().clone(), - generated_line: yaml_start_line, + generated_line: (yaml_start_line , yaml_end), }; - source_map.add_mapping(mapping); + source_map.add_mapping( stmt.filename.clone() , mapping); } value } diff --git a/kclvm/runtime/src/value/val_yaml.rs b/kclvm/runtime/src/value/val_yaml.rs index 2b2d8e80d..d51a0cdcd 100644 --- a/kclvm/runtime/src/value/val_yaml.rs +++ b/kclvm/runtime/src/value/val_yaml.rs @@ -6,6 +6,7 @@ extern crate serde_yaml; use crate::*; use serde::{Deserialize, Serialize}; +use std::cell::RefCell; /// YAML encode options. /// - sort_keys: Sort the encode result by keys (defaults to false). @@ -51,6 +52,10 @@ impl ValueRef { Ok(Self::from_json(ctx, serde_json::to_string(&json_value).unwrap().as_ref()).unwrap()) } + pub fn get_yaml_line_count() -> u32 { + YAML_LINE_COUNTER.with(|counter| *counter.borrow()) + } + /// Decode yaml stream string that contains `---` to a ValueRef. /// Returns [serde_yaml::Error] when decoding fails. pub fn from_yaml_stream(ctx: &mut Context, s: &str) -> Result { @@ -97,6 +102,11 @@ impl ValueRef { match serde_yaml::to_string(&yaml_value) { Ok(s) => { let s = s.strip_prefix("---\n").unwrap_or_else(|| s.as_ref()); + // Count newlines in generated YAML + let line_count = s.chars().filter(|&c| c == '\n').count(); + YAML_LINE_COUNTER.with(|counter| { + *counter.borrow_mut() += line_count as u32; + }); s.to_string() } Err(err) => panic!("{}", err), @@ -117,6 +127,11 @@ impl ValueRef { match serde_yaml::to_string(&yaml_value) { Ok(s) => { let s = s.strip_prefix("---\n").unwrap_or_else(|| s.as_ref()); + // Count newlines in generated YAML with options + let line_count = s.chars().filter(|&c| c == '\n').count(); + YAML_LINE_COUNTER.with(|counter| { + *counter.borrow_mut() += line_count as u32; + }); s.to_string() } Err(err) => panic!("{}", err), @@ -124,6 +139,10 @@ impl ValueRef { } } +thread_local! { + static YAML_LINE_COUNTER: RefCell = RefCell::new(0); +} + #[cfg(test)] mod test_value_yaml { use crate::*; From fa4ba5174053f8e6a97300212952256efdadcbfc Mon Sep 17 00:00:00 2001 From: Akash Date: Sat, 18 Jan 2025 18:14:00 +0530 Subject: [PATCH 11/11] refactor: clean up unused functions and improve source map handling in Evaluator Signed-off-by: Akash --- kclvm/evaluator/src/lib.rs | 18 ----------------- kclvm/evaluator/src/node.rs | 22 ++------------------- kclvm/runner/src/runner.rs | 39 +++++++++++++++++-------------------- 3 files changed, 20 insertions(+), 59 deletions(-) diff --git a/kclvm/evaluator/src/lib.rs b/kclvm/evaluator/src/lib.rs index 7ba390206..31b3c8543 100644 --- a/kclvm/evaluator/src/lib.rs +++ b/kclvm/evaluator/src/lib.rs @@ -162,24 +162,6 @@ impl<'ctx> Evaluator<'ctx> { /// Evaluate the program and return the JSON and YAML result. pub fn run(self: &Evaluator<'ctx> ) -> Result<(String, String)> { - //ab yaha pe kuch circus karna h -// The modules variable represents the AST modules from the main package of the KCL program. It's retrieved via self.program.get_modules_for_pkg(kclvm_ast::MAIN_PKG). - -// Looking at the code structure: - -// Type: Vec - -// Contains AST nodes for each KCL source file in the main package -// Source: kclvm_ast::ast::Program - -// Each module represents a single KCL source file -// Contains declarations, statements, and other AST nodes -// Usage: - -// Passed to compile_ast_modules() for compilation -// Contains source locations needed for sourcemap generation -// Each module has file path, line numbers, and column information -// This is where we need to start tracking source locations for the sourcemap, as these modules contain the original source position information. let modules = self.program.get_modules_for_pkg(kclvm_ast::MAIN_PKG); self.init_scope(kclvm_ast::MAIN_PKG); self.compile_ast_modules(&modules); diff --git a/kclvm/evaluator/src/node.rs b/kclvm/evaluator/src/node.rs index 2efb31371..a2642f7f2 100644 --- a/kclvm/evaluator/src/node.rs +++ b/kclvm/evaluator/src/node.rs @@ -28,15 +28,14 @@ use crate::union::union_entry; use crate::{backtrack_break_here, backtrack_update_break}; use crate::{error as kcl_error, GLOBAL_LEVEL, INNER_LEVEL}; use crate::{EvalResult, Evaluator}; - -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct KCLSourceMap { version: u8, sources: Vec, mappings: HashMap>, } -#[derive(Clone)] +#[derive(Clone , Debug)] pub struct Mapping { generated_line: (u32, u32), original_line: u32, @@ -1173,23 +1172,6 @@ impl<'ctx> Evaluator<'ctx> { self.source_map.clone() } - fn get_current_source_position(&self) -> u32 { - *self.current_source_pos.borrow() - } - - fn get_current_yaml_line(&self) -> u32 { - *self.yaml_line_counter.borrow() - } - - // Increment line counter during YAML generation - fn increment_yaml_line(&self) { - *self.yaml_line_counter.borrow_mut() += 1; - } - - // Reset counter when starting new evaluation - fn reset_yaml_line(&self) { - *self.yaml_line_counter.borrow_mut() = 0; - } pub fn walk_stmts_except_import(&self, stmts: &'ctx [Box>]) -> EvalResult { let mut result = self.ok_result(); diff --git a/kclvm/runner/src/runner.rs b/kclvm/runner/src/runner.rs index 8fce5b65b..bf6cae183 100644 --- a/kclvm/runner/src/runner.rs +++ b/kclvm/runner/src/runner.rs @@ -1,6 +1,7 @@ use anyhow::{anyhow, Result}; use kclvm_ast::token::LitKind::Str; use kclvm_evaluator::Evaluator; +use kclvm_sema::eval; use std::collections::HashMap; use std::{cell::RefCell, rc::Rc}; @@ -543,27 +544,20 @@ impl FastRunner { // During evaluation, track locations let evaluator_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { //boolean value to check if the sourcemap is enabled + let sourcemap_enabled = !args.sourcemap.is_empty(); let result = evaluator.run(); - - // // If sourcemap enabled, collect mappings - // if let Some(sourcemap) = &mut self.sourcemap { - // // Get source location from result - // // let source_loc = result.get_source_location(); - - // // // Create mapping - // // let mapping = Mapping { - // // generated_line: result.yaml_line, - // // generated_column: result.yaml_column, - // // original_line: source_loc.line, - // // original_column: source_loc.column, - // // source_index: source_loc.file_index, - // // }; - - // // Add to sourcemap - // sourcemap.add_mapping(source_loc.filename, mapping); - // } - - result + let sourcemap = evaluator.source_map; + if sourcemap_enabled { + result.map(|r| { + let (json, yaml) = r; + (json, yaml, sourcemap) + }) + } else { + result.map(|r| { + let (json, yaml) = r; + (json, yaml, None) + }) + } })); #[cfg(not(target_arch = "wasm32"))] @@ -579,9 +573,12 @@ impl FastRunner { let is_err = evaluator_result.is_err(); match evaluator_result { Ok(r) => match r { - Ok((json, yaml)) => { + Ok((json, yaml , sourcemap)) => { result.json_result = json; result.yaml_result = yaml; + if let Some(sourcemap) = sourcemap { + result.sourcemap_result = format!("{:?}", sourcemap); + } } Err(err) => { result.err_message = err.to_string();