feat: impl yaml stream decode/encode system module functions

Signed-off-by: peefy <[email protected]>

kclvm/runtime/src/_kclvm.h

@@ -652,10 +652,16 @@ kclvm_value_ref_t* kclvm_value_union_all(kclvm_context_t* ctx, kclvm_value_ref_t
 
 kclvm_value_ref_t* kclvm_yaml_decode(kclvm_context_t* ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* _kwargs);
 
+kclvm_value_ref_t* kclvm_yaml_decode_all(kclvm_context_t* ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* _kwargs);
+
+kclvm_value_ref_t* kclvm_yaml_dump_all_to_file(kclvm_context_t* _ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* kwargs);
+
 kclvm_value_ref_t* kclvm_yaml_dump_to_file(kclvm_context_t* _ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* kwargs);
 
 kclvm_value_ref_t* kclvm_yaml_encode(kclvm_context_t* ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* kwargs);
 
+kclvm_value_ref_t* kclvm_yaml_encode_all(kclvm_context_t* ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* kwargs);
+
 #ifdef __cplusplus
 } // extern "C"
 #endif

kclvm/runtime/src/_kclvm.ll

@@ -600,10 +600,16 @@ declare %kclvm_value_ref_t* @kclvm_value_union_all(%kclvm_context_t* %ctx, %kclv
 
 declare %kclvm_value_ref_t* @kclvm_yaml_decode(%kclvm_context_t* %ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %_kwargs);
 
+declare %kclvm_value_ref_t* @kclvm_yaml_decode_all(%kclvm_context_t* %ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %_kwargs);
+
+declare %kclvm_value_ref_t* @kclvm_yaml_dump_all_to_file(%kclvm_context_t* %_ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %kwargs);
+
 declare %kclvm_value_ref_t* @kclvm_yaml_dump_to_file(%kclvm_context_t* %_ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %kwargs);
 
 declare %kclvm_value_ref_t* @kclvm_yaml_encode(%kclvm_context_t* %ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %kwargs);
 
+declare %kclvm_value_ref_t* @kclvm_yaml_encode_all(%kclvm_context_t* %ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %kwargs);
+
 define void @__kcl_keep_link_runtime(%kclvm_value_ref_t* %_a, %kclvm_context_t* %_b) {
 	call %kclvm_value_ref_t* @kclvm_value_None(%kclvm_context_t* %_b)
 	ret void

kclvm/runtime/src/_kclvm.rs

@@ -312,8 +312,11 @@ pub enum ApiFunc {
     kclvm_value_union,
     kclvm_value_union_all,
     kclvm_yaml_decode,
+    kclvm_yaml_decode_all,
+    kclvm_yaml_dump_all_to_file,
     kclvm_yaml_dump_to_file,
     kclvm_yaml_encode,
+    kclvm_yaml_encode_all,
 }
 
 impl std::fmt::Display for ApiFunc {

kclvm/runtime/src/_kclvm_addr.rs

@@ -341,8 +341,11 @@ pub fn _kclvm_get_fn_ptr_by_name(name: &str) -> u64 {
         "kclvm_value_union" => crate::kclvm_value_union as *const () as u64,
         "kclvm_value_union_all" => crate::kclvm_value_union_all as *const () as u64,
         "kclvm_yaml_decode" => crate::kclvm_yaml_decode as *const () as u64,
+        "kclvm_yaml_decode_all" => crate::kclvm_yaml_decode_all as *const () as u64,
+        "kclvm_yaml_dump_all_to_file" => crate::kclvm_yaml_dump_all_to_file as *const () as u64,
         "kclvm_yaml_dump_to_file" => crate::kclvm_yaml_dump_to_file as *const () as u64,
         "kclvm_yaml_encode" => crate::kclvm_yaml_encode as *const () as u64,
+        "kclvm_yaml_encode_all" => crate::kclvm_yaml_encode_all as *const () as u64,
         _ => panic!("unknown {name}"),
     }
 }

kclvm/runtime/src/_kclvm_api_spec.rs

@@ -1146,11 +1146,23 @@
 // api-spec(c):    kclvm_value_ref_t* kclvm_yaml_encode(kclvm_context_t* ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* kwargs);
 // api-spec(llvm): declare %kclvm_value_ref_t* @kclvm_yaml_encode(%kclvm_context_t* %ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %kwargs);
 
+// api-spec:       kclvm_yaml_encode_all
+// api-spec(c):    kclvm_value_ref_t* kclvm_yaml_encode_all(kclvm_context_t* ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* kwargs);
+// api-spec(llvm): declare %kclvm_value_ref_t* @kclvm_yaml_encode_all(%kclvm_context_t* %ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %kwargs);
+
 // api-spec:       kclvm_yaml_decode
 // api-spec(c):    kclvm_value_ref_t* kclvm_yaml_decode(kclvm_context_t* ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* _kwargs);
 // api-spec(llvm): declare %kclvm_value_ref_t* @kclvm_yaml_decode(%kclvm_context_t* %ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %_kwargs);
 
+// api-spec:       kclvm_yaml_decode_all
+// api-spec(c):    kclvm_value_ref_t* kclvm_yaml_decode_all(kclvm_context_t* ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* _kwargs);
+// api-spec(llvm): declare %kclvm_value_ref_t* @kclvm_yaml_decode_all(%kclvm_context_t* %ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %_kwargs);
+
 // api-spec:       kclvm_yaml_dump_to_file
 // api-spec(c):    kclvm_value_ref_t* kclvm_yaml_dump_to_file(kclvm_context_t* _ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* kwargs);
 // api-spec(llvm): declare %kclvm_value_ref_t* @kclvm_yaml_dump_to_file(%kclvm_context_t* %_ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %kwargs);
 
+// api-spec:       kclvm_yaml_dump_all_to_file
+// api-spec(c):    kclvm_value_ref_t* kclvm_yaml_dump_all_to_file(kclvm_context_t* _ctx, kclvm_value_ref_t* args, kclvm_value_ref_t* kwargs);
+// api-spec(llvm): declare %kclvm_value_ref_t* @kclvm_yaml_dump_all_to_file(%kclvm_context_t* %_ctx, %kclvm_value_ref_t* %args, %kclvm_value_ref_t* %kwargs);
+

kclvm/runtime/src/value/val_plan.rs

@@ -7,7 +7,6 @@ use std::rc::Rc;
 
 pub const KCL_PRIVATE_VAR_PREFIX: &str = "_";
 const LIST_DICT_TEMP_KEY: &str = "$";
-const YAML_STREAM_SEP: &str = "\n---\n";
 const SCHEMA_TYPE_META_ATTR: &str = "_type";
 
 /// PlanOptions denotes the configuration required to execute the KCL

kclvm/runtime/src/yaml/mod.rs

@@ -1,8 +1,9 @@
 // Copyright 2021 The KCL Authors. All rights reserved.
 use crate::*;
 
-// encode(data, sort_keys=False, ignore_private=False, ignore_none=False):
+pub const YAML_STREAM_SEP: &str = "\n---\n";
 
+/// encode(data, sort_keys=False, ignore_private=False, ignore_none=False)
 #[no_mangle]
 #[runtime_fn]
 pub extern "C" fn kclvm_yaml_encode(
@@ -20,9 +21,35 @@ pub extern "C" fn kclvm_yaml_encode(
         );
         return s.into_raw(mut_ptr_as_ref(ctx));
     }
-    panic!("encode() missing 1 required positional argument: 'value'")
+    panic!("encode_all() missing 1 required positional argument: 'data'")
 }
 
+/// encode_all(data, sort_keys=False, ignore_private=False, ignore_none=False)
+#[no_mangle]
+#[runtime_fn]
+pub extern "C" fn kclvm_yaml_encode_all(
+    ctx: *mut kclvm_context_t,
+    args: *const kclvm_value_ref_t,
+    kwargs: *const kclvm_value_ref_t,
+) -> *const kclvm_value_ref_t {
+    let args = ptr_as_ref(args);
+    let kwargs = ptr_as_ref(kwargs);
+
+    if let Some(arg0) = args.arg_i(0) {
+        let opts = kwargs_to_opts(kwargs);
+        let results = arg0
+            .as_list_ref()
+            .values
+            .iter()
+            .map(|r| r.to_yaml_string_with_options(&opts))
+            .collect::<Vec<String>>();
+        let s = ValueRef::str(&results.join(YAML_STREAM_SEP));
+        return s.into_raw(mut_ptr_as_ref(ctx));
+    }
+    panic!("encode() missing 1 required positional argument: 'data'")
+}
+
+/// decode(value)
 #[no_mangle]
 #[runtime_fn]
 pub extern "C" fn kclvm_yaml_decode(
@@ -42,6 +69,27 @@ pub extern "C" fn kclvm_yaml_decode(
     panic!("decode() missing 1 required positional argument: 'value'")
 }
 
+/// decode_all(value)
+#[no_mangle]
+#[runtime_fn]
+pub extern "C" fn kclvm_yaml_decode_all(
+    ctx: *mut kclvm_context_t,
+    args: *const kclvm_value_ref_t,
+    _kwargs: *const kclvm_value_ref_t,
+) -> *const kclvm_value_ref_t {
+    let args = ptr_as_ref(args);
+
+    let ctx = mut_ptr_as_ref(ctx);
+    if let Some(arg0) = args.arg_i(0) {
+        match ValueRef::from_yaml_stream(ctx, arg0.as_str().as_ref()) {
+            Ok(x) => return x.into_raw(ctx),
+            Err(err) => panic!("{}", err),
+        }
+    }
+    panic!("decode_all() missing 1 required positional argument: 'value'")
+}
+
+/// dump_to_file(data, sort_keys=False, ignore_private=False, ignore_none=False)
 #[no_mangle]
 #[runtime_fn]
 pub extern "C" fn kclvm_yaml_dump_to_file(
@@ -63,6 +111,34 @@ pub extern "C" fn kclvm_yaml_dump_to_file(
     panic!("dump_to_file() missing 2 required positional arguments: 'data' and 'filename'")
 }
 
+/// dump_all_to_file(data, sort_keys=False, ignore_private=False, ignore_none=False)
+#[no_mangle]
+#[runtime_fn]
+pub extern "C" fn kclvm_yaml_dump_all_to_file(
+    _ctx: *mut kclvm_context_t,
+    args: *const kclvm_value_ref_t,
+    kwargs: *const kclvm_value_ref_t,
+) -> *const kclvm_value_ref_t {
+    let args = ptr_as_ref(args);
+    let kwargs = ptr_as_ref(kwargs);
+
+    if let Some(data) = args.arg_i(0) {
+        if let Some(filename) = args.arg_i(0) {
+            let filename = filename.as_str();
+            let opts = kwargs_to_opts(kwargs);
+            let results = data
+                .as_list_ref()
+                .values
+                .iter()
+                .map(|r| r.to_yaml_string_with_options(&opts))
+                .collect::<Vec<String>>();
+
+            std::fs::write(filename, results.join(YAML_STREAM_SEP)).expect("Unable to write file");
+        }
+    }
+    panic!("dump_all_to_file() missing 2 required positional arguments: 'data' and 'filename'")
+}
+
 fn kwargs_to_opts(kwargs: &ValueRef) -> YamlEncodeOptions {
     let mut opts = YamlEncodeOptions::default();
     if let Some(sort_keys) = kwargs.kwarg_bool("sort_keys", None) {

kclvm/sema/src/builtin/system_module.rs

@@ -842,7 +842,14 @@ register_regex_member! {
 // ------------------------------
 
 pub const YAML: &str = "yaml";
-pub const YAML_FUNCTION_NAMES: &[&str] = &["encode", "decode", "dump_to_file"];
+pub const YAML_FUNCTION_NAMES: &[&str] = &[
+    "encode",
+    "encode_all",
+    "decode",
+    "decode_all",
+    "dump_to_file",
+    "dump_all_to_file",
+];
 macro_rules! register_yaml_member {
     ($($name:ident => $ty:expr)*) => (
         pub const YAML_FUNCTION_TYPES: Lazy<IndexMap<String, Type>> = Lazy::new(|| {
@@ -882,6 +889,35 @@ register_yaml_member! {
         false,
         Some(1),
     )
+    encode_all => Type::function(
+        None,
+        Type::str_ref(),
+        &[
+            Parameter {
+                name: "data".to_string(),
+                ty: Type::list_ref(Type::any_ref()),
+                has_default: false,
+            },
+            Parameter {
+                name: "sort_keys".to_string(),
+                ty: Type::bool_ref(),
+                has_default: true,
+            },
+            Parameter {
+                name: "ignore_private".to_string(),
+                ty: Type::bool_ref(),
+                has_default: true,
+            },
+            Parameter {
+                name: "ignore_none".to_string(),
+                ty: Type::bool_ref(),
+                has_default: true,
+            },
+        ],
+        r#"Serialize a sequence of KCL objects into a YAML stream str."#,
+        false,
+        Some(1),
+    )
     decode => Type::function(
         None,
         Type::any_ref(),
@@ -896,6 +932,20 @@ register_yaml_member! {
         false,
         None,
     )
+    decode_all => Type::function(
+        None,
+        Type::list_ref(Type::any_ref()),
+        &[
+            Parameter {
+                name: "value".to_string(),
+                ty: Type::str_ref(),
+                has_default: false,
+            },
+        ],
+        r#"Parse all YAML documents in a stream and produce corresponding KCL objects."#,
+        false,
+        None,
+    )
     dump_to_file => Type::function(
         None,
         Type::str_ref(),
@@ -930,6 +980,40 @@ register_yaml_member! {
         false,
         Some(2),
     )
+    dump_all_to_file => Type::function(
+        None,
+        Type::str_ref(),
+        &[
+            Parameter {
+                name: "data".to_string(),
+                ty: Type::list_ref(Type::any_ref()),
+                has_default: false,
+            },
+            Parameter {
+                name: "filename".to_string(),
+                ty: Type::str_ref(),
+                has_default: false,
+            },
+            Parameter {
+                name: "sort_keys".to_string(),
+                ty: Type::bool_ref(),
+                has_default: true,
+            },
+            Parameter {
+                name: "ignore_private".to_string(),
+                ty: Type::bool_ref(),
+                has_default: true,
+            },
+            Parameter {
+                name: "ignore_none".to_string(),
+                ty: Type::bool_ref(),
+                has_default: true,
+            },
+        ],
+        r#"Serialize a sequence of KCL objects into a YAML stream str and write it into the file `filename`."#,
+        false,
+        Some(2),
+    )
 }
 
 // ------------------------------

kclvm/tools/src/LSP/src/notification.rs

@@ -94,7 +94,7 @@ impl LanguageServerState {
         let new_word_index = build_word_index_for_file_content(text.clone(), &text_document.uri);
         let binding = text_document.uri.path();
         let file_path = Path::new(binding); //todo rename
-        let mut word_index_map = &mut *self.word_index_map.write();
+        let word_index_map = &mut *self.word_index_map.write();
         for (key, value) in word_index_map {
             let workspace_folder_path = Path::new(key.path());
             if file_path.starts_with(workspace_folder_path) {

test/grammar/builtins/yaml/decode_all_0/main.k

@@ -0,0 +1,11 @@
+import yaml
+
+yamlStrList = [
+    'key: value',
+    '- 1\n- 2\n- 3',
+    '1',
+    '1.1',
+    'null',
+    'true',
+]
+data = [yaml.decode_all(s) for s in yamlStrList]

test/grammar/builtins/yaml/decode_all_0/stdout.golden

@@ -0,0 +1,19 @@
+yamlStrList:
+- 'key: value'
+- |-
+  - 1
+  - 2
+  - 3
+- '1'
+- '1.1'
+- 'null'
+- 'true'
+data:
+- key: value
+- - 1
+  - 2
+  - 3
+- 1
+- 1.1
+- null
+- true

test/grammar/builtins/yaml/decode_all_1/main.k

@@ -0,0 +1,9 @@
+import yaml
+
+yamlStrList = [
+    'key1: value2\n---\nkey2: [1, 2, 3]',
+    '- 1\n- 2\n- 3\n---\nkey: value',
+    '1\n---\n2',
+    '1.1\n---\nnull\n---\ntrue\n---\nfalse',
+]
+data = [yaml.decode_all(s) for s in yamlStrList]