feat!: simplify memory interface, use extism-convert for working with extism memory

Cargo.toml

@@ -14,13 +14,13 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 extism-pdk-derive = {path = "./derive", version = "0.3.1"}
 extism-manifest = {version = "0.5.0", optional = true}
-rmp-serde = {version = "1", optional = true}
+extism-convert = {git = "https://github.com/extism/extism", version = "0.1"} # TODO: change this after extism-convert has been released
 base64 = "0.21.0"
 
 [features]
 default = ["http", "msgpack"]
 http = ["extism-manifest"]
-msgpack = ["rmp-serde"]
+msgpack = ["extism-convert/msgpack"]
 
 [workspace]
 members = [

derive/src/lib.rs

@@ -1,12 +1,12 @@
 use quote::quote;
-use syn::{parse_macro_input, ItemFn, ItemForeignMod, ItemStruct};
+use syn::{parse_macro_input, ItemFn, ItemForeignMod};
 
 /// `plugin_fn` is used to define a function that will be exported by a plugin
 ///
 /// It should be added to a function you would like to export, the function should
 /// accept a parameter that implements `extism_pdk::FromBytes` and return a
 /// `extism_pdk::FnResult` that contains a value that implements
-/// `extism_pdk::ToMemory`.
+/// `extism_pdk::ToBytes`.
 #[proc_macro_attribute]
 pub fn plugin_fn(
     _attr: proc_macro::TokenStream,
@@ -26,9 +26,7 @@ pub fn plugin_fn(
     let output = &mut function.sig.output;
     let block = &function.block;
 
-    if inputs.is_empty() {
-        panic!("extism_pdk::plugin_fn expects a function with one argument, `()` may be used if no input is needed");
-    }
+    let no_args = inputs.is_empty();
 
     if name == "main" {
         panic!(
@@ -53,21 +51,56 @@ pub fn plugin_fn(
         }
     }
 
-    quote! {
-        #[no_mangle]
-        pub #constness #unsafety extern "C" fn #name() -> i32 {
-            #constness #unsafety fn inner #generics(#inputs) #output {
-                #block
+    if no_args {
+        quote! {
+            #[no_mangle]
+            pub #constness #unsafety extern "C" fn #name() -> i32 {
+                #constness #unsafety fn inner #generics() #output {
+                    #block
+                }
+
+                let output = match inner() {
+                    Ok(x) => x,
+                    Err(rc) => {
+                        let err = format!("{:?}", rc.0);
+                        let mut mem = extism_pdk::Memory::from_bytes(&err).unwrap();
+                        unsafe {
+                            extism_pdk::bindings::extism_error_set(mem.offset());
+                        }
+                        return rc.1;
+                    }
+                };
+                extism_pdk::unwrap!(extism_pdk::output(&output));
+                0
             }
+        }
+        .into()
+    } else {
+        quote! {
+            #[no_mangle]
+            pub #constness #unsafety extern "C" fn #name() -> i32 {
+                #constness #unsafety fn inner #generics(#inputs) #output {
+                    #block
+                }
 
-            let input = extism_pdk::unwrap!(extism_pdk::input());
-            let output = extism_pdk::unwrap!(inner(input));
-            let status = output.status();
-            unwrap!(extism_pdk::output(output));
-            0
+                let input = extism_pdk::unwrap!(extism_pdk::input());
+                let output = match inner(input) {
+                    Ok(x) => x,
+                    Err(rc) => {
+                        let err = format!("{:?}", rc.0);
+                        let mut mem = extism_pdk::Memory::from_bytes(&err).unwrap();
+                        unsafe {
+                            extism_pdk::bindings::extism_error_set(mem.offset());
+                        }
+                        return rc.1;
+                    }
+                };
+                extism_pdk::unwrap!(extism_pdk::output(&output));
+                0
+            }
         }
+        .into()
     }
-    .into()
 }
 
 /// `host_fn` is used to define a host function that will be callable from within a plugin
@@ -158,7 +191,7 @@ pub fn host_fn(
                             syn::Pat::Ident(i) => {
                                 if is_ptr {
                                     into_inputs.push(
-                                        quote!(extism_pdk::ToMemory::to_memory(&#i)?.keep().offset),
+                                        quote!(extism_pdk::ToMemory::to_memory(&&#i)?.offset()),
                                     );
                                 } else {
                                     into_inputs.push(quote!(#i));
@@ -195,7 +228,7 @@ pub fn host_fn(
 
                     #vis unsafe fn #name #generics (#original_inputs) -> Result<#output, extism_pdk::Error> {
                         let res = extism_pdk::Memory::from(#impl_name(#(#into_inputs),*));
-                        <#output as extism_pdk::FromBytes>::from_bytes(res.to_vec())
+                        <#output as extism_pdk::FromBytes>::from_bytes(&res.to_vec())
                     }
                 };
             } else {
@@ -215,63 +248,3 @@ pub fn host_fn(
 
     gen.into()
 }
-
-struct Args {
-    arg: Vec<syn::Path>,
-}
-
-impl syn::parse::Parse for Args {
-    fn parse(input: syn::parse::ParseStream) -> syn::Result<Self> {
-        let args =
-            syn::punctuated::Punctuated::<syn::Path, syn::Token![,]>::parse_terminated(input)?;
-        Ok(Args {
-            arg: args.into_iter().collect(),
-        })
-    }
-}
-
-/// `encoding` is used to add a new serde encoder/decoder. It accepts two parameters:
-/// 1) path to serialization function
-/// 2) path to deserialization function
-///
-/// ```rust,ignore
-/// #[encoding(serde_json::to_vec, serde_json::from_slice)]]
-/// pub struct Json;
-/// ```
-#[proc_macro_attribute]
-pub fn encoding(
-    attr: proc_macro::TokenStream,
-    item: proc_macro::TokenStream,
-) -> proc_macro::TokenStream {
-    let item = parse_macro_input!(item as ItemStruct);
-    let args = parse_macro_input!(attr as Args);
-
-    if args.arg.len() != 2 {
-        panic!("extism_pdk::encoding expects 2 arguments (encoding function and decoding function) but got {}", args.arg.len())
-    }
-
-    let vis = item.vis;
-    let name = &item.ident;
-
-    let encode = &args.arg[0];
-    let decode = &args.arg[1];
-
-    quote! {
-        #vis struct #name<T>(pub T);
-
-        impl<T: serde::de::DeserializeOwned> extism_pdk::FromBytes for #name<T> {
-            fn from_bytes(d: Vec<u8>) -> Result<Self, extism_pdk::Error> {
-                let x = #decode(&d)?;
-                Ok(#name(x))
-            }
-        }
-
-        impl<T: serde::Serialize> extism_pdk::ToMemory for #name<T> {
-            fn to_memory(&self) -> Result<extism_pdk::Memory, extism_pdk::Error> {
-                let x = #encode(&self.0)?;
-                Ok(extism_pdk::Memory::from_bytes(x))
-            }
-        }
-    }
-    .into()
-}

examples/count_vowels.rs

@@ -26,7 +26,7 @@ pub unsafe fn count_vowels<'a>(input: String) -> FnResult<Json<TestOutput<'a>>>
 
     let a = var::get("a")?.expect("variable 'a' set");
     let a = String::from_utf8(a).expect("string from varible value");
-    let config = config::get("thing").expect("'thing' key set in config");
+    let config = config::get("thing")?.expect("'thing' key set in config");
     let b = "new_value";
 
     let output = TestOutput {

examples/http.rs

@@ -5,6 +5,6 @@ use extism_pdk::*;
 #[plugin_fn]
 pub fn http_get(Json(req): Json<HttpRequest>) -> FnResult<Memory> {
     info!("Request to: {}", req.url);
-    let res = http::request::<String>(&req, None)?;
-    Ok(res.into_memory().keep())
+    let res = http::request::<()>(&req, None)?;
+    Ok(res.into_memory())
 }

src/config.rs

@@ -1,21 +1,24 @@
 use crate::*;
 
-pub fn get_memory(key: impl AsRef<str>) -> Option<Memory> {
-    let mem = Memory::from_bytes(key.as_ref().as_bytes());
+pub fn get_memory(key: impl AsRef<str>) -> Result<Option<Memory>, Error> {
+    let mem = Memory::from_bytes(key.as_ref().as_bytes())?;
 
-    let offset = unsafe { extism_config_get(mem.offset) };
+    let offset = unsafe { extism_config_get(mem.offset()) };
     if offset == 0 {
-        return None;
+        return Ok(None);
     }
 
     let len = unsafe { extism_length(offset) };
     if len == 0 {
-        return None;
+        return Ok(None);
     }
 
-    Some(Memory::wrap(offset, len, true))
+    Ok(Some(Memory(MemoryHandle {
+        offset,
+        length: len,
+    })))
 }
 
-pub fn get(key: impl AsRef<str>) -> Option<String> {
-    get_memory(key).map(|x| x.to_string().expect("Config value is not a valid string"))
+pub fn get(key: impl AsRef<str>) -> Result<Option<String>, Error> {
+    Ok(get_memory(key)?.map(|x| x.to_string().expect("Config value is not a valid string")))
 }

src/http.rs

@@ -37,17 +37,20 @@ pub fn request<T: ToMemory>(
     body: Option<T>,
 ) -> Result<HttpResponse, Error> {
     let enc = serde_json::to_vec(req)?;
-    let req = Memory::from_bytes(enc);
+    let req = Memory::from_bytes(enc)?;
     let body = match body {
         Some(b) => Some(b.to_memory()?),
         None => None,
     };
-    let data = body.as_ref().map(|x| x.offset).unwrap_or(0);
-    let offs = unsafe { extism_http_request(req.offset, data) };
+    let data = body.as_ref().map(|x| x.offset()).unwrap_or(0);
+    let offs = unsafe { extism_http_request(req.offset(), data) };
     let status = unsafe { extism_http_status_code() };
     let len = unsafe { extism_length(offs) };
     Ok(HttpResponse {
-        memory: Memory::wrap(offs, len, true),
+        memory: Memory(MemoryHandle {
+            offset: offs,
+            length: len,
+        }),
         status: status as u16,
     })
 }

src/lib.rs

@@ -5,8 +5,7 @@ mod macros;
 
 pub mod bindings;
 pub mod config;
-mod from_bytes;
-mod memory;
+pub mod memory;
 mod to_memory;
 pub mod var;
 
@@ -16,8 +15,9 @@ pub mod http;
 
 pub use anyhow::Error;
 pub(crate) use bindings::*;
-pub use extism_pdk_derive::{encoding, host_fn, plugin_fn};
-pub use from_bytes::FromBytes;
+pub use extism_convert::*;
+pub use extism_convert::{FromBytes, FromBytesOwned, ToBytes};
+pub use extism_pdk_derive::{host_fn, plugin_fn};
 pub use memory::Memory;
 pub use to_memory::ToMemory;
 
@@ -44,34 +44,28 @@ pub enum LogLevel {
 /// Re-export of `serde_json`
 pub use serde_json as json;
 
-use crate as extism_pdk;
-
-/// JSON encoding
-#[encoding(serde_json::to_vec, serde_json::from_slice)]
-pub struct Json;
-
-/// MsgPack encoding
-#[cfg_attr(
-    feature = "msgpack",
-    encoding(rmp_serde::to_vec, rmp_serde::from_slice)
-)]
-pub struct MsgPack;
-
-/// Base64 conversion
+/// Base64 string
 pub struct Base64(pub String);
 
-/// Get input from host
-pub fn input<T: FromBytes>() -> Result<T, Error> {
-    unsafe { T::from_bytes(extism_load_input()) }
+/// Get input bytes from host
+pub fn input_bytes() -> Vec<u8> {
+    unsafe { extism_load_input() }
+}
+
+/// Get input bytes from host and convert into `T`
+pub fn input<T: FromBytesOwned>() -> Result<T, Error> {
+    let data = input_bytes();
+    T::from_bytes_owned(&data)
 }
 
 /// Set output for host
-pub fn output(data: impl ToMemory) -> Result<(), Error> {
-    data.to_memory()?.set_output();
+pub fn output<'a, T: ToMemory>(data: T) -> Result<(), Error> {
+    let data = data.to_memory()?;
+    data.set_output();
     Ok(())
 }
 
-pub struct WithReturnCode<T>(T, i32);
+pub struct WithReturnCode<T>(pub T, pub i32);
 
 impl<E: Into<Error>> From<E> for WithReturnCode<Error> {
     fn from(value: E) -> Self {