diff --git a/Cargo.lock b/Cargo.lock index b3438c5f5..5c9e919fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -536,9 +536,9 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "autocfg" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "autometrics" @@ -1500,18 +1500,18 @@ dependencies = [ [[package]] name = "cranelift-bforest" -version = "0.107.0" +version = "0.107.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79b27922a6879b5b5361d0a084cb0b1941bf109a98540addcb932da13b68bed4" +checksum = "3c7a700bc3e2e834c81c7e9e57c6b27049172fc6156a6a406dd2265d64c30409" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.107.0" +version = "0.107.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "304c455b28bf56372729acb356afbb55d622f2b0f2f7837aa5e57c138acaac4d" +checksum = "35bb55da792dc46a35e6e283b02b9d6aa7f21349691f41176d8ecf4a1494b6ad" dependencies = [ "bumpalo", "cranelift-bforest", @@ -1530,33 +1530,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.107.0" +version = "0.107.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1653c56b99591d07f67c5ca7f9f25888948af3f4b97186bff838d687d666f613" +checksum = "bf590672e1a7580158968898ade9c82801d6ebeaa4649d6f99f9cbb411512a8b" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.107.0" +version = "0.107.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5b6a9cf6b6eb820ee3f973a0db313c05dc12d370f37b4fe9630286e1672573f" +checksum = "3ab16a95d8b10b72eaa8bb9f0ce0fdd013e5f225bea92f69d2dbd2424aae5381" [[package]] name = "cranelift-control" -version = "0.107.0" +version = "0.107.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9d06e6bf30075fb6bed9e034ec046475093392eea1aff90eb5c44c4a033d19a" +checksum = "60448a5af01f5716877e616ec1613424fb65f427320c6e6447315966507f12da" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.107.0" +version = "0.107.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29be04f931b73cdb9694874a295027471817f26f26d2f0ebe5454153176b6e3a" +checksum = "78689d6588f53981bf005b82d94125cc40cf9856b667bc276d1acdede400b633" dependencies = [ "serde", "serde_derive", @@ -1564,9 +1564,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.107.0" +version = "0.107.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a07fd7393041d7faa2f37426f5dc7fc04003b70988810e8c063beefeff1cd8f9" +checksum = "c86da6e45adc39e8d66a73d6fb782b0d7961df9e44b724246e01e515f86b1f0d" dependencies = [ "cranelift-codegen", "log", @@ -1576,15 +1576,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.107.0" +version = "0.107.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f341d7938caa6dff8149dac05bb2b53fc680323826b83b4cf175ab9f5139a3c9" +checksum = "dcb6f70d94826074699bc350c33dab82eb6f06022a3518bb466f501569d7379e" [[package]] name = "cranelift-native" -version = "0.107.0" +version = "0.107.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82af6066e6448d26eeabb7aa26a43f7ff79f8217b06bade4ee6ef230aecc8880" +checksum = "e27ec0ef4115eb9cc15be9c841085db17233403f8ed325e02bd6a9d78d6c939c" dependencies = [ "cranelift-codegen", "libc", @@ -1593,9 +1593,9 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.107.0" +version = "0.107.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2766fab7284a914a7f17f90ebe865c86453225fb8637ac31f123f5028fee69cd" +checksum = "fce009efc27c598ac0c019c7f3dc582403835547128aeb497edf76cb77d75250" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -3106,9 +3106,9 @@ dependencies = [ [[package]] name = "http-range-header" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ce4ef31cda248bbdb6e6820603b82dfcd9e833db65a43e997a0ccec777d11fe" +checksum = "08a397c49fec283e3d6211adbe480be95aae5f304cfb923e9970e08956d5168a" [[package]] name = "http-signatures" @@ -4307,6 +4307,7 @@ name = "kitsune-wasm-mrf" version = "0.0.1-pre.6" dependencies = [ "async-trait", + "blake3", "color-eyre", "derive_more 1.0.0-beta.6", "enum_dispatch", @@ -4329,6 +4330,7 @@ dependencies = [ "walkdir", "wasmtime", "wasmtime-wasi", + "wat", ] [[package]] @@ -5062,16 +5064,15 @@ dependencies = [ [[package]] name = "multer" -version = "3.0.0" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15d522be0a9c3e46fd2632e272d178f56387bdb5c9fbb3a36c649062e9b5219" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" dependencies = [ "bytes", "encoding_rs", "futures-util", "http 1.1.0", "httparse", - "log", "memchr", "mime", "spin 0.9.8", @@ -5185,9 +5186,9 @@ dependencies = [ [[package]] name = "num-iter" -version = "0.1.44" +version = "0.1.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" dependencies = [ "autocfg", "num-integer", @@ -5208,9 +5209,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", "libm", @@ -7759,9 +7760,9 @@ dependencies = [ [[package]] name = "testcontainers-modules" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "174f65718996e5c8e6771cc69eab5b5c5b877f1e7db9696800f8b7edf99a7b36" +checksum = "e8da66a6ebd55684c8e3c58c7374dc2b2d0d1884ac688987e7ffb2d02103a3ee" dependencies = [ "testcontainers", ] @@ -8829,9 +8830,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5990663c28d81015ddbb02a068ac1bf396a4ea296eba7125b2dfc7c00cb52e" +checksum = "2364a810370f08ece49d013255058c3c88ca6c0a080de66549233b7d2ca078b8" dependencies = [ "addr2line", "anyhow", @@ -8869,18 +8870,18 @@ dependencies = [ [[package]] name = "wasmtime-asm-macros" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625ee94c72004f3ea0228989c9506596e469517d7d0ed66f7300d1067bdf1ca9" +checksum = "c52cceae147514e279460ac3c43c1ea440c51c39202842611623b3f9734f73a8" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-component-macro" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64f84414a25ee3a624c8b77550f3fe7b5d8145bd3405ca58886ee6900abb6dc2" +checksum = "c8fd43a734bf424e3983363f1b7532a997edb206f52104568058e412e18db9ba" dependencies = [ "anyhow", "proc-macro2", @@ -8893,15 +8894,15 @@ dependencies = [ [[package]] name = "wasmtime-component-util" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78580bdb4e04c7da3bf98088559ca1d29382668536e4d5c7f2f966d79c390307" +checksum = "44d65e6a21c3e3482240ff03cc26f5c7ae7ee5df524c2283f39cf492ab711a15" [[package]] name = "wasmtime-cranelift" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b60df0ee08c6a536c765f69e9e8205273435b66d02dd401e938769a2622a6c1a" +checksum = "e2a6ed70499769b4d51f6f656204b0806b3d783f6ec5a8517c68e8b75ecaad19" dependencies = [ "anyhow", "cfg-if", @@ -8923,9 +8924,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64ffc1613db69ee47c96738861534f9a405e422a5aa00224fbf5d410b03fb445" +checksum = "90f7b5dbae8c3c6586e22f063ddb9e5cbf02c09629df75e5d8710f7bf880b117" dependencies = [ "anyhow", "bincode", @@ -8947,9 +8948,9 @@ dependencies = [ [[package]] name = "wasmtime-fiber" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f043514a23792761c5765f8ba61a4aa7d67f260c0c37494caabceb41d8ae81de" +checksum = "582e7ef625be814c57b1f8c3924c8899560d315f2285436c9184a09985dc3756" dependencies = [ "anyhow", "cc", @@ -8962,9 +8963,9 @@ dependencies = [ [[package]] name = "wasmtime-jit-icache-coherence" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a9f93a3289057b26dc75eb84d6e60d7694f7d169c7c09597495de6e016a13ff" +checksum = "b55b43e693c0beeca494d522f4850afca53cb46acf309483aef32a125276ee78" dependencies = [ "cfg-if", "libc", @@ -8973,9 +8974,9 @@ dependencies = [ [[package]] name = "wasmtime-runtime" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6332a2b0af4224c3ea57c857ad39acd2780ccc2b0c99ba1baa01864d90d7c94" +checksum = "1d2b876c09b7863d8a01bf87eb45f3b121fab245f8afbff7c38c38c1c9059aee" dependencies = [ "anyhow", "cc", @@ -9001,15 +9002,15 @@ dependencies = [ [[package]] name = "wasmtime-slab" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b3655075824a374c536a2b2cc9283bb765fcdf3d58b58587862c48571ad81ef" +checksum = "05c8ddfb8ebbab6ac186bc1f8c02ed988bc9ea455fea10f72bc3a07503309b4b" [[package]] name = "wasmtime-types" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b98cf64a242b0b9257604181ca28b28a5fcaa4c9ea1d396f76d1d2d1c5b40eef" +checksum = "3fa3a1f3c0deb3034d76e7dcf340c5df670a6603019ee5b58adb70870649c769" dependencies = [ "cranelift-entity", "serde", @@ -9020,9 +9021,9 @@ dependencies = [ [[package]] name = "wasmtime-versioned-export-macros" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8561d9e2920db2a175213d557d71c2ac7695831ab472bbfafb9060cd1034684f" +checksum = "b85321f0a1cd3c859b94e728533ba00074d3eca62362acf6998be0eab6f4001c" dependencies = [ "proc-macro2", "quote", @@ -9031,9 +9032,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34e1f53a9d4688a138282580fa7a46cbf1a41524f0e50c7e402e1407246f0155" +checksum = "0c5966b1aa330f07ef58f83b074908ea210ee864948ae3697f8892c91104e6e3" dependencies = [ "anyhow", "async-trait", @@ -9061,9 +9062,9 @@ dependencies = [ [[package]] name = "wasmtime-winch" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06b573d14ac846a0fb8c541d8fca6a64acf9a1d176176982472274ab1d2fa5d" +checksum = "e0d446696aa83f680d85e188670631cb7958957f63d027d6c36b945c2baa3e1e" dependencies = [ "anyhow", "cranelift-codegen", @@ -9078,9 +9079,9 @@ dependencies = [ [[package]] name = "wasmtime-wit-bindgen" -version = "20.0.0" +version = "20.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595bc7bb3b0ff4aa00fab718c323ea552c3034d77abc821a35112552f2ea487a" +checksum = "4d1d0c83af38eb6918af9c7cbc07d39f741a7baa9ddd152e19d9f93ff627dc05" dependencies = [ "anyhow", "heck 0.4.1", @@ -9205,9 +9206,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "0.18.0" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb23450977f9d4a23c02439cf6899340b2d68887b19465c5682740d9cc37d52e" +checksum = "720aabcf6838b31b42c7adc04d847696b066ddc5877efd6071ed5f08ae75bf20" dependencies = [ "anyhow", "cranelift-codegen", diff --git a/crates/kitsune-config/src/lib.rs b/crates/kitsune-config/src/lib.rs index 71dbcb50c..5dd19ec0d 100644 --- a/crates/kitsune-config/src/lib.rs +++ b/crates/kitsune-config/src/lib.rs @@ -41,6 +41,7 @@ pub struct Configuration { } impl Configuration { + #[inline] pub async fn load

(path: P) -> Result where P: AsRef, diff --git a/crates/kitsune-config/src/mrf.rs b/crates/kitsune-config/src/mrf.rs index be7188f74..67adce6b8 100644 --- a/crates/kitsune-config/src/mrf.rs +++ b/crates/kitsune-config/src/mrf.rs @@ -2,6 +2,11 @@ use serde::{Deserialize, Serialize}; use smol_str::SmolStr; use std::{collections::HashMap, num::NonZeroUsize}; +#[derive(Clone, Debug, Deserialize, Serialize)] +pub struct ArtifactCache { + pub path: SmolStr, +} + #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub struct FsKvStorage { @@ -25,6 +30,7 @@ pub enum KvStorage { #[derive(Clone, Debug, Deserialize, Serialize)] #[serde(rename_all = "kebab-case")] pub struct Configuration { + pub artifact_cache: Option, pub module_dir: SmolStr, pub module_config: HashMap, pub storage: KvStorage, diff --git a/crates/kitsune-db/Cargo.toml b/crates/kitsune-db/Cargo.toml index cd576a3af..791d9060c 100644 --- a/crates/kitsune-db/Cargo.toml +++ b/crates/kitsune-db/Cargo.toml @@ -26,7 +26,7 @@ kitsune-error = { path = "../kitsune-error" } kitsune-language = { path = "../kitsune-language" } kitsune-type = { path = "../kitsune-type" } num-derive = "0.4.2" -num-traits = "0.2.18" +num-traits = "0.2.19" rustls = { version = "0.23.5", default-features = false, features = [ "logging", "ring", diff --git a/crates/kitsune-test/Cargo.toml b/crates/kitsune-test/Cargo.toml index 20998c115..c47d1acf1 100644 --- a/crates/kitsune-test/Cargo.toml +++ b/crates/kitsune-test/Cargo.toml @@ -24,7 +24,7 @@ redis = { version = "0.25.3", default-features = false, features = [ ] } rusty-s3 = { version = "0.5.0", default-features = false } testcontainers = "0.16.7" -testcontainers-modules = { version = "0.4.1", features = [ +testcontainers-modules = { version = "0.4.2", features = [ "minio", "postgres", "redis", diff --git a/crates/kitsune-wasm-mrf/Cargo.toml b/crates/kitsune-wasm-mrf/Cargo.toml index 8b744d844..51f9dbedf 100644 --- a/crates/kitsune-wasm-mrf/Cargo.toml +++ b/crates/kitsune-wasm-mrf/Cargo.toml @@ -8,6 +8,7 @@ build = "build.rs" [dependencies] async-trait = "0.1.80" +blake3 = "1.5.1" color-eyre = "0.6.3" derive_more = { version = "1.0.0-beta.6", features = ["from"] } enum_dispatch = "0.3.13" @@ -31,7 +32,7 @@ tokio = { version = "1.37.0", features = ["fs"] } tracing = "0.1.40" typed-builder = "0.18.2" walkdir = "2.5.0" -wasmtime = { version = "20.0.0", default-features = false, features = [ +wasmtime = { version = "20.0.1", default-features = false, features = [ "addr2line", "async", "component-model", @@ -40,12 +41,13 @@ wasmtime = { version = "20.0.0", default-features = false, features = [ "pooling-allocator", "runtime", ] } -wasmtime-wasi = { version = "20.0.0", default-features = false } +wasmtime-wasi = { version = "20.0.1", default-features = false } [dev-dependencies] tempfile = "3.10.1" tokio = { version = "1.37.0", features = ["macros", "rt"] } tracing-subscriber = "0.3.18" +wat = "1.206.0" [lints] workspace = true diff --git a/crates/kitsune-wasm-mrf/src/cache.rs b/crates/kitsune-wasm-mrf/src/cache.rs new file mode 100644 index 000000000..14581fc5c --- /dev/null +++ b/crates/kitsune-wasm-mrf/src/cache.rs @@ -0,0 +1,81 @@ +use color_eyre::eyre::{self, Result}; +use std::path::Path; +use wasmtime::component::Component; + +pub struct Cache { + inner: sled::Db, +} + +impl Cache { + #[inline] + pub fn open

(path: P) -> Result + where + P: AsRef, + { + Ok(Self { + inner: sled::open(path)?, + }) + } + + #[inline] + #[instrument(skip_all)] + pub fn load( + &self, + engine: &wasmtime::Engine, + component_src: &[u8], + ) -> Result> { + let hash = blake3::hash(component_src); + let Some(precompiled) = self.inner.get(hash.as_bytes())? else { + return Ok(None); + }; + + debug!(hash = %hash.to_hex(), "hit component cache"); + + // SAFETY: The function is defined as unsafe since it is only doing very simple checks whether the precompiled component inside is actually valid + // But since we source our cache from disk, we can assume that the files are fine. If they aren't, the user has tempered with them or they were otherwise corrupted. + // If that's the case the user has bigger issues than a little memory unsafety here. And it's also nothing we can really protect against. + #[allow(unsafe_code)] + Ok(unsafe { Component::deserialize(engine, precompiled).ok() }) + } + + #[inline] + #[instrument(skip_all)] + pub fn store(&self, component_src: &[u8], component: &Component) -> Result<()> { + let hash = blake3::hash(component_src); + self.inner.insert( + hash.as_bytes(), + component.serialize().map_err(eyre::Report::msg)?, + )?; + + debug!(hash = %hash.to_hex(), "stored component in cache"); + + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::Cache; + use wasmtime::component::Component; + + #[test] + fn roundtrip() { + let tempdir = tempfile::tempdir().unwrap(); + let cache = Cache::open(tempdir.path()).unwrap(); + + let mut config = wasmtime::Config::new(); + config.wasm_component_model(true); + let engine = wasmtime::Engine::new(&config).unwrap(); + + let component_src = wat::parse_str("( component )").unwrap(); + let component = Component::from_binary(&engine, &component_src).unwrap(); + + cache.store(&component_src, &component).unwrap(); + let loaded_component = cache.load(&engine, &component_src).unwrap().unwrap(); + + assert_eq!( + loaded_component.serialize().unwrap(), + component.serialize().unwrap() + ); + } +} diff --git a/crates/kitsune-wasm-mrf/src/lib.rs b/crates/kitsune-wasm-mrf/src/lib.rs index 650e98be9..b519f5be7 100644 --- a/crates/kitsune-wasm-mrf/src/lib.rs +++ b/crates/kitsune-wasm-mrf/src/lib.rs @@ -2,6 +2,7 @@ extern crate tracing; use self::{ + cache::Cache, ctx::{construct_store, Context}, mrf_wit::v1::fep::mrf::types::{Direction, Error as MrfError}, }; @@ -29,6 +30,7 @@ use wasmtime::{ Config, Engine, InstanceAllocationStrategy, }; +mod cache; mod ctx; mod logging; mod mrf_wit; @@ -57,15 +59,31 @@ where } #[inline] +#[instrument(skip_all, fields(module_path = %module_path.display()))] fn load_mrf_module( + cache: Option<&Cache>, engine: &Engine, module_path: &Path, bytes: &[u8], ) -> eyre::Result, Component)>> { - let component = Component::new(engine, bytes) - .map_err(eyre::Report::msg) - .with_note(|| format!("path to the module: {}", module_path.display())) - .suggestion("Did you make the WASM file a component via `wasm-tools`?")?; + let compile_component = || { + Component::new(engine, bytes) + .map_err(eyre::Report::msg) + .with_note(|| format!("path to the module: {}", module_path.display())) + .suggestion("Did you make the WASM file a component via `wasm-tools`?") + }; + + let component = if let Some(cache) = cache { + if let Some(component) = cache.load(engine, bytes)? { + component + } else { + let component = compile_component()?; + cache.store(bytes, &component)?; + component + } + } else { + compile_component()? + }; let Some((manifest, _section_range)) = mrf_manifest::decode(bytes)? else { error!("missing manifest. skipping load."); @@ -123,6 +141,12 @@ impl MrfService { #[instrument(skip_all, fields(module_dir = %config.module_dir))] pub async fn from_config(config: &MrfConfiguration) -> eyre::Result { + let cache = config + .artifact_cache + .as_ref() + .map(|cache_config| Cache::open(cache_config.path.as_str())) + .transpose()?; + let storage = match config.storage { KvStorage::Fs(FsKvStorage { ref path }) => { kv_storage::FsBackend::from_path(path.as_str())?.into() @@ -145,10 +169,12 @@ impl MrfService { let wasm_data_stream = find_mrf_modules(config.module_dir.as_str()) .map_err(eyre::Report::from) .and_then(|(module_path, wasm_data)| { + let cache = cache.as_ref(); let engine = &engine; - async move { load_mrf_module(engine, &module_path, &wasm_data) } + async move { load_mrf_module(cache, engine, &module_path, &wasm_data) } }); + tokio::pin!(wasm_data_stream); let mut modules = Vec::new(); diff --git a/docs/src/configuring/mrf.md b/docs/src/configuring/mrf.md index 7f35706ab..94271907c 100644 --- a/docs/src/configuring/mrf.md +++ b/docs/src/configuring/mrf.md @@ -22,3 +22,52 @@ This configuration option tells Kitsune where to scan for WASM modules to load a [mrf] module-dir = "mrf-modules" ``` + +### `artifact-cache` + +This configuration option tells Kitsune to cache compiled versions of the WASM components on-disk to improve startup times +by not having to recompile the components every time Kitsune starts up. + +Simply omitting this configuration disables the caching. + +```toml +[mrf.artifact-cache] +path = "./artifact-cache" +``` + +> Note: DO NOT modify the files inside the artifact cache. You can delete the cache to reclaim space, but DO NOT modify the files. +> The files inside the cache are treated by Kitsune as trusted executables. Modifying them may lead to unexpected behaviour and crashes. + +### `storage` + +Kitsune provides MRF modules with scoped key-value storages to allow them to persist data across runs for things like counters or spam lists. + +We provide multiple backends here: + +#### Redis + +```toml +[mrf.storage] +type = "redis" +url = "redis://localhost" +pool-size = 5 +``` + +#### Filesystem + +```toml +[mrf.storage] +type = "fs" +path = "./mrf-storage" +``` + +### `module-config` + +WASM MRFs can have configuration passed to them upon invocation. In Kitsune you configure them via key-value pairs inside your configuration as follows: + +```toml +[mrf.module-config] +module-name = "my configuration" +``` + +The names that are used to select the configuration are sourced from the module manifest.