Add HTTP API preview (#604)

* Add HTTP API preview

* dummy implement traits

* introduce memory limiter

* finish basic http impl

* change reexports

* fmt

* up

* rename wit to http-client

* set a default request timeout

* fix tests

* add http client test

* fix warnings and errors

* up

Cargo.toml

@@ -48,6 +48,7 @@ members = [
     "crates/kitsune-util",
     "crates/kitsune-wasm-mrf",
     "crates/kitsune-wasm-mrf/example-mrf",
+    "crates/kitsune-wasm-mrf/tests/http-client",
     "crates/kitsune-webfinger",
 
     "kitsune",
@@ -84,7 +85,7 @@ askama = { version = "0.12.1", default-features = false, features = [
 ] }
 asynk-strim = "0.1.2"
 clap = { version = "4.5.21", features = ["derive", "wrap_help"] }
-diesel = { version = "2.2.4", default-features = false, features = [
+diesel = { version = "2.2.5", default-features = false, features = [
     "32-column-tables",
     "with-deprecated",
 ] }
@@ -114,7 +115,7 @@ hickory-resolver = { version = "0.25.0-alpha.3", default-features = false, featu
     "native-certs",
     "tokio-runtime",
 ] }
-iso8601-timestamp = "0.3.0"
+iso8601-timestamp = "0.3.1"
 itertools = { version = "0.13.0", default-features = false }
 moka = { version = "=0.12.7", features = ["sync"] }
 simdutf8 = { version = "0.1.5", features = ["aarch64_neon"] }

crates/kitsune-activitypub/Cargo.toml

@@ -38,14 +38,14 @@ speedy-uuid = { workspace = true }
 tracing = "0.1.40"
 triomphe = { workspace = true }
 typed-builder = "0.20.0"
-url = "2.5.3"
+url = "2.5.4"
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 sha2 = { version = "0.10.8", features = ["asm"] }
 
 [dev-dependencies]
 http-body-util = "0.1.2"
-hyper = "1.5.0"
+hyper = "1.5.1"
 kitsune-config = { workspace = true }
 kitsune-test = { workspace = true }
 kitsune-webfinger = { workspace = true }

crates/kitsune-activitypub/src/fetcher/mod.rs

@@ -87,7 +87,7 @@ impl Fetcher {
                 .eq_ignore_ascii_case("application/ld+json")
                 && content_type
                     .get_param("profile")
-                    .map_or(false, |profile_urls| {
+                    .is_some_and(|profile_urls| {
                         profile_urls
                             .as_str()
                             .split_whitespace()

crates/kitsune-db/Cargo.toml

@@ -22,13 +22,13 @@ kitsune-language = { workspace = true }
 kitsune-type = { workspace = true }
 num-derive = "0.4.2"
 num-traits = "0.2.19"
-rustls = { version = "0.23.16", default-features = false, features = [
+rustls = { version = "0.23.18", default-features = false, features = [
     "logging",
     "ring",
     "std",
     "tls12",
 ] }
-rustls-native-certs = "0.8.0"
+rustls-native-certs = "0.8.1"
 serde = { version = "1.0.215", features = ["derive"] }
 sonic-rs = { workspace = true }
 speedy-uuid = { workspace = true, features = ["diesel"] }

crates/kitsune-derive/impl/Cargo.toml

@@ -9,9 +9,9 @@ license.workspace = true
 proc-macro = true
 
 [dependencies]
-proc-macro2 = "1.0.89"
+proc-macro2 = "1.0.92"
 quote = "1.0.37"
-syn = { version = "2.0.87", features = ["full"] }
+syn = { version = "2.0.89", features = ["full"] }
 
 [lints]
 workspace = true

crates/kitsune-error/Cargo.toml

@@ -11,7 +11,7 @@ eyre = "0.6.12"
 garde = { workspace = true }
 http = "1.1.0"
 sonic-rs = { workspace = true }
-sync_wrapper = "1.0.1"
+sync_wrapper = "1.0.2"
 tracing = "0.1.40"
 
 [lints]

crates/kitsune-federation-filter/Cargo.toml

@@ -11,7 +11,7 @@ kitsune-config = { workspace = true }
 kitsune-derive = { workspace = true }
 kitsune-error = { workspace = true }
 kitsune-type = { workspace = true }
-url = "2.5.3"
+url = "2.5.4"
 
 [lints]
 workspace = true

crates/kitsune-http-client/Cargo.toml

@@ -12,10 +12,11 @@ futures-util = { version = "0.3.31", default-features = false, features = [
     "alloc",
 ] }
 hickory-resolver = { workspace = true }
+http = "1.1.0"
 http-body = "1.0.1"
 http-body-util = "0.1.2"
 http-signatures = { workspace = true }
-hyper = "1.5.0"
+hyper = "1.5.1"
 hyper-util = { version = "0.1.10", features = [
     "client-legacy",
     "http1",
@@ -37,7 +38,7 @@ serde = "1.0.215"
 simdutf8 = { workspace = true }
 sonic-rs = { workspace = true }
 tower = { version = "0.5.1", features = ["util"] }
-tower-http = { version = "0.6.1", features = [
+tower-http = { version = "0.6.2", features = [
     # Explicitly exclude `zstd`
     # It's not widely adopted and takes a long time to build
     "decompression-br",

crates/kitsune-http-client/src/lib.rs

@@ -5,12 +5,12 @@ use self::{resolver::Resolver, util::BoxCloneService};
 use bytes::Buf;
 use futures_util::{Stream, StreamExt};
 use hickory_resolver::config::{ResolverConfig, ResolverOpts};
+use http::HeaderValue;
 use http_body::Body as HttpBody;
 use http_body_util::{BodyExt, BodyStream, Limited};
 use hyper::{
     body::Bytes,
     header::{HeaderName, USER_AGENT},
-    http::{self, HeaderValue},
     HeaderMap, Request, Response as HyperResponse, StatusCode, Uri, Version,
 };
 use hyper_rustls::HttpsConnectorBuilder;
@@ -39,9 +39,15 @@ type Result<T, E = Error> = std::result::Result<T, E>;
 /// Default body limit of 1MB
 const DEFAULT_BODY_LIMIT: usize = 1024 * 1024;
 
+/// Default request timeout of 30s (same as Firefox)
+const DEFAULT_REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
+
 /// Alias for our internal HTTP body type
 pub use self::body::Body;
 
+/// Response body type
+pub type ResponseBody = BoxBody;
+
 /// Client error type
 pub struct Error {
     inner: BoxError,
@@ -159,11 +165,11 @@ impl ClientBuilder {
     #[must_use]
     pub fn build(mut self) -> Client {
         let resolver = self.dns_resolver.take().unwrap_or_else(|| {
-            let resolver = hickory_resolver::TokioResolver::tokio(
+            hickory_resolver::TokioResolver::tokio(
                 ResolverConfig::quad9_tls(),
                 ResolverOpts::default(),
-            );
-            resolver.into()
+            )
+            .into()
         });
 
         let connector = HttpsConnectorBuilder::new()
@@ -230,7 +236,7 @@ impl Default for ClientBuilder {
             content_length_limit: Some(DEFAULT_BODY_LIMIT),
             default_headers: HeaderMap::default(),
             dns_resolver: None,
-            timeout: Option::default(),
+            timeout: Some(DEFAULT_REQUEST_TIMEOUT),
         };
 
         builder

crates/kitsune-observability/Cargo.toml

@@ -9,7 +9,7 @@ license.workspace = true
 async-trait = "0.1.83"
 eyre = "0.6.12"
 http-body-util = "0.1.2"
-hyper = { version = "1.5.0", default-features = false }
+hyper = { version = "1.5.1", default-features = false }
 kitsune-config = { workspace = true }
 kitsune-core = { workspace = true }
 kitsune-http-client = { workspace = true }

crates/kitsune-oidc/Cargo.toml

@@ -25,7 +25,7 @@ serde = { version = "1.0.215", features = ["derive"] }
 sonic-rs = { workspace = true }
 speedy-uuid = { workspace = true }
 thiserror = "2.0.3"
-url = "2.5.3"
+url = "2.5.4"
 
 [dev-dependencies]
 tokio = { version = "1.41.1", features = ["macros", "rt"] }

crates/kitsune-s3/Cargo.toml

@@ -11,7 +11,7 @@ futures-util = { version = "0.3.31", default-features = false }
 http = "1.1.0"
 kitsune-error = { workspace = true }
 kitsune-http-client = { workspace = true }
-quick-xml = { version = "0.37.0", features = ["serialize"] }
+quick-xml = { version = "0.37.1", features = ["serialize"] }
 rusty-s3 = "0.5.0"
 serde = { version = "1.0.215", features = ["derive"] }
 typed-builder = "0.20.0"

crates/kitsune-service/Cargo.toml

@@ -52,13 +52,13 @@ tokio = { workspace = true, features = ["macros", "sync"] }
 tracing = "0.1.40"
 triomphe = { workspace = true }
 typed-builder = "0.20.0"
-url = "2.5.3"
+url = "2.5.4"
 zxcvbn = { version = "3.1.0", default-features = false }
 
 [dev-dependencies]
 hex-simd = "0.8.0"
 http-body-util = "0.1.2"
-hyper = "1.5.0"
+hyper = "1.5.1"
 kitsune-activitypub = { workspace = true }
 kitsune-config = { workspace = true }
 kitsune-federation-filter = { workspace = true }

crates/kitsune-service/src/custom_emoji.rs

@@ -1,3 +1,5 @@
+#![allow(unexpected_cfgs)] // garde expansion issue
+
 use super::attachment::{AttachmentService, Upload};
 use bytes::Bytes;
 use diesel::{

crates/kitsune-service/src/user.rs

@@ -1,3 +1,5 @@
+#![allow(unexpected_cfgs)] // garde expansion issue
+
 use super::{
     captcha::CaptchaService,
     job::{Enqueue, JobService},

crates/kitsune-test/Cargo.toml

@@ -21,7 +21,7 @@ rand = "0.8.5"
 rusty-s3 = { version = "0.5.0", default-features = false }
 tokio = { workspace = true, features = ["time"] }
 triomphe = { workspace = true }
-url = "2.5.3"
+url = "2.5.4"
 uuid = { version = "1.11.0", features = ["fast-rng", "v4"] }
 
 [lints]

crates/kitsune-wasm-mrf/Cargo.toml

@@ -16,9 +16,12 @@ fred = { workspace = true }
 futures-util = { version = "0.3.31", default-features = false, features = [
     "alloc",
 ] }
+http = "1.1.0"
+http-body-util = "0.1.2"
 kitsune-config = { workspace = true }
 kitsune-derive = { workspace = true }
 kitsune-error = { workspace = true }
+kitsune-http-client = { workspace = true }
 kitsune-type = { workspace = true }
 mrf-manifest = { workspace = true, features = ["decode"] }
 redb = { version = "2.2.0", features = ["logging"] }
@@ -29,7 +32,7 @@ tokio = { workspace = true, features = ["fs"] }
 tracing = "0.1.40"
 triomphe = { workspace = true }
 walkdir = "2.5.0"
-wasmtime = { version = "26.0.1", default-features = false, features = [
+wasmtime = { version = "27.0.0", default-features = false, features = [
     "addr2line",
     "async",
     "component-model",
@@ -38,11 +41,13 @@ wasmtime = { version = "26.0.1", default-features = false, features = [
     "pooling-allocator",
     "runtime",
 ] }
-wasmtime-wasi = { version = "26.0.1", default-features = false }
+wasmtime-wasi = { version = "27.0.0", default-features = false }
 
 [dev-dependencies]
+bytes = "1.8.0"
 tempfile = "3.14.0"
 tokio = { workspace = true, features = ["macros", "rt"] }
+tower = "0.5.1"
 tracing-subscriber = "0.3.18"
 wat = "1.220.0"
 

crates/kitsune-wasm-mrf/example-mrf/manifest.json

@@ -3,5 +3,7 @@
     "apiVersion": "v1",
     "name": "example-mrf",
     "version": "1.0.0",
-    "activityTypes": ["*"]
-}
+    "activityTypes": [
+        "*"
+    ]
+}

crates/kitsune-wasm-mrf/src/ctx.rs

@@ -1,9 +1,12 @@
-use crate::{kv_storage, mrf_wit::v1::fep::mrf::keyvalue};
+use crate::{
+    kv_storage,
+    mrf_wit::v1::fep::mrf::{http_client, keyvalue},
+};
 use slab::Slab;
 use triomphe::Arc;
 use wasmtime::{
     component::{Resource, ResourceTable},
-    Engine, Store,
+    Engine, Store, StoreLimits, StoreLimitsBuilder,
 };
 use wasmtime_wasi::{WasiCtx, WasiCtxBuilder, WasiView};
 
@@ -23,8 +26,25 @@ impl KvContext {
     }
 }
 
+pub struct HttpContext {
+    pub client: kitsune_http_client::Client,
+    pub bodies: Slab<crate::http_client::Body>,
+}
+
+impl HttpContext {
+    #[inline]
+    pub fn get_body(
+        &mut self,
+        rep: &Resource<http_client::ResponseBody>,
+    ) -> &mut crate::http_client::Body {
+        &mut self.bodies[rep.rep() as usize]
+    }
+}
+
 pub struct Context {
+    pub http_ctx: HttpContext,
     pub kv_ctx: KvContext,
+    pub resource_limiter: StoreLimits,
     pub resource_table: ResourceTable,
     pub wasi_ctx: WasiCtx,
 }
@@ -42,6 +62,7 @@ impl WasiView for Context {
 #[inline]
 pub fn construct_store(
     engine: &Engine,
+    http_client: kitsune_http_client::Client,
     storage: Arc<kv_storage::BackendDispatch>,
 ) -> Store<Context> {
     let wasi_ctx = WasiCtxBuilder::new()
@@ -50,16 +71,24 @@ pub fn construct_store(
         .allow_udp(false)
         .build();
 
-    Store::new(
-        engine,
-        Context {
-            kv_ctx: KvContext {
-                module_name: None,
-                storage,
-                buckets: Slab::new(),
-            },
-            resource_table: ResourceTable::new(),
-            wasi_ctx,
+    let data = Context {
+        http_ctx: HttpContext {
+            client: http_client,
+            bodies: Slab::new(),
         },
-    )
+        kv_ctx: KvContext {
+            module_name: None,
+            storage,
+            buckets: Slab::new(),
+        },
+        resource_limiter: StoreLimitsBuilder::new()
+            .memory_size(15 * 1024 * 1024)
+            .build(),
+        resource_table: ResourceTable::new(),
+        wasi_ctx,
+    };
+
+    let mut store = Store::new(engine, data);
+    store.limiter(|store| &mut store.resource_limiter);
+    store
 }