From 53d9a423e8822b972d41322c6327d03efd43b4b4 Mon Sep 17 00:00:00 2001 From: aumetra Date: Wed, 25 Oct 2023 22:24:05 +0200 Subject: [PATCH 01/16] Check `Content-Type` header when fetching entities --- Cargo.lock | 1 + Cargo.toml | 5 --- crates/kitsune-core/Cargo.toml | 1 + .../kitsune-core/src/activitypub/fetcher.rs | 33 ++++++++++++++++--- crates/kitsune-core/src/error.rs | 3 ++ 5 files changed, 34 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a3774819..c345039fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2925,6 +2925,7 @@ dependencies = [ "kitsune-type", "mime", "mime_guess", + "once_cell", "password-hash", "pkcs8", "post-process", diff --git a/Cargo.toml b/Cargo.toml index c1b103412..0682428f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,3 @@ -[profile.dev.package.backtrace] -opt-level = 3 - -[profile.dev.package.num-bigint-dig] -opt-level = 3 [profile.release] codegen-units = 1 diff --git a/crates/kitsune-core/Cargo.toml b/crates/kitsune-core/Cargo.toml index 8ab05908a..2614cd6a7 100644 --- a/crates/kitsune-core/Cargo.toml +++ b/crates/kitsune-core/Cargo.toml @@ -50,6 +50,7 @@ kitsune-storage = { path = "../kitsune-storage" } kitsune-type = { path = "../kitsune-type" } mime = "0.3.17" mime_guess = { version = "2.0.4", default-features = false } +once_cell = "1.18.0" password-hash = { version = "0.5.0", features = ["std"] } pkcs8 = { version = "0.10.2", features = ["std"] } post-process = { path = "../../lib/post-process" } diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index b8f67f5da..3f6fd649f 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -12,7 +12,7 @@ use async_recursion::async_recursion; use autometrics::autometrics; use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper}; use diesel_async::RunQueryDsl; -use http::HeaderValue; +use http::{header::CONTENT_TYPE, HeaderValue}; use kitsune_cache::{ArcCache, CacheBackend}; use kitsune_db::{ model::{ @@ -25,8 +25,12 @@ use kitsune_db::{ use kitsune_embed::Client as EmbedClient; use kitsune_http_client::Client; use kitsune_search::{SearchBackend, SearchService}; -use kitsune_type::ap::{actor::Actor, Object}; +use kitsune_type::{ + ap::{actor::Actor, Object}, + jsonld::RdfNode, +}; use scoped_futures::ScopedFutureExt; +use serde::de::DeserializeOwned; use typed_builder::TypedBuilder; use url::Url; @@ -88,6 +92,27 @@ pub struct Fetcher { } impl Fetcher { + async fn fetch_ap_resource(&self, url: &str) -> Result + where + T: DeserializeOwned + RdfNode, + { + let response = self.client.get(url).await?; + let content_type = response + .headers() + .get(CONTENT_TYPE) + .map(HeaderValue::to_str) + .transpose()?; + + if content_type + != Some("application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") + || content_type != Some("application/activity+json") + { + return Err(ApiError::BadRequest.into()); + } + + Ok(response.jsonld().await?) + } + /// Fetch an ActivityPub actor /// /// # Panics @@ -127,7 +152,7 @@ impl Fetcher { return Err(ApiError::Unauthorised.into()); } - let mut actor: Actor = self.client.get(url.as_str()).await?.jsonld().await?; + let mut actor: Actor = self.fetch_ap_resource(url.as_str()).await?; let mut domain = url.host_str().unwrap(); let domain_buf; @@ -292,7 +317,7 @@ impl Fetcher { } let url = Url::parse(url)?; - let object: Object = self.client.get(url.as_str()).await?.jsonld().await?; + let object: Object = self.fetch_ap_resource(url.as_str()).await?; let process_data = ProcessNewObject::builder() .call_depth(call_depth) diff --git a/crates/kitsune-core/src/error.rs b/crates/kitsune-core/src/error.rs index 7ee9b24e4..191060889 100644 --- a/crates/kitsune-core/src/error.rs +++ b/crates/kitsune-core/src/error.rs @@ -98,6 +98,9 @@ pub enum Error { #[error(transparent)] HttpClient(#[from] kitsune_http_client::Error), + #[error(transparent)] + HttpHeaderToStr(#[from] http::header::ToStrError), + #[error(transparent)] JobQueue(#[from] athena::Error), From 7d5147420609c7c5ab117c23f9b6533215478ea8 Mon Sep 17 00:00:00 2001 From: aumetra Date: Wed, 25 Oct 2023 22:27:10 +0200 Subject: [PATCH 02/16] remove once_cell --- Cargo.lock | 1 - crates/kitsune-core/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c345039fd..9a3774819 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2925,7 +2925,6 @@ dependencies = [ "kitsune-type", "mime", "mime_guess", - "once_cell", "password-hash", "pkcs8", "post-process", diff --git a/crates/kitsune-core/Cargo.toml b/crates/kitsune-core/Cargo.toml index 2614cd6a7..8ab05908a 100644 --- a/crates/kitsune-core/Cargo.toml +++ b/crates/kitsune-core/Cargo.toml @@ -50,7 +50,6 @@ kitsune-storage = { path = "../kitsune-storage" } kitsune-type = { path = "../kitsune-type" } mime = "0.3.17" mime_guess = { version = "2.0.4", default-features = false } -once_cell = "1.18.0" password-hash = { version = "0.5.0", features = ["std"] } pkcs8 = { version = "0.10.2", features = ["std"] } post-process = { path = "../../lib/post-process" } From 228ca912e49a691f693ae3d0a118eba5fdb9cb21 Mon Sep 17 00:00:00 2001 From: aumetra Date: Wed, 25 Oct 2023 22:32:13 +0200 Subject: [PATCH 03/16] add activitypub header to tests --- crates/kitsune-core/src/activitypub/fetcher.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index 3f6fd649f..1822643c9 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -853,22 +853,31 @@ mod test { } async fn handle(req: Request) -> Result, Infallible> { + macro_rules! build_ap_response { + ($body:expr) => { + Response::builder() + .header("Content-Type", "application/activity+json") + .body(Body::from($body)) + .unwrap() + }; + } + match req.uri().path_and_query().unwrap().as_str() { "/users/0x0" => { let body = include_str!("../../../../test-fixtures/0x0_actor.json"); - Ok::<_, Infallible>(Response::new(Body::from(body))) + Ok::<_, Infallible>(build_ap_response!(body)) } "/@0x0/109501674056556919" => { let body = include_str!( "../../../../test-fixtures/corteximplant.com_109501674056556919.json" ); - Ok::<_, Infallible>(Response::new(Body::from(body))) + Ok::<_, Infallible>(build_ap_response!(body)) } "/users/0x0/statuses/109501659207519785" => { let body = include_str!( "../../../../test-fixtures/corteximplant.com_109501659207519785.json" ); - Ok::<_, Infallible>(Response::new(Body::from(body))) + Ok::<_, Infallible>(build_ap_response!(body)) } "/.well-known/webfinger?resource=acct:0x0@corteximplant.com" => { let body = include_str!("../../../../test-fixtures/0x0_jrd.json"); From f8fcc5abd5be59859b2d730b44b35f06e0d83de5 Mon Sep 17 00:00:00 2001 From: aumetra Date: Wed, 25 Oct 2023 22:35:40 +0200 Subject: [PATCH 04/16] add test --- .../kitsune-core/src/activitypub/fetcher.rs | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index 1822643c9..0aca99b44 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -752,6 +752,39 @@ mod test { .await; } + #[tokio::test] + #[serial_test::serial] + async fn check_ap_content_type() { + database_test(|db_pool| async move { + let client = service_fn(|_: Request<_>| async { + Ok::<_, Infallible>(Response::new(Body::empty())) + }); + let client = Client::builder().service(client); + + let fetcher = Fetcher::builder() + .client(client.clone()) + .db_pool(db_pool) + .embed_client(None) + .federation_filter( + FederationFilterService::new(&FederationFilterConfiguration::Deny { + domains: Vec::new(), + }) + .unwrap(), + ) + .search_service(NoopSearchService) + .webfinger(Webfinger::with_client(client, Arc::new(NoopCache.into()))) + .post_cache(Arc::new(NoopCache.into())) + .user_cache(Arc::new(NoopCache.into())) + .build(); + + assert!(matches!( + fetcher.fetch_object("https://example.com/fakeobject").await, + Err(Error::Api(ApiError::BadRequest)) + )); + }) + .await; + } + #[tokio::test] #[serial_test::serial] async fn federation_allow() { From 0a6628c1f128a72ee7bd2514a31e76a83be68484 Mon Sep 17 00:00:00 2001 From: aumetra Date: Wed, 25 Oct 2023 22:36:21 +0200 Subject: [PATCH 05/16] fix boolean logic --- crates/kitsune-core/src/activitypub/fetcher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index 0aca99b44..4bab37549 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -105,7 +105,7 @@ impl Fetcher { if content_type != Some("application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") - || content_type != Some("application/activity+json") + && content_type != Some("application/activity+json") { return Err(ApiError::BadRequest.into()); } From 65e489dcbf289d8ec1d423ef9a0f904c9b8595c4 Mon Sep 17 00:00:00 2001 From: aumetra Date: Wed, 25 Oct 2023 22:44:10 +0200 Subject: [PATCH 06/16] fix test --- Cargo.lock | 39 ++++++++++--------- .../kitsune-core/src/activitypub/fetcher.rs | 19 ++++----- crates/kitsune-core/src/resolve/post.rs | 4 +- crates/kitsune-test/Cargo.toml | 1 + crates/kitsune-test/src/lib.rs | 13 +++++++ 5 files changed, 43 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a3774819..fbf52112c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -40,9 +40,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7d5a2cecb58716e47d67d5703a249964b14c7be1ec3cad3affc295b2d1c35d" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if", "getrandom", @@ -437,7 +437,7 @@ dependencies = [ name = "athena" version = "0.0.1-pre.4" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.6", "async-trait", "deadpool-redis", "either", @@ -819,9 +819,9 @@ checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "bytecount" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56f99948f5dd725fb7e351c80765ae80044f243169e141188a874105da83e9e" +checksum = "e1e5f035d16fc623ae5f74981db80a439803888314e3a555fd6f04acd51a3205" [[package]] name = "byteorder" @@ -2281,7 +2281,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ff8ae62cd3a9102e5637afc8452c55acf3844001bd5374e0b0bd7b6616c038" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.6", ] [[package]] @@ -2290,7 +2290,7 @@ version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.6", "allocator-api2", ] @@ -3077,7 +3077,7 @@ dependencies = [ name = "kitsune-messaging" version = "0.0.1-pre.4" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.6", "async-trait", "futures-util", "kitsune-retry-policies", @@ -3165,6 +3165,7 @@ dependencies = [ "diesel", "diesel-async", "futures-util", + "http", "kitsune-db", "pin-project-lite", "redis", @@ -3195,9 +3196,9 @@ dependencies = [ [[package]] name = "lettre" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d47084ad58f99c26816d174702f60e873f861fcef3f9bd6075b4ad2dd72d07d5" +checksum = "a466bc111374ccf4d90877dba636924a2185e67e5be4b35d32043199365097b2" dependencies = [ "async-trait", "base64 0.21.5", @@ -3535,7 +3536,7 @@ version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fde3af1a009ed76a778cb84fdef9e7dbbdf5775ae3e4cc1f434a6a307f6f76c5" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.6", "metrics-macros", "portable-atomic", ] @@ -4736,7 +4737,7 @@ name = "redis" version = "0.23.3" source = "git+https://github.com/aumetra/redis-rs.git?rev=3c4ee09d432a69e1d87d66dcba14c519467c9b81#3c4ee09d432a69e1d87d66dcba14c519467c9b81" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.6", "arc-swap", "async-trait", "bytes", @@ -5121,7 +5122,7 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c95a930e03325234c18c7071fd2b60118307e025d6fff3e12745ffbf63a3d29c" dependencies = [ - "ahash 0.8.5", + "ahash 0.8.6", "cssparser", "ego-tree", "html5ever", @@ -5976,9 +5977,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" dependencies = [ "bytes", "futures-core", @@ -6846,18 +6847,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.12" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8db0ac2df3d060f81ec0380ccc5b71c2a7c092cfced671feeee1320e95559c87" +checksum = "81ba595b9f2772fbee2312de30eeb80ec773b4cb2f1e8098db024afadda6c06f" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.12" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b6093bc6d5265ff40b479c834cdd25d8e20784781a2a29a8106327393d0a9ff" +checksum = "772666c41fb6dceaf520b564b962d738a8e1a83b41bd48945f50837aed78bb1d" dependencies = [ "proc-macro2", "quote", diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index 4bab37549..fd99e7dae 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -363,7 +363,7 @@ mod test { use kitsune_db::{model::account::Account, schema::accounts}; use kitsune_http_client::Client; use kitsune_search::NoopSearchService; - use kitsune_test::database_test; + use kitsune_test::{build_ap_response, database_test}; use kitsune_type::{ ap::{ actor::{Actor, ActorType, PublicKey}, @@ -612,6 +612,7 @@ mod test { let client = service_fn(move |req: Request<_>| { let count = request_counter.fetch_add(1, Ordering::SeqCst); assert!(MAX_FETCH_DEPTH * 3 >= count); + async move { let author_id = "https://example.com/users/1".to_owned(); let author = Actor { @@ -658,11 +659,14 @@ mod test { to: vec![PUBLIC_IDENTIFIER.into()], cc: Vec::new(), }; + let body = simd_json::to_string(¬e).unwrap(); - Ok::<_, Infallible>(Response::new(Body::from(body))) + + Ok::<_, Infallible>(build_ap_response!(body)) } else if req.uri().path_and_query().unwrap() == Uri::try_from(&author.id).unwrap().path_and_query().unwrap() { let body = simd_json::to_string(&author).unwrap(); - Ok::<_, Infallible>(Response::new(Body::from(body))) + + Ok::<_, Infallible>(build_ap_response!(body)) } else { handle(req).await } @@ -886,15 +890,6 @@ mod test { } async fn handle(req: Request) -> Result, Infallible> { - macro_rules! build_ap_response { - ($body:expr) => { - Response::builder() - .header("Content-Type", "application/activity+json") - .body(Body::from($body)) - .unwrap() - }; - } - match req.uri().path_and_query().unwrap().as_str() { "/users/0x0" => { let body = include_str!("../../../../test-fixtures/0x0_actor.json"); diff --git a/crates/kitsune-core/src/resolve/post.rs b/crates/kitsune-core/src/resolve/post.rs index 0978b830a..7736fae79 100644 --- a/crates/kitsune-core/src/resolve/post.rs +++ b/crates/kitsune-core/src/resolve/post.rs @@ -96,7 +96,7 @@ mod test { use kitsune_http_client::Client; use kitsune_search::NoopSearchService; use kitsune_storage::fs::Storage as FsStorage; - use kitsune_test::{database_test, redis_test}; + use kitsune_test::{build_ap_response, database_test, redis_test}; use pretty_assertions::assert_eq; use scoped_futures::ScopedFutureExt; use std::sync::Arc; @@ -117,7 +117,7 @@ mod test { } "/users/0x0" => { let body = include_str!("../../../../test-fixtures/0x0_actor.json"); - Ok::<_, Infallible>(Response::new(Body::from(body))) + Ok::<_, Infallible>(build_ap_response!(body)) } path => panic!("HTTP client hit unexpected route: {path}"), } diff --git a/crates/kitsune-test/Cargo.toml b/crates/kitsune-test/Cargo.toml index 1a32e5d36..8e91d35bc 100644 --- a/crates/kitsune-test/Cargo.toml +++ b/crates/kitsune-test/Cargo.toml @@ -8,6 +8,7 @@ deadpool-redis = "0.13.0" diesel = "2.1.3" diesel-async = "0.4.1" futures-util = "0.3.28" +http = "0.2.9" kitsune-db = { path = "../kitsune-db" } pin-project-lite = "0.2.13" redis = "0.23.3" diff --git a/crates/kitsune-test/src/lib.rs b/crates/kitsune-test/src/lib.rs index 8a270ef8f..11148a197 100644 --- a/crates/kitsune-test/src/lib.rs +++ b/crates/kitsune-test/src/lib.rs @@ -11,8 +11,21 @@ use std::{env, error::Error, panic}; mod catch_panic; +#[doc(hidden)] +pub use http; + type BoxError = Box; +#[macro_export] +macro_rules! build_ap_response { + ($body:expr) => { + $crate::http::Response::builder() + .header("Content-Type", "application/activity+json") + .body(Body::from($body)) + .unwrap() + }; +} + pub async fn database_test(func: F) -> Fut::Output where F: FnOnce(PgPool) -> Fut, From 720872403d6bbdbd531782703902f3bf970319e7 Mon Sep 17 00:00:00 2001 From: aumetra Date: Wed, 25 Oct 2023 23:16:13 +0200 Subject: [PATCH 07/16] i still have no idea why this keeps happening --- Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 0682428f9..c1b103412 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,8 @@ +[profile.dev.package.backtrace] +opt-level = 3 + +[profile.dev.package.num-bigint-dig] +opt-level = 3 [profile.release] codegen-units = 1 From ea49c4e71c10eb53bcb022887ea75b832964a758 Mon Sep 17 00:00:00 2001 From: aumetra Date: Thu, 26 Oct 2023 09:00:30 +0200 Subject: [PATCH 08/16] add hyper to test crate --- Cargo.lock | 13 +++++++------ Cargo.toml | 5 ----- crates/kitsune-test/Cargo.toml | 1 + crates/kitsune-test/src/lib.rs | 4 +++- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fbf52112c..f6b4c189f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -685,9 +685,9 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "basic-toml" -version = "0.1.4" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bfc506e7a2370ec239e1d072507b2a80c833083699d3c6fa176fbb4de8448c6" +checksum = "24c12265665aebaa236af9bbe266681bcc9c5666192119e3d8335cf083aca26f" dependencies = [ "serde", ] @@ -3166,6 +3166,7 @@ dependencies = [ "diesel-async", "futures-util", "http", + "hyper", "kitsune-db", "pin-project-lite", "redis", @@ -5209,9 +5210,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.189" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e422a44e74ad4001bdc8eede9a4570ab52f71190e9c076d14369f38b9200537" +checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" dependencies = [ "serde_derive", ] @@ -5228,9 +5229,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.189" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e48d1f918009ce3145511378cf68d613e3b3d9137d67272562080d68a2b32d5" +checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index c1b103412..0682428f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,8 +1,3 @@ -[profile.dev.package.backtrace] -opt-level = 3 - -[profile.dev.package.num-bigint-dig] -opt-level = 3 [profile.release] codegen-units = 1 diff --git a/crates/kitsune-test/Cargo.toml b/crates/kitsune-test/Cargo.toml index 8e91d35bc..188f9f9b1 100644 --- a/crates/kitsune-test/Cargo.toml +++ b/crates/kitsune-test/Cargo.toml @@ -9,6 +9,7 @@ diesel = "2.1.3" diesel-async = "0.4.1" futures-util = "0.3.28" http = "0.2.9" +hyper = "0.14.27" kitsune-db = { path = "../kitsune-db" } pin-project-lite = "0.2.13" redis = "0.23.3" diff --git a/crates/kitsune-test/src/lib.rs b/crates/kitsune-test/src/lib.rs index 11148a197..c3ab387f9 100644 --- a/crates/kitsune-test/src/lib.rs +++ b/crates/kitsune-test/src/lib.rs @@ -13,6 +13,8 @@ mod catch_panic; #[doc(hidden)] pub use http; +#[doc(hidden)] +pub use hyper; type BoxError = Box; @@ -21,7 +23,7 @@ macro_rules! build_ap_response { ($body:expr) => { $crate::http::Response::builder() .header("Content-Type", "application/activity+json") - .body(Body::from($body)) + .body($crate::hyper::Body::from($body)) .unwrap() }; } From 614f4269352003b40cc45d0a3166fcce34aa0a7f Mon Sep 17 00:00:00 2001 From: aumetra Date: Thu, 26 Oct 2023 09:49:13 +0200 Subject: [PATCH 09/16] i swear.. --- Cargo.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 0682428f9..c1b103412 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,3 +1,8 @@ +[profile.dev.package.backtrace] +opt-level = 3 + +[profile.dev.package.num-bigint-dig] +opt-level = 3 [profile.release] codegen-units = 1 From 566fb5554833108ef00238e9b013e1d92fb8b58e Mon Sep 17 00:00:00 2001 From: aumetra Date: Thu, 26 Oct 2023 12:13:47 +0200 Subject: [PATCH 10/16] replace macro with fn --- .../kitsune-core/src/activitypub/fetcher.rs | 10 ++++----- crates/kitsune-core/src/resolve/post.rs | 2 +- crates/kitsune-test/src/lib.rs | 22 ++++++++----------- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index fd99e7dae..36b2d812b 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -662,11 +662,11 @@ mod test { let body = simd_json::to_string(¬e).unwrap(); - Ok::<_, Infallible>(build_ap_response!(body)) + Ok::<_, Infallible>(build_ap_response(body)) } else if req.uri().path_and_query().unwrap() == Uri::try_from(&author.id).unwrap().path_and_query().unwrap() { let body = simd_json::to_string(&author).unwrap(); - Ok::<_, Infallible>(build_ap_response!(body)) + Ok::<_, Infallible>(build_ap_response(body)) } else { handle(req).await } @@ -893,19 +893,19 @@ mod test { match req.uri().path_and_query().unwrap().as_str() { "/users/0x0" => { let body = include_str!("../../../../test-fixtures/0x0_actor.json"); - Ok::<_, Infallible>(build_ap_response!(body)) + Ok::<_, Infallible>(build_ap_response(body)) } "/@0x0/109501674056556919" => { let body = include_str!( "../../../../test-fixtures/corteximplant.com_109501674056556919.json" ); - Ok::<_, Infallible>(build_ap_response!(body)) + Ok::<_, Infallible>(build_ap_response(body)) } "/users/0x0/statuses/109501659207519785" => { let body = include_str!( "../../../../test-fixtures/corteximplant.com_109501659207519785.json" ); - Ok::<_, Infallible>(build_ap_response!(body)) + Ok::<_, Infallible>(build_ap_response(body)) } "/.well-known/webfinger?resource=acct:0x0@corteximplant.com" => { let body = include_str!("../../../../test-fixtures/0x0_jrd.json"); diff --git a/crates/kitsune-core/src/resolve/post.rs b/crates/kitsune-core/src/resolve/post.rs index 7736fae79..6f2b7f032 100644 --- a/crates/kitsune-core/src/resolve/post.rs +++ b/crates/kitsune-core/src/resolve/post.rs @@ -117,7 +117,7 @@ mod test { } "/users/0x0" => { let body = include_str!("../../../../test-fixtures/0x0_actor.json"); - Ok::<_, Infallible>(build_ap_response!(body)) + Ok::<_, Infallible>(build_ap_response(body)) } path => panic!("HTTP client hit unexpected route: {path}"), } diff --git a/crates/kitsune-test/src/lib.rs b/crates/kitsune-test/src/lib.rs index c3ab387f9..184917e37 100644 --- a/crates/kitsune-test/src/lib.rs +++ b/crates/kitsune-test/src/lib.rs @@ -5,27 +5,23 @@ use self::catch_panic::CatchPanic; use diesel_async::RunQueryDsl; use futures_util::Future; +use http::header::CONTENT_TYPE; use kitsune_db::PgPool; use scoped_futures::ScopedFutureExt; use std::{env, error::Error, panic}; mod catch_panic; -#[doc(hidden)] -pub use http; -#[doc(hidden)] -pub use hyper; - type BoxError = Box; -#[macro_export] -macro_rules! build_ap_response { - ($body:expr) => { - $crate::http::Response::builder() - .header("Content-Type", "application/activity+json") - .body($crate::hyper::Body::from($body)) - .unwrap() - }; +pub fn build_ap_response(body: B) -> http::Response +where + hyper::Body: From, +{ + http::Response::builder() + .header(CONTENT_TYPE, "application/activity+json") + .body(body.into()) + .unwrap() } pub async fn database_test(func: F) -> Fut::Output From 0f0bb29e6b5a106cdcecba02f908c759821b8a0a Mon Sep 17 00:00:00 2001 From: aumetra Date: Thu, 26 Oct 2023 12:22:35 +0200 Subject: [PATCH 11/16] add byte literal comparison --- crates/kitsune-core/src/activitypub/fetcher.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index 36b2d812b..bca6be948 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -100,12 +100,11 @@ impl Fetcher { let content_type = response .headers() .get(CONTENT_TYPE) - .map(HeaderValue::to_str) - .transpose()?; + .map(HeaderValue::as_bytes); if content_type - != Some("application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") - && content_type != Some("application/activity+json") + != Some(b"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") + && content_type != Some(b"application/activity+json") { return Err(ApiError::BadRequest.into()); } From 98a52cb8cc99d328253dbb2a246a793c2ae415d4 Mon Sep 17 00:00:00 2001 From: aumetra Date: Thu, 26 Oct 2023 12:30:06 +0200 Subject: [PATCH 12/16] update test case --- crates/kitsune-core/src/activitypub/fetcher.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index bca6be948..33932daea 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -354,7 +354,7 @@ mod test { }; use diesel::{QueryDsl, SelectableHelper}; use diesel_async::RunQueryDsl; - use http::uri::PathAndQuery; + use http::{header::CONTENT_TYPE, uri::PathAndQuery}; use hyper::{Body, Request, Response, StatusCode, Uri}; use iso8601_timestamp::Timestamp; use kitsune_cache::NoopCache; @@ -759,8 +759,10 @@ mod test { #[serial_test::serial] async fn check_ap_content_type() { database_test(|db_pool| async move { - let client = service_fn(|_: Request<_>| async { - Ok::<_, Infallible>(Response::new(Body::empty())) + let client = service_fn(|req: Request<_>| async { + let mut res = handle(req).await.unwrap(); + res.headers_mut().remove(CONTENT_TYPE); + Ok::<_, Infallible>(res) }); let client = Client::builder().service(client); From e95aa5507c3df319ea4e5cfe244f3872d251be70 Mon Sep 17 00:00:00 2001 From: aumetra Date: Thu, 26 Oct 2023 12:32:50 +0200 Subject: [PATCH 13/16] fix url --- crates/kitsune-core/src/activitypub/fetcher.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index 33932daea..0e84b8c6f 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -783,7 +783,9 @@ mod test { .build(); assert!(matches!( - fetcher.fetch_object("https://example.com/fakeobject").await, + fetcher + .fetch_object("https://corteximplant.com/users/0x0") + .await, Err(Error::Api(ApiError::BadRequest)) )); }) From 7325b7cde404defd1bbb9fae2590b1fa425684f7 Mon Sep 17 00:00:00 2001 From: aumetra Date: Fri, 27 Oct 2023 19:53:42 +0200 Subject: [PATCH 14/16] add better content-type checks --- Cargo.lock | 1 + crates/kitsune-core/Cargo.toml | 1 + .../kitsune-core/src/activitypub/fetcher.rs | 33 ++++++++++++++----- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f6b4c189f..59c412f4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2903,6 +2903,7 @@ dependencies = [ "futures-util", "garde", "globset", + "headers", "hex-simd", "http", "hyper", diff --git a/crates/kitsune-core/Cargo.toml b/crates/kitsune-core/Cargo.toml index 8ab05908a..c147b4863 100644 --- a/crates/kitsune-core/Cargo.toml +++ b/crates/kitsune-core/Cargo.toml @@ -30,6 +30,7 @@ garde = { version = "0.16.0", default-features = false, features = [ "serde", ] } globset = "0.4.13" +headers = "0.3.9" hex-simd = "0.8.0" http = "0.2.9" img-parts = "0.3.0" diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index 0e84b8c6f..c35094bec 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -12,7 +12,8 @@ use async_recursion::async_recursion; use autometrics::autometrics; use diesel::{ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper}; use diesel_async::RunQueryDsl; -use http::{header::CONTENT_TYPE, HeaderValue}; +use headers::{ContentType, HeaderMapExt}; +use http::HeaderValue; use kitsune_cache::{ArcCache, CacheBackend}; use kitsune_db::{ model::{ @@ -29,6 +30,7 @@ use kitsune_type::{ ap::{actor::Actor, Object}, jsonld::RdfNode, }; +use mime::Mime; use scoped_futures::ScopedFutureExt; use serde::de::DeserializeOwned; use typed_builder::TypedBuilder; @@ -97,15 +99,30 @@ impl Fetcher { T: DeserializeOwned + RdfNode, { let response = self.client.get(url).await?; - let content_type = response + let Some(content_type) = response .headers() - .get(CONTENT_TYPE) - .map(HeaderValue::as_bytes); + .typed_get::() + .map(Mime::from) + else { + return Err(ApiError::BadRequest.into()); + }; - if content_type - != Some(b"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"") - && content_type != Some(b"application/activity+json") - { + let is_json_ld_activitystreams = content_type + .essence_str() + .eq_ignore_ascii_case("application/ld+json") + && content_type + .get_param("profile") + .map_or(false, |profile_name| { + profile_name + .as_str() + .eq_ignore_ascii_case("https://www.w3.org/ns/activitystreams") + }); + + let is_activity_json = content_type + .essence_str() + .eq_ignore_ascii_case("application/activity+json"); + + if !is_json_ld_activitystreams && !is_activity_json { return Err(ApiError::BadRequest.into()); } From 351d357aecb5cf8bcbba9cdfe7f1954fba22abc0 Mon Sep 17 00:00:00 2001 From: aumetra Date: Fri, 27 Oct 2023 20:48:15 +0200 Subject: [PATCH 15/16] make computations lazy --- .../kitsune-core/src/activitypub/fetcher.rs | 36 ++++++++++--------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index c35094bec..c7d046c38 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -107,22 +107,26 @@ impl Fetcher { return Err(ApiError::BadRequest.into()); }; - let is_json_ld_activitystreams = content_type - .essence_str() - .eq_ignore_ascii_case("application/ld+json") - && content_type - .get_param("profile") - .map_or(false, |profile_name| { - profile_name - .as_str() - .eq_ignore_ascii_case("https://www.w3.org/ns/activitystreams") - }); - - let is_activity_json = content_type - .essence_str() - .eq_ignore_ascii_case("application/activity+json"); - - if !is_json_ld_activitystreams && !is_activity_json { + let is_json_ld_activitystreams = || { + content_type + .essence_str() + .eq_ignore_ascii_case("application/ld+json") + && content_type + .get_param("profile") + .map_or(false, |profile_name| { + profile_name + .as_str() + .eq_ignore_ascii_case("https://www.w3.org/ns/activitystreams") + }) + }; + + let is_activity_json = || { + content_type + .essence_str() + .eq_ignore_ascii_case("application/activity+json") + }; + + if !is_json_ld_activitystreams() && !is_activity_json() { return Err(ApiError::BadRequest.into()); } From c88f75b76d556357bf96ae3bd9f0f8b03b3d504b Mon Sep 17 00:00:00 2001 From: aumetra Date: Sat, 28 Oct 2023 13:51:18 +0200 Subject: [PATCH 16/16] add support for multiple profile values --- crates/kitsune-core/src/activitypub/fetcher.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/kitsune-core/src/activitypub/fetcher.rs b/crates/kitsune-core/src/activitypub/fetcher.rs index c7d046c38..a7e9121e0 100644 --- a/crates/kitsune-core/src/activitypub/fetcher.rs +++ b/crates/kitsune-core/src/activitypub/fetcher.rs @@ -113,10 +113,11 @@ impl Fetcher { .eq_ignore_ascii_case("application/ld+json") && content_type .get_param("profile") - .map_or(false, |profile_name| { - profile_name + .map_or(false, |profile_urls| { + profile_urls .as_str() - .eq_ignore_ascii_case("https://www.w3.org/ns/activitystreams") + .split_whitespace() + .any(|url| url == "https://www.w3.org/ns/activitystreams") }) };