share http client throughout kitsune (where possible and useful) to m…

…aximize pooling effectiveness
kitsune-soc · Nov 13, 2024 · 857fd3b · 857fd3b
1 parent 2a3d7eb
commit 857fd3b
Show file tree

Hide file tree

Showing 27 changed files with 124 additions and 102 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/kitsune-activitypub/src/deliverer/core.rs b/crates/kitsune-activitypub/src/deliverer/core.rs
@@ -1,6 +1,5 @@
 use futures_util::{stream::FuturesUnordered, Stream, StreamExt};
 use http::{Method, Request};
-use kitsune_core::consts::USER_AGENT;
 use kitsune_db::model::{account::Account, user::User};
 use kitsune_error::{Error, Result};
 use kitsune_federation_filter::FederationFilter;
@@ -17,8 +16,7 @@ use url::Url;
 /// Does not need to be Arc wrapped for cheap cloning. It's inherently cheap to clone.
 #[derive(Clone, TypedBuilder)]
 pub struct Deliverer {
-    #[builder(default = Client::builder().user_agent(USER_AGENT).unwrap().build())]
-    client: Client,
+    http_client: Client,
     federation_filter: FederationFilter,
     mrf_service: MrfService,
 }
@@ -55,7 +53,7 @@ impl Deliverer {
             .body(body.into())?;
 
         let response = self
-            .client
+            .http_client
             .execute_signed(request, &account.public_key_id, &user.private_key)
             .await?;
 

diff --git a/crates/kitsune-activitypub/src/fetcher/mod.rs b/crates/kitsune-activitypub/src/fetcher/mod.rs
@@ -1,15 +1,12 @@
 use async_trait::async_trait;
 use headers::{ContentType, HeaderMapExt};
-use http::HeaderValue;
+use http::{header::ACCEPT, HeaderValue, Request};
 use kitsune_cache::ArcCache;
 use kitsune_config::language_detection::Configuration as LanguageDetectionConfig;
-use kitsune_core::{
-    consts::USER_AGENT,
-    traits::{
-        coerce::CoerceResolver,
-        fetcher::{AccountFetchOptions, PostFetchOptions},
-        Fetcher as FetcherTrait, Resolver,
-    },
+use kitsune_core::traits::{
+    coerce::CoerceResolver,
+    fetcher::{AccountFetchOptions, PostFetchOptions},
+    Fetcher as FetcherTrait, Resolver,
 };
 use kitsune_db::{
     model::{account::Account, custom_emoji::CustomEmoji, post::Post},
@@ -32,23 +29,14 @@ mod actor;
 mod emoji;
 mod object;
 
+static ACCEPT_VALUE: HeaderValue = HeaderValue::from_static(
+    "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\", application/activity+json",
+);
+
 #[derive(TypedBuilder)]
 #[builder(build_method(into = Arc<Fetcher>))]
 pub struct Fetcher {
-    #[builder(default =
-        Client::builder()
-            .default_header(
-                "accept",
-                HeaderValue::from_static(
-                    "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\", application/activity+json",
-                ),
-            )
-            .unwrap()
-            .user_agent(USER_AGENT)
-            .unwrap()
-            .build()
-    )]
-    client: Client,
+    http_client: Client,
     db_pool: PgPool,
     embed_client: Option<EmbedClient>,
     federation_filter: FederationFilter,
@@ -74,8 +62,13 @@ impl Fetcher {
             bail!("instance is blocked");
         }
 
-        let response = self.client.get(url.as_str()).await?;
+        let request = Request::builder()
+            .method(http::Method::GET)
+            .uri(url.as_str())
+            .header(ACCEPT, &ACCEPT_VALUE)
+            .body(kitsune_http_client::Body::empty())?;
 
+        let response = self.http_client.execute(request).await?;
         if !response.status().is_success() {
             return Ok(None);
         }

diff --git a/crates/kitsune-activitypub/tests/fetcher/basic.rs b/crates/kitsune-activitypub/tests/fetcher/basic.rs
@@ -25,7 +25,7 @@ async fn fetch_actor() {
         let client = Client::builder().service(service_fn(handle));
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool)
             .embed_client(None)
             .federation_filter(
@@ -36,7 +36,7 @@ async fn fetch_actor() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();
@@ -64,7 +64,7 @@ async fn fetch_emoji() {
         let client = Client::builder().service(service_fn(handle));
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool.clone())
             .embed_client(None)
             .federation_filter(
@@ -75,7 +75,7 @@ async fn fetch_emoji() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();
@@ -115,7 +115,7 @@ async fn fetch_note() {
         let client = Client::builder().service(service_fn(handle));
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool.clone())
             .embed_client(None)
             .federation_filter(
@@ -126,7 +126,7 @@ async fn fetch_note() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();

diff --git a/crates/kitsune-activitypub/tests/fetcher/filter.rs b/crates/kitsune-activitypub/tests/fetcher/filter.rs
@@ -45,9 +45,9 @@ async fn federation_allow() {
         let client = Client::builder().service(client);
         let fetcher = builder
             .clone()
-            .client(client.clone())
+            .http_client(client.clone())
             .language_detection_config(language_detection_config())
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .build();
 
         assert_blocked!(fetcher
@@ -63,9 +63,9 @@ async fn federation_allow() {
         let client = Client::builder().service(service_fn(handle));
         let fetcher = builder
             .clone()
-            .client(client.clone())
+            .http_client(client.clone())
             .language_detection_config(language_detection_config())
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .build();
 
         assert!(matches!(
@@ -90,7 +90,7 @@ async fn federation_deny() {
         let client = Client::builder().service(client);
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool)
             .embed_client(None)
             .federation_filter(
@@ -101,7 +101,7 @@ async fn federation_deny() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();

diff --git a/crates/kitsune-activitypub/tests/fetcher/infinite.rs b/crates/kitsune-activitypub/tests/fetcher/infinite.rs
@@ -91,7 +91,7 @@ async fn fetch_infinitely_long_reply_chain() {
             let client = Client::builder().service(client);
 
             let fetcher = Fetcher::builder()
-                .client(client.clone())
+                .http_client(client.clone())
                 .db_pool(db_pool)
                 .embed_client(None)
                 .federation_filter(
@@ -102,7 +102,7 @@ async fn fetch_infinitely_long_reply_chain() {
                 )
                 .language_detection_config(language_detection_config())
                 .search_backend(NoopSearchService)
-                .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+                .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
                 .account_cache(Arc::new(NoopCache.into()))
                 .post_cache(Arc::new(NoopCache.into()))
                 .build();

diff --git a/crates/kitsune-activitypub/tests/fetcher/origin.rs b/crates/kitsune-activitypub/tests/fetcher/origin.rs
@@ -37,9 +37,9 @@ async fn check_ap_id_authority() {
         let client = Client::builder().service(client);
         let fetcher = builder
             .clone()
-            .client(client.clone())
+            .http_client(client.clone())
             .language_detection_config(language_detection_config())
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .build();
 
         // The mock HTTP client ensures that the fetcher doesn't access the correct server
@@ -60,9 +60,9 @@ async fn check_ap_id_authority() {
         let client = Client::builder().service(client);
         let fetcher = builder
             .clone()
-            .client(client.clone())
+            .http_client(client.clone())
             .language_detection_config(language_detection_config())
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .build();
 
         let _ = fetcher
@@ -84,7 +84,7 @@ async fn check_ap_content_type() {
         let client = Client::builder().service(client);
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool)
             .embed_client(None)
             .federation_filter(
@@ -95,7 +95,7 @@ async fn check_ap_content_type() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();

diff --git a/crates/kitsune-activitypub/tests/fetcher/webfinger.rs b/crates/kitsune-activitypub/tests/fetcher/webfinger.rs
@@ -47,7 +47,7 @@ async fn fetch_actor_with_custom_acct() {
         let client = Client::builder().service(client);
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool)
             .embed_client(None)
             .federation_filter(
@@ -58,7 +58,7 @@ async fn fetch_actor_with_custom_acct() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();
@@ -125,7 +125,7 @@ async fn ignore_fake_webfinger_acct() {
         let client = Client::builder().service(client);
 
         let fetcher = Fetcher::builder()
-            .client(client.clone())
+            .http_client(client.clone())
             .db_pool(db_pool)
             .embed_client(None)
             .federation_filter(
@@ -136,7 +136,7 @@ async fn ignore_fake_webfinger_acct() {
             )
             .language_detection_config(language_detection_config())
             .search_backend(NoopSearchService)
-            .resolver(Arc::new(Webfinger::with_client(client, Arc::new(NoopCache.into()))).coerce())
+            .resolver(Arc::new(Webfinger::new(client, Arc::new(NoopCache.into()))).coerce())
             .account_cache(Arc::new(NoopCache.into()))
             .post_cache(Arc::new(NoopCache.into()))
             .build();

diff --git a/crates/kitsune-captcha/src/hcaptcha.rs b/crates/kitsune-captcha/src/hcaptcha.rs
@@ -10,8 +10,7 @@ pub struct Captcha {
     pub site_key: String,
     pub secret_key: String,
 
-    #[builder(default)]
-    client: Client,
+    http_client: Client,
 }
 
 #[derive(Serialize, Deserialize, TypedBuilder)]
@@ -46,7 +45,7 @@ impl CaptchaBackend for Captcha {
             .header("Accept", "application/json")
             .body(body.into())?;
 
-        let response = self.client.execute(request).await?;
+        let response = self.http_client.execute(request).await?;
 
         let verification_result = response.json::<HCaptchaResponse>().await?;
         if !verification_result.success {

diff --git a/crates/kitsune-captcha/src/mcaptcha.rs b/crates/kitsune-captcha/src/mcaptcha.rs
@@ -11,8 +11,7 @@ pub struct Captcha {
     pub secret_key: String,
     pub site_key: String,
 
-    #[builder(default)]
-    client: Client,
+    http_client: Client,
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -41,7 +40,7 @@ impl CaptchaBackend for Captcha {
             .header("Accept", "application/json")
             .body(body.into())?;
 
-        let response = self.client.execute(request).await?;
+        let response = self.http_client.execute(request).await?;
 
         let verification_result = response.json::<MCaptchaResponse>().await?;
         if !verification_result.valid {

diff --git a/crates/kitsune-federation/Cargo.toml b/crates/kitsune-federation/Cargo.toml
@@ -13,6 +13,7 @@ kitsune-core = { workspace = true }
 kitsune-db = { workspace = true }
 kitsune-embed = { workspace = true }
 kitsune-federation-filter = { workspace = true }
+kitsune-http-client.workspace = true
 kitsune-search = { workspace = true }
 kitsune-service = { workspace = true }
 kitsune-url = { workspace = true }

diff --git a/crates/kitsune-federation/src/activitypub.rs b/crates/kitsune-federation/src/activitypub.rs
@@ -27,6 +27,7 @@ pub struct PrepareDeliverer {
     attachment_service: AttachmentService,
     db_pool: PgPool,
     federation_filter: FederationFilter,
+    http_client: kitsune_http_client::Client,
     mrf_service: MrfService,
     url_service: UrlService,
 }
@@ -38,6 +39,7 @@ pub struct PrepareFetcher {
     db_pool: PgPool,
     embed_client: Option<kitsune_embed::Client>,
     federation_filter: FederationFilter,
+    http_client: kitsune_http_client::Client,
     language_detection_config: LanguageDetectionConfig,
     post_cache: ArcCache<str, Post>,
     search_backend: AnySearchBackend,
@@ -47,6 +49,7 @@ pub struct PrepareFetcher {
 pub(crate) fn prepare_deliverer(prepare: PrepareDeliverer) -> Arc<dyn Deliverer> {
     let core_deliverer = kitsune_activitypub::CoreDeliverer::builder()
         .federation_filter(prepare.federation_filter)
+        .http_client(prepare.http_client)
         .mrf_service(prepare.mrf_service)
         .build();
 
@@ -67,13 +70,14 @@ pub(crate) fn prepare_deliverer(prepare: PrepareDeliverer) -> Arc<dyn Deliverer>
 
 #[inline]
 pub(crate) fn prepare_fetcher(prepare: PrepareFetcher) -> Arc<dyn Fetcher> {
-    let webfinger = Webfinger::new(prepare.account_resource_cache);
+    let webfinger = Webfinger::new(prepare.http_client.clone(), prepare.account_resource_cache);
 
     ActivityPubFetcher::builder()
         .account_cache(prepare.account_cache)
         .db_pool(prepare.db_pool.clone())
         .embed_client(prepare.embed_client)
         .federation_filter(prepare.federation_filter.clone())
+        .http_client(prepare.http_client)
         .language_detection_config(prepare.language_detection_config)
         .post_cache(prepare.post_cache)
         .resolver(Arc::new(webfinger).coerce())