From a4412dec29da785cbf317e08dbf7f249b71a93a3 Mon Sep 17 00:00:00 2001 From: aumetra Date: Sun, 15 Oct 2023 12:19:41 +0200 Subject: [PATCH] Replace `aws-sdk-s3` with `rusty-s3` (#373) --- Cargo.lock | 349 ++------------------------ crates/kitsune-core/Cargo.toml | 5 +- crates/kitsune-core/src/lib.rs | 43 ++-- crates/kitsune-storage/Cargo.toml | 11 +- crates/kitsune-storage/examples/s3.rs | 20 +- crates/kitsune-storage/src/s3.rs | 149 +++++------ 6 files changed, 136 insertions(+), 441 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac35e9927..d1a766774 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -419,9 +419,9 @@ checksum = "b9441c6b2fe128a7c2bf680a44c34d0df31ce09e5b7e401fcca3faa483dbc921" [[package]] name = "async-trait" -version = "0.1.73" +version = "0.1.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", @@ -499,306 +499,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "aws-credential-types" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70a66ac8ef5fa9cf01c2d999f39d16812e90ec1467bd382cbbb74ba23ea86201" -dependencies = [ - "aws-smithy-async", - "aws-smithy-types", - "fastrand 2.0.1", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-http" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e626370f9ba806ae4c439e49675fd871f5767b093075cdf4fef16cac42ba900" -dependencies = [ - "aws-credential-types", - "aws-smithy-http", - "aws-smithy-types", - "aws-types", - "bytes", - "http", - "http-body", - "lazy_static", - "percent-encoding", - "pin-project-lite", - "tracing", -] - -[[package]] -name = "aws-runtime" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ac5cf0ff19c1bca0cea7932e11b239d1025a45696a4f44f72ea86e2b8bdd07" -dependencies = [ - "aws-credential-types", - "aws-http", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-types", - "fastrand 2.0.1", - "http", - "percent-encoding", - "tracing", - "uuid", -] - -[[package]] -name = "aws-sdk-s3" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73018483d9cb78e1a0d4dcbc94327b01d532e7cb28f26c5bceff97f8f0e4c6eb" -dependencies = [ - "aws-credential-types", - "aws-http", - "aws-runtime", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-client", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "http", - "http-body", - "once_cell", - "percent-encoding", - "regex", - "tokio-stream", - "tracing", - "url", -] - -[[package]] -name = "aws-sigv4" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7b28f4910bb956b7ab320b62e98096402354eca976c587d1eeccd523d9bac03" -dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-http", - "bytes", - "form_urlencoded", - "hex", - "hmac", - "http", - "once_cell", - "percent-encoding", - "regex", - "sha2", - "time", - "tracing", -] - -[[package]] -name = "aws-smithy-async" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cdb73f85528b9d19c23a496034ac53703955a59323d581c06aa27b4e4e247af" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", - "tokio-stream", -] - -[[package]] -name = "aws-smithy-checksums" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb15946af1b8d3beeff53ad991d9bff68ac22426b6d40372b958a75fa61eaed" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "crc32c", - "crc32fast", - "hex", - "http", - "http-body", - "md-5", - "pin-project-lite", - "sha1", - "sha2", - "tracing", -] - -[[package]] -name = "aws-smithy-client" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c27b2756264c82f830a91cb4d2d485b2d19ad5bea476d9a966e03d27f27ba59a" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand 2.0.1", - "http", - "http-body", - "hyper", - "hyper-rustls", - "lazy_static", - "pin-project-lite", - "rustls", - "tokio", - "tower", - "tracing", -] - -[[package]] -name = "aws-smithy-eventstream" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "850233feab37b591b7377fd52063aa37af615687f5896807abe7f49bd4e1d25b" -dependencies = [ - "aws-smithy-types", - "bytes", - "crc32fast", -] - -[[package]] -name = "aws-smithy-http" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54cdcf365d8eee60686885f750a34c190e513677db58bbc466c44c588abf4199" -dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "http", - "http-body", - "hyper", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "aws-smithy-http-tower" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822de399d0ce62829a69dfa8c5cd08efdbe61a7426b953e2268f8b8b52a607bd" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "http", - "http-body", - "pin-project-lite", - "tower", - "tracing", -] - -[[package]] -name = "aws-smithy-json" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1e7ab8fa7ad10c193af7ae56d2420989e9f4758bf03601a342573333ea34f" -dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-runtime" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745e096b3553e7e0f40622aa04971ce52765af82bebdeeac53aa6fc82fe801e6" -dependencies = [ - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-runtime-api", - "aws-smithy-types", - "bytes", - "fastrand 2.0.1", - "http", - "http-body", - "once_cell", - "pin-project-lite", - "pin-utils", - "tokio", - "tracing", -] - -[[package]] -name = "aws-smithy-runtime-api" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93d0ae0c9cfd57944e9711ea610b48a963fb174a53aabacc08c5794a594b1d02" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "http", - "tokio", - "tracing", -] - -[[package]] -name = "aws-smithy-types" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d90dbc8da2f6be461fa3c1906b20af8f79d14968fe47f2b7d29d086f62a51728" -dependencies = [ - "base64-simd", - "itoa", - "num-integer", - "ryu", - "serde", - "time", -] - -[[package]] -name = "aws-smithy-xml" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e01d2dedcdd8023043716cfeeb3c6c59f2d447fce365d8e194838891794b23b6" -dependencies = [ - "xmlparser", -] - -[[package]] -name = "aws-types" -version = "0.56.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85aa0451bf8af1bf22a4f028d5d28054507a14be43cb8ac0597a8471fba9edfe" -dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-types", - "http", - "rustc_version", - "tracing", -] - [[package]] name = "axum" version = "0.6.20" @@ -1156,16 +856,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bytes-utils" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" -dependencies = [ - "bytes", - "either", -] - [[package]] name = "camino" version = "1.1.6" @@ -1494,15 +1184,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc32c" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74" -dependencies = [ - "rustc_version", -] - [[package]] name = "crc32fast" version = "1.3.2" @@ -3201,8 +2882,6 @@ dependencies = [ "async-trait", "athena", "autometrics", - "aws-credential-types", - "aws-sdk-s3", "base64-simd", "bytes", "const_format", @@ -3243,6 +2922,7 @@ dependencies = [ "rayon", "redis", "rsa", + "rusty-s3", "scoped-futures", "serde", "serial_test", @@ -3430,16 +3110,13 @@ name = "kitsune-storage" version = "0.0.1-pre.3" dependencies = [ "async-trait", - "aws-credential-types", - "aws-sdk-s3", - "aws-smithy-http", "bytes", "enum_dispatch", "futures-util", "http", - "http-body", - "pin-project-lite", - "sync_wrapper", + "hyper", + "kitsune-http-client", + "rusty-s3", "tempfile", "tokio", "tokio-util", @@ -5222,6 +4899,20 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +[[package]] +name = "rusty-s3" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa883f1b986a5249641e574ca0e11ac4fb9970b009c6fbb96fedaf4fa78db8" +dependencies = [ + "hmac", + "percent-encoding", + "sha2", + "time", + "url", + "zeroize", +] + [[package]] name = "ryu" version = "1.0.15" diff --git a/crates/kitsune-core/Cargo.toml b/crates/kitsune-core/Cargo.toml index eff454512..5d0ae8be1 100644 --- a/crates/kitsune-core/Cargo.toml +++ b/crates/kitsune-core/Cargo.toml @@ -11,10 +11,6 @@ async-stream = "0.3.5" async-trait = "0.1.73" athena = { path = "../../lib/athena" } autometrics = { version = "0.6.0", default-features = false } -aws-credential-types = { version = "0.56.1", features = [ - "hardcoded-credentials", -] } -aws-sdk-s3 = "0.33.0" base64-simd = "0.8.0" bytes = "1.5.0" const_format = "0.2.32" @@ -59,6 +55,7 @@ pulldown-cmark = { version = "0.9.3", default-features = false, features = [ rand = "0.8.5" rayon = "1.8.0" rsa = "0.9.2" +rusty-s3 = { version = "0.5.0", default-features = false } scoped-futures = "0.1.3" serde = { version = "1.0.189", features = ["derive"] } sha2 = { version = "0.10.8", features = ["asm"] } diff --git a/crates/kitsune-core/src/lib.rs b/crates/kitsune-core/src/lib.rs index d1f4113ca..a4b49c94e 100644 --- a/crates/kitsune-core/src/lib.rs +++ b/crates/kitsune-core/src/lib.rs @@ -45,8 +45,6 @@ use self::{ webfinger::Webfinger, }; use athena::JobQueue; -use aws_credential_types::Credentials; -use aws_sdk_s3::config::Region; use eyre::Context; use kitsune_cache::{ArcCache, InMemoryCache, NoopCache, RedisCache}; use kitsune_captcha::{hcaptcha::Captcha as HCaptcha, mcaptcha::Captcha as MCaptcha, Captcha}; @@ -61,6 +59,7 @@ use kitsune_messaging::{ }; use kitsune_search::{NoopSearchService, SearchService, SqlSearchService}; use kitsune_storage::{fs::Storage as FsStorage, s3::Storage as S3Storage, Storage}; +use rusty_s3::{Bucket as S3Bucket, Credentials as S3Credentials}; use serde::{de::DeserializeOwned, Serialize}; use std::{ fmt::Display, @@ -121,26 +120,34 @@ fn prepare_captcha(config: &CaptchaConfiguration) -> Captcha { } } -fn prepare_storage(config: &Configuration) -> Storage { - match config.storage { +fn prepare_storage(config: &Configuration) -> eyre::Result { + let storage = match config.storage { StorageConfiguration::Fs(ref fs_config) => { FsStorage::new(fs_config.upload_dir.as_str().into()).into() } StorageConfiguration::S3(ref s3_config) => { - let s3_client_config = aws_sdk_s3::Config::builder() - .region(Region::new(s3_config.region.to_string())) - .endpoint_url(s3_config.endpoint_url.as_str()) - .force_path_style(s3_config.force_path_style) - .credentials_provider(Credentials::from_keys( - s3_config.access_key.as_str(), - s3_config.secret_access_key.as_str(), - None, - )) - .build(); - - S3Storage::new(s3_config.bucket_name.to_string(), s3_client_config).into() + let path_style = if s3_config.force_path_style { + rusty_s3::UrlStyle::Path + } else { + rusty_s3::UrlStyle::VirtualHost + }; + + let s3_credentials = S3Credentials::new( + s3_config.access_key.as_str(), + s3_config.secret_access_key.as_str(), + ); + let s3_bucket = S3Bucket::new( + s3_config.endpoint_url.parse()?, + path_style, + s3_config.bucket_name.to_string(), + s3_config.region.to_string(), + )?; + + S3Storage::new(s3_bucket, s3_credentials).into() } - } + }; + + Ok(storage) } fn prepare_mail_sender( @@ -248,7 +255,7 @@ pub async fn prepare_state( let attachment_service = AttachmentService::builder() .db_pool(db_pool.clone()) .media_proxy_enabled(config.server.media_proxy_enabled) - .storage_backend(prepare_storage(config)) + .storage_backend(prepare_storage(config)?) .url_service(url_service.clone()) .build(); diff --git a/crates/kitsune-storage/Cargo.toml b/crates/kitsune-storage/Cargo.toml index 2418c4db3..750101fe7 100644 --- a/crates/kitsune-storage/Cargo.toml +++ b/crates/kitsune-storage/Cargo.toml @@ -5,21 +5,16 @@ edition.workspace = true [dependencies] async-trait = "0.1.73" -aws-sdk-s3 = "0.33.0" -aws-smithy-http = "0.56.1" bytes = "1.5.0" enum_dispatch = "0.3.12" futures-util = "0.3.28" http = "0.2.9" -http-body = "0.4.5" -pin-project-lite = "0.2.13" -sync_wrapper = "0.1.2" +hyper = { version = "0.14.27", features = ["stream"] } +kitsune-http-client = { path = "../kitsune-http-client" } +rusty-s3 = { version = "0.5.0", default-features = false } tokio = { version = "1.33.0", features = ["fs", "io-util"] } tokio-util = { version = "0.7.9", features = ["io"] } [dev-dependencies] -aws-credential-types = { version = "0.56.1", features = [ - "hardcoded-credentials", -] } tempfile = "3.8.0" tokio = { version = "1.33.0", features = ["macros", "rt"] } diff --git a/crates/kitsune-storage/examples/s3.rs b/crates/kitsune-storage/examples/s3.rs index ec9a042e1..eae2f5719 100644 --- a/crates/kitsune-storage/examples/s3.rs +++ b/crates/kitsune-storage/examples/s3.rs @@ -1,8 +1,7 @@ -use aws_credential_types::Credentials; -use aws_sdk_s3::{config::Region, Config}; use bytes::{BufMut, BytesMut}; use futures_util::{future, stream, StreamExt, TryStreamExt}; use kitsune_storage::{s3::Storage, StorageBackend}; +use rusty_s3::{Bucket, Credentials, UrlStyle}; use std::{env, str}; const TEST_DATA: &str = r#" @@ -65,14 +64,15 @@ async fn main() { let endpoint_url = env::var("ENDPOINT_URL").unwrap(); let region = env::var("REGION").unwrap(); - let credentials = Credentials::from_keys(access_key, secret_access_key, None); - let config = Config::builder() - .region(Region::new(region)) - .force_path_style(true) - .credentials_provider(credentials) - .endpoint_url(endpoint_url) - .build(); - let backend = Storage::new(bucket_name, config); + let credentials = Credentials::new(access_key, secret_access_key); + let bucket = Bucket::new( + endpoint_url.parse().unwrap(), + UrlStyle::VirtualHost, + bucket_name, + region, + ) + .unwrap(); + let backend = Storage::new(bucket, credentials); let operation = env::args().nth(1).unwrap(); diff --git a/crates/kitsune-storage/src/s3.rs b/crates/kitsune-storage/src/s3.rs index 474f60195..b8cd271e5 100644 --- a/crates/kitsune-storage/src/s3.rs +++ b/crates/kitsune-storage/src/s3.rs @@ -2,68 +2,99 @@ //! An S3 backed implementation of the [`StorageBackend`] trait //! -use crate::{BoxError, Result, StorageBackend}; +use crate::{Result, StorageBackend}; use async_trait::async_trait; -use aws_sdk_s3::{Client, Config}; -use aws_smithy_http::{ - body::{BoxBody, SdkBody}, - byte_stream::ByteStream, -}; use bytes::Bytes; use futures_util::{stream::BoxStream, Stream, StreamExt, TryStreamExt}; -use http::HeaderMap; -use http_body::Body; -use pin_project_lite::pin_project; -use std::{ - pin::Pin, - task::{self, Poll}, +use http::Request; +use hyper::Body; +use kitsune_http_client::Client as HttpClient; +use rusty_s3::{ + actions::{DeleteObject, GetObject, PutObject}, + Bucket, Credentials, S3Action, }; -use sync_wrapper::SyncWrapper; +use std::{sync::Arc, time::Duration}; + +const FIVE_MINUTES: Duration = Duration::from_secs(5 * 60); -pin_project! { - struct StreamBody { - #[pin] - inner: SyncWrapper, +const fn s3_method_to_http(method: rusty_s3::Method) -> http::Method { + match method { + rusty_s3::Method::Head => http::Method::HEAD, + rusty_s3::Method::Get => http::Method::GET, + rusty_s3::Method::Post => http::Method::POST, + rusty_s3::Method::Put => http::Method::PUT, + rusty_s3::Method::Delete => http::Method::DELETE, } } -impl Body for StreamBody -where - S: Stream>, -{ - type Data = Bytes; - type Error = BoxError; - - fn poll_data( - self: Pin<&mut Self>, - cx: &mut task::Context<'_>, - ) -> Poll>> { - let this = self.project(); - this.inner.get_pin_mut().poll_next(cx) +struct S3Client { + bucket: Bucket, + credentials: Credentials, + http_client: HttpClient, +} + +impl S3Client { + pub async fn delete_object(&self, path: &str) -> Result<()> { + let delete_action = self.bucket.delete_object(Some(&self.credentials), path); + + let request = Request::builder() + .uri(String::from(delete_action.sign(FIVE_MINUTES))) + .method(s3_method_to_http(DeleteObject::METHOD)) + .body(Body::empty())?; + + self.http_client.execute(request).await?; + + Ok(()) + } + + pub async fn get_object(&self, path: &str) -> Result>> { + let get_action = self.bucket.get_object(Some(&self.credentials), path); + + let request = Request::builder() + .uri(String::from(get_action.sign(FIVE_MINUTES))) + .method(s3_method_to_http(GetObject::METHOD)) + .body(Body::empty())?; + + let response = self.http_client.execute(request).await?; + + Ok(response.stream().map_err(Into::into)) } - fn poll_trailers( - self: Pin<&mut Self>, - _cx: &mut task::Context<'_>, - ) -> Poll, Self::Error>> { - Poll::Ready(Ok(None)) + pub async fn put_object(&self, path: &str, stream: S) -> Result<()> + where + S: Stream> + Send + 'static, + { + let put_action = self.bucket.put_object(Some(&self.credentials), path); + + let request = Request::builder() + .uri(String::from(put_action.sign(FIVE_MINUTES))) + .method(s3_method_to_http(PutObject::METHOD)) + .body(Body::wrap_stream(stream))?; + + self.http_client.execute(request).await?; + + Ok(()) } } #[derive(Clone)] /// S3-backed storage pub struct Storage { - bucket_name: String, - client: Client, + client: Arc, } impl Storage { /// Create a new storage instance #[must_use] - pub fn new(bucket_name: String, config: Config) -> Self { + pub fn new(bucket: Bucket, credentials: Credentials) -> Self { + let http_client = HttpClient::builder().content_length_limit(None).build(); + Self { - bucket_name, - client: Client::from_conf(config), + client: Arc::new(S3Client { + bucket, + credentials, + http_client, + }), } } } @@ -71,44 +102,18 @@ impl Storage { #[async_trait] impl StorageBackend for Storage { async fn delete(&self, path: &str) -> Result<()> { - self.client - .delete_object() - .bucket(&self.bucket_name) - .key(path) - .send() - .await?; - - Ok(()) + self.client.delete_object(path).await } async fn get(&self, path: &str) -> Result>> { - let response = self - .client - .get_object() - .bucket(&self.bucket_name) - .key(path) - .send() - .await?; - - Ok(response.body.map_err(Into::into).boxed()) + let stream = self.client.get_object(path).await?.boxed(); + Ok(stream) } - async fn put(&self, path: &str, input_stream: T) -> Result<()> + async fn put(&self, path: &str, input_stream: S) -> Result<()> where - T: Stream> + Send + 'static, + S: Stream> + Send + 'static, { - let body = BoxBody::new(StreamBody { - inner: SyncWrapper::new(input_stream), - }); - - self.client - .put_object() - .bucket(&self.bucket_name) - .key(path) - .body(ByteStream::new(SdkBody::from_dyn(body))) - .send() - .await?; - - Ok(()) + self.client.put_object(path, input_stream).await } }