From 44785f098425e99f49ee9a0c2a30dcb4f4455b16 Mon Sep 17 00:00:00 2001 From: Robert Detjens Date: Sat, 8 Feb 2025 01:11:14 -0800 Subject: [PATCH] Split upload map into separate function This makes things cleaner than a big `async move` block Signed-off-by: Robert Detjens --- src/deploy/s3.rs | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/deploy/s3.rs b/src/deploy/s3.rs index c2986f7..9a862df 100644 --- a/src/deploy/s3.rs +++ b/src/deploy/s3.rs @@ -1,9 +1,10 @@ use std::fs::File; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use anyhow::{anyhow, bail, Context, Error, Ok, Result}; use futures::future::try_join_all; use itertools::Itertools; +use s3::Bucket; use simplelog::*; use tokio; @@ -31,22 +32,10 @@ pub async fn upload_assets( debug!("uploading assets for chal {:?}", chal.directory); - let uploaded = try_join_all(result.assets.iter().map(|asset| async move { - let path_in_bucket = format!( - "assets/{chal_slug}/{file}", - chal_slug = chal.directory.to_string_lossy(), - file = asset.file_name().unwrap().to_string_lossy() - ); - - trace!("uploading {:?} to bucket path {:?}", asset, &path_in_bucket); - - // TODO: move to async/streaming to better handle large files and report progress - let mut asset_file = tokio::fs::File::open(asset).await?; - bucket - .put_object_stream(&mut asset_file, &path_in_bucket) - .await?; - - Ok(path_in_bucket.into()) + let uploaded = try_join_all(result.assets.iter().map(|asset_file| async move { + upload_single_file(bucket, chal, asset_file) + .await + .with_context(|| format!("failed to upload file {asset_file:?}")) })) .await .with_context(|| format!("failed to upload asset files for chal {:?}", chal.directory))?; @@ -59,3 +48,27 @@ pub async fn upload_assets( })) .await } + +async fn upload_single_file( + bucket: &Bucket, + chal: &ChallengeConfig, + file: &Path, +) -> Result { + // e.g. s3.example.domain/assets/misc/foo/stuff.zip + let path_in_bucket = format!( + "assets/{chal_slug}/{file}", + chal_slug = chal.directory.to_string_lossy(), + file = file.file_name().unwrap().to_string_lossy() + ); + + trace!("uploading {:?} to bucket path {:?}", file, &path_in_bucket); + + // TODO: move to async/streaming to better handle large files and report progress + let mut asset_file = tokio::fs::File::open(file).await?; + let r = bucket + .put_object_stream(&mut asset_file, &path_in_bucket) + .await?; + trace!("uploaded {} bytes for file {:?}", r.uploaded_bytes(), file); + + Ok(PathBuf::from(path_in_bucket)) +}