Skip to content

Commit

Permalink
Split upload map into separate function
Browse files Browse the repository at this point in the history
This makes things cleaner than a big `async move` block

Signed-off-by: Robert Detjens <[email protected]>
  • Loading branch information
detjensrobert committed Feb 8, 2025
1 parent f7c03fe commit 44785f0
Showing 1 changed file with 30 additions and 17 deletions.
47 changes: 30 additions & 17 deletions src/deploy/s3.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
use std::fs::File;
use std::path::PathBuf;
use std::path::{Path, PathBuf};

use anyhow::{anyhow, bail, Context, Error, Ok, Result};
use futures::future::try_join_all;
use itertools::Itertools;
use s3::Bucket;
use simplelog::*;
use tokio;

Expand Down Expand Up @@ -31,22 +32,10 @@ pub async fn upload_assets(

debug!("uploading assets for chal {:?}", chal.directory);

let uploaded = try_join_all(result.assets.iter().map(|asset| async move {
let path_in_bucket = format!(
"assets/{chal_slug}/{file}",
chal_slug = chal.directory.to_string_lossy(),
file = asset.file_name().unwrap().to_string_lossy()
);

trace!("uploading {:?} to bucket path {:?}", asset, &path_in_bucket);

// TODO: move to async/streaming to better handle large files and report progress
let mut asset_file = tokio::fs::File::open(asset).await?;
bucket
.put_object_stream(&mut asset_file, &path_in_bucket)
.await?;

Ok(path_in_bucket.into())
let uploaded = try_join_all(result.assets.iter().map(|asset_file| async move {
upload_single_file(bucket, chal, asset_file)
.await
.with_context(|| format!("failed to upload file {asset_file:?}"))
}))
.await
.with_context(|| format!("failed to upload asset files for chal {:?}", chal.directory))?;
Expand All @@ -59,3 +48,27 @@ pub async fn upload_assets(
}))
.await
}

async fn upload_single_file(
bucket: &Bucket,
chal: &ChallengeConfig,
file: &Path,
) -> Result<PathBuf> {
// e.g. s3.example.domain/assets/misc/foo/stuff.zip
let path_in_bucket = format!(
"assets/{chal_slug}/{file}",
chal_slug = chal.directory.to_string_lossy(),
file = file.file_name().unwrap().to_string_lossy()
);

trace!("uploading {:?} to bucket path {:?}", file, &path_in_bucket);

// TODO: move to async/streaming to better handle large files and report progress
let mut asset_file = tokio::fs::File::open(file).await?;
let r = bucket
.put_object_stream(&mut asset_file, &path_in_bucket)
.await?;
trace!("uploaded {} bytes for file {:?}", r.uploaded_bytes(), file);

Ok(PathBuf::from(path_in_bucket))
}

0 comments on commit 44785f0

Please sign in to comment.