Skip to content

Commit

Permalink
hotfix: image uploading in drone tm
Browse files Browse the repository at this point in the history
  • Loading branch information
nrjadkry committed Jan 30, 2025
1 parent 8bb9fe9 commit b6e029f
Showing 1 changed file with 9 additions and 3 deletions.
12 changes: 9 additions & 3 deletions src/backend/app/projects/image_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -132,18 +132,24 @@ async def download_images_from_s3(
get_presigned_url(bucket_name, obj.object_name, 12) for obj in objects
]

total_files = len(object_urls)
async with aiohttp.ClientSession() as session:
for i in range(0, len(object_urls), batch_size):
for i in range(0, total_files, batch_size):
batch = object_urls[i : i + batch_size]
batch_number = i // batch_size + 1
total_batches = (total_files + batch_size - 1) // batch_size

log.info(f"Processing batch {batch_number}/{total_batches}")
tasks = [
self.download_image(
session, url, os.path.join(local_dir, f"file_{i + 1}.jpg")
session, url, os.path.join(local_dir, f"file_{i + j + 1}.jpg")
)
for i, url in enumerate(batch)
for j, url in enumerate(batch)
]
await asyncio.gather(*tasks)

log.info(f"Completed downloading {total_files} images")

def process_new_task(
self,
images: List[str],
Expand Down

0 comments on commit b6e029f

Please sign in to comment.