From b6e029fdb69712fb2f46969f8ab960e847d6d73d Mon Sep 17 00:00:00 2001 From: Niraj Adhikari Date: Thu, 30 Jan 2025 14:55:05 +0545 Subject: [PATCH] hotfix: image uploading in drone tm --- src/backend/app/projects/image_processing.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/backend/app/projects/image_processing.py b/src/backend/app/projects/image_processing.py index da819f98..da47e62b 100644 --- a/src/backend/app/projects/image_processing.py +++ b/src/backend/app/projects/image_processing.py @@ -132,18 +132,24 @@ async def download_images_from_s3( get_presigned_url(bucket_name, obj.object_name, 12) for obj in objects ] + total_files = len(object_urls) async with aiohttp.ClientSession() as session: - for i in range(0, len(object_urls), batch_size): + for i in range(0, total_files, batch_size): batch = object_urls[i : i + batch_size] + batch_number = i // batch_size + 1 + total_batches = (total_files + batch_size - 1) // batch_size + log.info(f"Processing batch {batch_number}/{total_batches}") tasks = [ self.download_image( - session, url, os.path.join(local_dir, f"file_{i + 1}.jpg") + session, url, os.path.join(local_dir, f"file_{i + j + 1}.jpg") ) - for i, url in enumerate(batch) + for j, url in enumerate(batch) ] await asyncio.gather(*tasks) + log.info(f"Completed downloading {total_files} images") + def process_new_task( self, images: List[str],