diff --git a/config.json b/config.json index 1d88d4a..9a480d7 100644 --- a/config.json +++ b/config.json @@ -6,7 +6,7 @@ "export" ], "description": "images and JSON annotations", - "docker_image": "supervisely/import-export:6.72.206", + "docker_image": "supervisely/import-export:6.72.205", "instance_version": "6.8.48", "main_script": "src/main.py", "modal_template": "src/modal.html", diff --git a/src/sly_functions.py b/src/sly_functions.py index 9212c46..8f18d40 100644 --- a/src/sly_functions.py +++ b/src/sly_functions.py @@ -1,32 +1,47 @@ import os import time + import supervisely as sly def _download_batch_with_retry(api: sly.Api, dataset_id, image_ids): retry_cnt = 5 - curr_retry = 0 - while curr_retry <= retry_cnt: - try: - imgs_bytes = api.image.download_bytes(dataset_id, image_ids) - if len(imgs_bytes) != len(image_ids): - raise RuntimeError( - f"Downloaded {len(imgs_bytes)} images, but {len(image_ids)} expected." - ) - return imgs_bytes - except Exception as e: - curr_retry += 1 - if curr_retry <= retry_cnt: + curr_retry = 1 + try: + imgs_bytes = api.image.download_bytes(dataset_id, image_ids) + if len(imgs_bytes) != len(image_ids): + raise RuntimeError( + f"Downloaded {len(imgs_bytes)} images, but {len(image_ids)} expected." + ) + return imgs_bytes + except Exception as e: + sly.logger.warn(f"Failed to download images... Error: {e}") + while curr_retry <= retry_cnt: + try: + sly.logger.warn(f"Retry {curr_retry}/{retry_cnt} to download images") time.sleep(2 * curr_retry) - sly.logger.warn( - f"Failed to download images, retry {curr_retry} of {retry_cnt}... Error: {e}" - ) + imgs_bytes = api.image.download_bytes(dataset_id, image_ids) + if len(imgs_bytes) != len(image_ids): + raise RuntimeError( + f"Downloaded {len(imgs_bytes)} images, but {len(image_ids)} expected." + ) + return imgs_bytes + except Exception as e: + curr_retry += 1 raise RuntimeError( f"Failed to download images with ids {image_ids}. Check your data and try again later." ) -def download_project(api: sly.Api, project_id, dest_dir, dataset_ids=None, log_progress=False, batch_size=10, save_image_meta=True): +def download_project( + api: sly.Api, + project_id, + dest_dir, + dataset_ids=None, + log_progress=True, + batch_size=10, + save_image_meta=True, +): dataset_ids = set(dataset_ids) if (dataset_ids is not None) else None project_fs = sly.Project(dest_dir, sly.OpenMode.CREATE) meta = sly.ProjectMeta.from_json(api.project.get_meta(project_id)) @@ -54,7 +69,7 @@ def download_project(api: sly.Api, project_id, dest_dir, dataset_ids=None, log_p total_cnt=len(images), ) - for batch in sly.batched(images, batch_size=10): + for batch in sly.batched(images, batch_size=batch_size): image_ids = [image_info.id for image_info in batch] image_names = [image_info.name for image_info in batch]