Skip to content

Commit

Permalink
Update logs (#14)
Browse files Browse the repository at this point in the history
  • Loading branch information
almazgimaev authored Dec 1, 2023
1 parent 79f007d commit edc46b8
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 18 deletions.
2 changes: 1 addition & 1 deletion config.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"export"
],
"description": "images and JSON annotations",
"docker_image": "supervisely/import-export:6.72.206",
"docker_image": "supervisely/import-export:6.72.205",
"instance_version": "6.8.48",
"main_script": "src/main.py",
"modal_template": "src/modal.html",
Expand Down
49 changes: 32 additions & 17 deletions src/sly_functions.py
Original file line number Diff line number Diff line change
@@ -1,32 +1,47 @@
import os
import time

import supervisely as sly


def _download_batch_with_retry(api: sly.Api, dataset_id, image_ids):
retry_cnt = 5
curr_retry = 0
while curr_retry <= retry_cnt:
try:
imgs_bytes = api.image.download_bytes(dataset_id, image_ids)
if len(imgs_bytes) != len(image_ids):
raise RuntimeError(
f"Downloaded {len(imgs_bytes)} images, but {len(image_ids)} expected."
)
return imgs_bytes
except Exception as e:
curr_retry += 1
if curr_retry <= retry_cnt:
curr_retry = 1
try:
imgs_bytes = api.image.download_bytes(dataset_id, image_ids)
if len(imgs_bytes) != len(image_ids):
raise RuntimeError(
f"Downloaded {len(imgs_bytes)} images, but {len(image_ids)} expected."
)
return imgs_bytes
except Exception as e:
sly.logger.warn(f"Failed to download images... Error: {e}")
while curr_retry <= retry_cnt:
try:
sly.logger.warn(f"Retry {curr_retry}/{retry_cnt} to download images")
time.sleep(2 * curr_retry)
sly.logger.warn(
f"Failed to download images, retry {curr_retry} of {retry_cnt}... Error: {e}"
)
imgs_bytes = api.image.download_bytes(dataset_id, image_ids)
if len(imgs_bytes) != len(image_ids):
raise RuntimeError(
f"Downloaded {len(imgs_bytes)} images, but {len(image_ids)} expected."
)
return imgs_bytes
except Exception as e:
curr_retry += 1
raise RuntimeError(
f"Failed to download images with ids {image_ids}. Check your data and try again later."
)


def download_project(api: sly.Api, project_id, dest_dir, dataset_ids=None, log_progress=False, batch_size=10, save_image_meta=True):
def download_project(
api: sly.Api,
project_id,
dest_dir,
dataset_ids=None,
log_progress=True,
batch_size=10,
save_image_meta=True,
):
dataset_ids = set(dataset_ids) if (dataset_ids is not None) else None
project_fs = sly.Project(dest_dir, sly.OpenMode.CREATE)
meta = sly.ProjectMeta.from_json(api.project.get_meta(project_id))
Expand Down Expand Up @@ -54,7 +69,7 @@ def download_project(api: sly.Api, project_id, dest_dir, dataset_ids=None, log_p
total_cnt=len(images),
)

for batch in sly.batched(images, batch_size=10):
for batch in sly.batched(images, batch_size=batch_size):
image_ids = [image_info.id for image_info in batch]
image_names = [image_info.name for image_info in batch]

Expand Down

0 comments on commit edc46b8

Please sign in to comment.