From eb82571166bfe2fbaf2fbf8a52c184d0169c48c7 Mon Sep 17 00:00:00 2001 From: vorozhkog Date: Thu, 28 Nov 2024 13:04:14 +0000 Subject: [PATCH 01/11] change image download method to async --- src/main.py | 61 +++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 7 deletions(-) diff --git a/src/main.py b/src/main.py index ba4a859..f4464df 100644 --- a/src/main.py +++ b/src/main.py @@ -2,6 +2,7 @@ import json import os from distutils.util import strtobool +import asyncio import supervisely as sly from dotenv import load_dotenv @@ -80,15 +81,31 @@ def export_to_coco(api: sly.Api) -> None: total_cnt=len(images), min_report_percent=5, ) - for batch in sly.batched(images): - image_ids = [image_info.id for image_info in batch] - sly.logger.info(f"Working with batch of {len(batch)} images with ids: {image_ids}") - if selected_output == "images": - image_paths = [os.path.join(img_dir, image_info.name) for image_info in batch] - f.download_batch_with_retry(api, dataset.id, image_ids, image_paths) + dataset_path = os.path.join(img_dir, project.name, dataset.name) + os.makedirs(dataset_path, exist_ok=True) + + if selected_output == "images": + image_ids = [image_info.id for image_info in images] + paths = [os.path.join(dataset_path, image_info.name) for image_info in images] + if api.server_address.startswith("https://"): + semaphore = asyncio.Semaphore(100) + else: + semaphore = None + # api._get_default_semaphore() + coro = api.image.download_paths_async( + image_ids, paths, semaphore, progress_cb=ds_progress.iters_done_report + ) + loop = sly.utils.get_or_create_event_loop() + if loop.is_running(): + future = asyncio.run_coroutine_threadsafe(coro, loop) + future.result() + else: + loop.run_until_complete(coro) - ann_infos = api.annotation.download_batch(dataset.id, image_ids) + for batch in sly.batched(images): + batch_ids = [image_info.id for image_info in batch] + ann_infos = api.annotation.download_batch(dataset.id, batch_ids) anns = [] sly.logger.info(f"Preparing to convert {len(ann_infos)} annotations...") for ann_info, img_info in zip(ann_infos, batch): @@ -109,6 +126,36 @@ def export_to_coco(api: sly.Api) -> None: include_captions, RECTANGLE_MARK, ) + + # for batch in sly.batched(images): + # image_ids = [image_info.id for image_info in batch] + # sly.logger.info(f"Working with batch of {len(batch)} images with ids: {image_ids}") + + # if selected_output == "images": + # image_paths = [os.path.join(img_dir, image_info.name) for image_info in batch] + # f.download_batch_with_retry(api, dataset.id, image_ids, image_paths) + + # ann_infos = api.annotation.download_batch(dataset.id, image_ids) + # anns = [] + # sly.logger.info(f"Preparing to convert {len(ann_infos)} annotations...") + # for ann_info, img_info in zip(ann_infos, batch): + # ann = convert_geometry.convert_annotation( + # ann_info, img_info, project_meta, meta, RECTANGLE_MARK + # ) + # anns.append(ann) + # sly.logger.info(f"{len(anns)} annotations converted...") + # coco_instances, label_id, coco_captions, caption_id = f.create_coco_annotation( + # categories_mapping, + # batch, + # anns, + # label_id, + # coco_instances, + # caption_id, + # coco_captions, + # ds_progress, + # include_captions, + # RECTANGLE_MARK, + # ) with open(os.path.join(ann_dir, "instances.json"), "w") as file: json.dump(coco_instances, file) if coco_captions is not None and include_captions: From 6448cd90f978817b88212f09b65463fdd5eea7e0 Mon Sep 17 00:00:00 2001 From: vorozhkog Date: Thu, 28 Nov 2024 15:26:16 +0000 Subject: [PATCH 02/11] time downloading --- dev_requirements.txt | 2 +- local.env | 4 +++- src/main.py | 28 ++++++++++++++++++---------- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index b3a3959..a76f6fb 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1 +1 @@ -supervisely==6.73.162 +git+https://github.com/supervisely/supervisely.git@project_download_async \ No newline at end of file diff --git a/local.env b/local.env index 7998f37..319f259 100644 --- a/local.env +++ b/local.env @@ -1,6 +1,6 @@ TEAM_ID = 431 WORKSPACE_ID = 1019 -PROJECT_ID = 42759 +PROJECT_ID = 41703 # options: "images" "annotations" modal.state.selectedOutput="images" @@ -9,3 +9,5 @@ modal.state.selectedFilter="all" modal.state.allDatasets=True modal.state.datasets=[71754] modal.state.captions=False + +LOGLEVEL = "debug" diff --git a/src/main.py b/src/main.py index f4464df..0b84fa4 100644 --- a/src/main.py +++ b/src/main.py @@ -10,6 +10,7 @@ import convert_geometry import functions as f import workflow as w +from tinytimer import Timer # region constants USER_NAME = "Supervisely" @@ -82,7 +83,7 @@ def export_to_coco(api: sly.Api) -> None: min_report_percent=5, ) - dataset_path = os.path.join(img_dir, project.name, dataset.name) + dataset_path = os.path.join(coco_dataset_dir, project.name, dataset.name) os.makedirs(dataset_path, exist_ok=True) if selected_output == "images": @@ -93,15 +94,18 @@ def export_to_coco(api: sly.Api) -> None: else: semaphore = None # api._get_default_semaphore() - coro = api.image.download_paths_async( - image_ids, paths, semaphore, progress_cb=ds_progress.iters_done_report + + with Timer() as t: + coro = api.image.download_paths_async(image_ids, paths, semaphore) + loop = sly.utils.get_or_create_event_loop() + if loop.is_running(): + future = asyncio.run_coroutine_threadsafe(coro, loop) + future.result() + else: + loop.run_until_complete(coro) + sly.logger.info( + f"Downloading time: {t.elapsed:.4f} seconds per {len(image_ids)} images ({t.elapsed/len(image_ids):.4f} seconds per image)" ) - loop = sly.utils.get_or_create_event_loop() - if loop.is_running(): - future = asyncio.run_coroutine_threadsafe(coro, loop) - future.result() - else: - loop.run_until_complete(coro) for batch in sly.batched(images): batch_ids = [image_info.id for image_info in batch] @@ -133,7 +137,11 @@ def export_to_coco(api: sly.Api) -> None: # if selected_output == "images": # image_paths = [os.path.join(img_dir, image_info.name) for image_info in batch] - # f.download_batch_with_retry(api, dataset.id, image_ids, image_paths) + # with Timer() as t: + # f.download_batch_with_retry(api, dataset.id, image_ids, image_paths) + # sly.logger.info( + # f"Downloading time: {t.elapsed:.4f} seconds per {len(image_ids)} images ({t.elapsed/len(image_ids):.4f} seconds per image)" + # ) # ann_infos = api.annotation.download_batch(dataset.id, image_ids) # anns = [] From 988f88798e3510615d365bfc004a8d918d0a4cd5 Mon Sep 17 00:00:00 2001 From: vorozhkog Date: Fri, 29 Nov 2024 15:15:31 +0000 Subject: [PATCH 03/11] replace tinytimer --- local.env | 2 +- src/convert_geometry.py | 1 + src/main.py | 15 ++++++++++++--- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/local.env b/local.env index 319f259..b42a09a 100644 --- a/local.env +++ b/local.env @@ -1,6 +1,6 @@ TEAM_ID = 431 WORKSPACE_ID = 1019 -PROJECT_ID = 41703 +PROJECT_ID = 40721 # options: "images" "annotations" modal.state.selectedOutput="images" diff --git a/src/convert_geometry.py b/src/convert_geometry.py index ccd5f79..31ba5fe 100644 --- a/src/convert_geometry.py +++ b/src/convert_geometry.py @@ -5,6 +5,7 @@ from typing import List import uuid + def prepare_meta(meta: sly.ProjectMeta): new_classes = [] for cls in meta.obj_classes: diff --git a/src/main.py b/src/main.py index 0b84fa4..239efe7 100644 --- a/src/main.py +++ b/src/main.py @@ -10,7 +10,7 @@ import convert_geometry import functions as f import workflow as w -from tinytimer import Timer +import time # region constants USER_NAME = "Supervisely" @@ -43,6 +43,16 @@ ) +class Timer: + def __enter__(self): + self.start = time.perf_counter() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.end = time.perf_counter() + self.elapsed = self.end - self.start + + def export_to_coco(api: sly.Api) -> None: project = api.project.get_info_by_id(project_id) project_meta = sly.ProjectMeta.from_json(api.project.get_meta(project_id)) @@ -90,10 +100,9 @@ def export_to_coco(api: sly.Api) -> None: image_ids = [image_info.id for image_info in images] paths = [os.path.join(dataset_path, image_info.name) for image_info in images] if api.server_address.startswith("https://"): - semaphore = asyncio.Semaphore(100) + semaphore = asyncio.Semaphore(10) else: semaphore = None - # api._get_default_semaphore() with Timer() as t: coro = api.image.download_paths_async(image_ids, paths, semaphore) From 6f6c48a4ff3c2520363052c3e7600fbe9d0c140f Mon Sep 17 00:00:00 2001 From: vorozhkog Date: Fri, 29 Nov 2024 15:19:32 +0000 Subject: [PATCH 04/11] rename requirements --- dev_requirements.txt => requirements.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dev_requirements.txt => requirements.txt (100%) diff --git a/dev_requirements.txt b/requirements.txt similarity index 100% rename from dev_requirements.txt rename to requirements.txt From 70c71a81de7913ebb0ae7096e16703137712c5d8 Mon Sep 17 00:00:00 2001 From: vorozhkog Date: Fri, 29 Nov 2024 15:21:29 +0000 Subject: [PATCH 05/11] change sdk version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a76f6fb..57c96f3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -git+https://github.com/supervisely/supervisely.git@project_download_async \ No newline at end of file +supervisely==6.73.241 \ No newline at end of file From 27c7f087547510efa33e6f9dedac84cf17e220ed Mon Sep 17 00:00:00 2001 From: vorozhkog Date: Mon, 2 Dec 2024 10:35:50 +0000 Subject: [PATCH 06/11] batched --- src/main.py | 92 +++++++++++++++++++++-------------------------------- 1 file changed, 37 insertions(+), 55 deletions(-) diff --git a/src/main.py b/src/main.py index 239efe7..d4b5041 100644 --- a/src/main.py +++ b/src/main.py @@ -27,7 +27,9 @@ team_id = sly.env.team_id() workspace_id = sly.env.workspace_id() project_id = sly.env.project_id() -selected_output = os.environ["modal.state.selectedOutput"] +export_images = True +if os.environ["modal.state.selectedOutput"] == "annotations": + export_images = False selected_filter = os.environ["modal.state.selectedFilter"] all_datasets = bool(strtobool(os.getenv("modal.state.allDatasets"))) selected_datasets = ast.literal_eval(os.environ["modal.state.datasets"]) @@ -35,15 +37,20 @@ # endregion sly.logger.info(f"Team ID: {team_id}, Workspace ID: {workspace_id}, Project ID: {project_id}") sly.logger.info( - f"Selected output: {selected_output}, " + f"Selected output: {os.environ["modal.state.selectedOutput"]}, " f"Selected filter: {selected_filter}, " f"All datasets: {all_datasets}, " f"Selected datasets: {selected_datasets}, " f"Included captions: {include_captions}" ) - class Timer: + + def __init__(self, message=None, items_cnt=None): + self.message = message + self.items_cnt = items_cnt + self.elapsed = 0 + def __enter__(self): self.start = time.perf_counter() return self @@ -51,6 +58,12 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): self.end = time.perf_counter() self.elapsed = self.end - self.start + msg = self.message or f"Block execution" + if self.items_cnt is not None: + log_msg = f"{msg} time: {self.elapsed:.3f} seconds per {self.items_cnt} items ({self.elapsed/self.items_cnt:.3f} seconds per item)" + else: + log_msg = f"{msg} time: {self.elapsed:.3f} seconds" + sly.logger.info(log_msg) def export_to_coco(api: sly.Api) -> None: @@ -93,32 +106,34 @@ def export_to_coco(api: sly.Api) -> None: min_report_percent=5, ) + # to move dataset_path = os.path.join(coco_dataset_dir, project.name, dataset.name) os.makedirs(dataset_path, exist_ok=True) - if selected_output == "images": - image_ids = [image_info.id for image_info in images] - paths = [os.path.join(dataset_path, image_info.name) for image_info in images] - if api.server_address.startswith("https://"): - semaphore = asyncio.Semaphore(10) - else: - semaphore = None - - with Timer() as t: - coro = api.image.download_paths_async(image_ids, paths, semaphore) + for batch in sly.batched(images): + batch_ids = [info.id for info in batch] + batch_paths = [os.path.join(dataset_path, image_info.name) for image_info in batch] + + if export_images: + with Timer("Image downloading", len(batch_ids)): + coro = api.image.download_paths_async(batch_ids, batch_paths) + loop = sly.utils.get_or_create_event_loop() + if loop.is_running(): + future = asyncio.run_coroutine_threadsafe(coro, loop) + future.result() + else: + loop.run_until_complete(coro) + + ann_infos = [] + with Timer("Annotation downloading", len(batch_ids)): + coro = api.annotation.download_batch_async(dataset.id, batch_ids) loop = sly.utils.get_or_create_event_loop() if loop.is_running(): future = asyncio.run_coroutine_threadsafe(coro, loop) - future.result() + ann_infos.extend(future.result()) else: - loop.run_until_complete(coro) - sly.logger.info( - f"Downloading time: {t.elapsed:.4f} seconds per {len(image_ids)} images ({t.elapsed/len(image_ids):.4f} seconds per image)" - ) - - for batch in sly.batched(images): - batch_ids = [image_info.id for image_info in batch] - ann_infos = api.annotation.download_batch(dataset.id, batch_ids) + ann_infos.extend(loop.run_until_complete(coro)) + anns = [] sly.logger.info(f"Preparing to convert {len(ann_infos)} annotations...") for ann_info, img_info in zip(ann_infos, batch): @@ -140,39 +155,6 @@ def export_to_coco(api: sly.Api) -> None: RECTANGLE_MARK, ) - # for batch in sly.batched(images): - # image_ids = [image_info.id for image_info in batch] - # sly.logger.info(f"Working with batch of {len(batch)} images with ids: {image_ids}") - - # if selected_output == "images": - # image_paths = [os.path.join(img_dir, image_info.name) for image_info in batch] - # with Timer() as t: - # f.download_batch_with_retry(api, dataset.id, image_ids, image_paths) - # sly.logger.info( - # f"Downloading time: {t.elapsed:.4f} seconds per {len(image_ids)} images ({t.elapsed/len(image_ids):.4f} seconds per image)" - # ) - - # ann_infos = api.annotation.download_batch(dataset.id, image_ids) - # anns = [] - # sly.logger.info(f"Preparing to convert {len(ann_infos)} annotations...") - # for ann_info, img_info in zip(ann_infos, batch): - # ann = convert_geometry.convert_annotation( - # ann_info, img_info, project_meta, meta, RECTANGLE_MARK - # ) - # anns.append(ann) - # sly.logger.info(f"{len(anns)} annotations converted...") - # coco_instances, label_id, coco_captions, caption_id = f.create_coco_annotation( - # categories_mapping, - # batch, - # anns, - # label_id, - # coco_instances, - # caption_id, - # coco_captions, - # ds_progress, - # include_captions, - # RECTANGLE_MARK, - # ) with open(os.path.join(ann_dir, "instances.json"), "w") as file: json.dump(coco_instances, file) if coco_captions is not None and include_captions: From 5c76c8b947e0ad522da1f2f38ff94ca394e5026c Mon Sep 17 00:00:00 2001 From: vorozhkog Date: Mon, 2 Dec 2024 11:58:39 +0000 Subject: [PATCH 07/11] fix invalid syntax --- src/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main.py b/src/main.py index d4b5041..1a19a8c 100644 --- a/src/main.py +++ b/src/main.py @@ -27,9 +27,9 @@ team_id = sly.env.team_id() workspace_id = sly.env.workspace_id() project_id = sly.env.project_id() -export_images = True -if os.environ["modal.state.selectedOutput"] == "annotations": - export_images = False + +output_mode = os.environ["modal.state.selectedOutput"] +export_images = True if output_mode == "images" else False selected_filter = os.environ["modal.state.selectedFilter"] all_datasets = bool(strtobool(os.getenv("modal.state.allDatasets"))) selected_datasets = ast.literal_eval(os.environ["modal.state.datasets"]) @@ -37,7 +37,7 @@ # endregion sly.logger.info(f"Team ID: {team_id}, Workspace ID: {workspace_id}, Project ID: {project_id}") sly.logger.info( - f"Selected output: {os.environ["modal.state.selectedOutput"]}, " + f"Selected output: {output_mode}, " f"Selected filter: {selected_filter}, " f"All datasets: {all_datasets}, " f"Selected datasets: {selected_datasets}, " @@ -133,7 +133,7 @@ def export_to_coco(api: sly.Api) -> None: ann_infos.extend(future.result()) else: ann_infos.extend(loop.run_until_complete(coro)) - + anns = [] sly.logger.info(f"Preparing to convert {len(ann_infos)} annotations...") for ann_info, img_info in zip(ann_infos, batch): From dc21e575c134962634c9c70c76492b29b0f4ee8d Mon Sep 17 00:00:00 2001 From: vorozhkog Date: Tue, 3 Dec 2024 02:47:26 +0000 Subject: [PATCH 08/11] update sdk version --- config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config.json b/config.json index 011a4d7..9062c63 100644 --- a/config.json +++ b/config.json @@ -7,8 +7,8 @@ "export" ], "description": "Converts Supervisely to COCO format and prepares tar archive for download", - "docker_image": "supervisely/import-export:6.73.162", - "min_instance_version": "6.11.8", + "docker_image": "supervisely/import-export:6.73.242", + "min_instance_version": "6.12.5", "main_script": "src/main.py", "modal_template": "src/modal.html", "task_location": "workspace_tasks", From 237b8c6bc0bad1fb0748caedbe0ccded2f127659 Mon Sep 17 00:00:00 2001 From: GoldenAnpu Date: Wed, 18 Dec 2024 00:20:53 +0100 Subject: [PATCH 09/11] Update Semaphore for app isntance Remove Timer Upgrade SDK to v6.73.256 --- .vscode/launch.json | 2 +- .vscode/settings.json | 4 ++- config.json | 57 ++++++++++++++++++++----------------------- dev_requirements.txt | 1 + requirements.txt | 1 - src/main.py | 52 +++++++++++++-------------------------- 6 files changed, 48 insertions(+), 69 deletions(-) create mode 100644 dev_requirements.txt delete mode 100644 requirements.txt diff --git a/.vscode/launch.json b/.vscode/launch.json index 0382918..101c512 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -3,7 +3,7 @@ "configurations": [ { "name": "Python: Current File", - "type": "python", + "type": "debugpy", "request": "launch", "program": "src/main.py", "console": "integratedTerminal", diff --git a/.vscode/settings.json b/.vscode/settings.json index 7604595..a669591 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -16,7 +16,9 @@ "editor.defaultFormatter": "esbenp.prettier-vscode" }, "[json]": { - "editor.defaultFormatter": "esbenp.prettier-vscode" + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.formatOnSave": true, + "editor.tabSize": 4, }, "[python]": { "editor.defaultFormatter": "ms-python.black-formatter", diff --git a/config.json b/config.json index 9062c63..9d92ff4 100644 --- a/config.json +++ b/config.json @@ -1,32 +1,27 @@ { - "name": "Export to COCO", - "type": "app", - "version": "2.0.0", - "categories": [ - "images", - "export" - ], - "description": "Converts Supervisely to COCO format and prepares tar archive for download", - "docker_image": "supervisely/import-export:6.73.242", - "min_instance_version": "6.12.5", - "main_script": "src/main.py", - "modal_template": "src/modal.html", - "task_location": "workspace_tasks", - "icon": "https://i.imgur.com/9uY44aW.png", - "icon_background": "#FFFFFF", - "headless": true, - "modal_template_state": { - "allDatasets": true, - "datasets": [], - "selectedFilter": "all", - "selectedOutput": "images", - "captions": false - }, - "context_menu": { - "target": [ - "images_project" - ], - "context_root": "Download as" - }, - "poster": "https://user-images.githubusercontent.com/48913536/183899083-64d7683d-57f9-4f7a-b5f4-bf9e7ffd3246.png" -} \ No newline at end of file + "name": "Export to COCO", + "type": "app", + "version": "2.0.0", + "categories": ["images", "export"], + "description": "Converts Supervisely to COCO format and prepares tar archive for download", + "docker_image": "supervisely/import-export:6.73.256", + "min_instance_version": "6.12.12", + "main_script": "src/main.py", + "modal_template": "src/modal.html", + "task_location": "workspace_tasks", + "icon": "https://i.imgur.com/9uY44aW.png", + "icon_background": "#FFFFFF", + "headless": true, + "modal_template_state": { + "allDatasets": true, + "datasets": [], + "selectedFilter": "all", + "selectedOutput": "images", + "captions": false + }, + "context_menu": { + "target": ["images_project"], + "context_root": "Download as" + }, + "poster": "https://user-images.githubusercontent.com/48913536/183899083-64d7683d-57f9-4f7a-b5f4-bf9e7ffd3246.png" +} diff --git a/dev_requirements.txt b/dev_requirements.txt new file mode 100644 index 0000000..5aadc50 --- /dev/null +++ b/dev_requirements.txt @@ -0,0 +1 @@ +supervisely==6.73.256 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 57c96f3..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -supervisely==6.73.241 \ No newline at end of file diff --git a/src/main.py b/src/main.py index 1a19a8c..a40b8f4 100644 --- a/src/main.py +++ b/src/main.py @@ -44,27 +44,6 @@ f"Included captions: {include_captions}" ) -class Timer: - - def __init__(self, message=None, items_cnt=None): - self.message = message - self.items_cnt = items_cnt - self.elapsed = 0 - - def __enter__(self): - self.start = time.perf_counter() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.end = time.perf_counter() - self.elapsed = self.end - self.start - msg = self.message or f"Block execution" - if self.items_cnt is not None: - log_msg = f"{msg} time: {self.elapsed:.3f} seconds per {self.items_cnt} items ({self.elapsed/self.items_cnt:.3f} seconds per item)" - else: - log_msg = f"{msg} time: {self.elapsed:.3f} seconds" - sly.logger.info(log_msg) - def export_to_coco(api: sly.Api) -> None: project = api.project.get_info_by_id(project_id) @@ -115,24 +94,23 @@ def export_to_coco(api: sly.Api) -> None: batch_paths = [os.path.join(dataset_path, image_info.name) for image_info in batch] if export_images: - with Timer("Image downloading", len(batch_ids)): - coro = api.image.download_paths_async(batch_ids, batch_paths) - loop = sly.utils.get_or_create_event_loop() - if loop.is_running(): - future = asyncio.run_coroutine_threadsafe(coro, loop) - future.result() - else: - loop.run_until_complete(coro) - - ann_infos = [] - with Timer("Annotation downloading", len(batch_ids)): - coro = api.annotation.download_batch_async(dataset.id, batch_ids) + coro = api.image.download_paths_async(batch_ids, batch_paths) loop = sly.utils.get_or_create_event_loop() if loop.is_running(): future = asyncio.run_coroutine_threadsafe(coro, loop) - ann_infos.extend(future.result()) + future.result() else: - ann_infos.extend(loop.run_until_complete(coro)) + loop.run_until_complete(coro) + + ann_infos = [] + + coro = api.annotation.download_batch_async(dataset.id, batch_ids) + loop = sly.utils.get_or_create_event_loop() + if loop.is_running(): + future = asyncio.run_coroutine_threadsafe(coro, loop) + ann_infos.extend(future.result()) + else: + ann_infos.extend(loop.run_until_complete(coro)) anns = [] sly.logger.info(f"Preparing to convert {len(ann_infos)} annotations...") @@ -176,4 +154,8 @@ def export_to_coco(api: sly.Api) -> None: if __name__ == "__main__": api = sly.Api.from_env() + if api.server_address == "https://app.supervisely.com": + semaphore = api.get_default_semaphore() + if semaphore._value == 10: + api.set_semaphore_size(7) export_to_coco(api) From 5bc32e7117416e6329cfc20cc388fe0152beb85a Mon Sep 17 00:00:00 2001 From: GoldenAnpu Date: Wed, 18 Dec 2024 12:28:18 +0100 Subject: [PATCH 10/11] remove unused import for time module --- src/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main.py b/src/main.py index a40b8f4..4598103 100644 --- a/src/main.py +++ b/src/main.py @@ -10,7 +10,6 @@ import convert_geometry import functions as f import workflow as w -import time # region constants USER_NAME = "Supervisely" From d90f8d26e854e588af072897473fb43e20988661 Mon Sep 17 00:00:00 2001 From: GoldenAnpu Date: Wed, 18 Dec 2024 17:18:21 +0100 Subject: [PATCH 11/11] Update Docker image and SDK version to 6.73.257; remove unused semaphore size adjustment --- config.json | 2 +- dev_requirements.txt | 2 +- src/main.py | 4 ---- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/config.json b/config.json index 9d92ff4..7f30bfe 100644 --- a/config.json +++ b/config.json @@ -4,7 +4,7 @@ "version": "2.0.0", "categories": ["images", "export"], "description": "Converts Supervisely to COCO format and prepares tar archive for download", - "docker_image": "supervisely/import-export:6.73.256", + "docker_image": "supervisely/import-export:6.73.257", "min_instance_version": "6.12.12", "main_script": "src/main.py", "modal_template": "src/modal.html", diff --git a/dev_requirements.txt b/dev_requirements.txt index 5aadc50..fabe988 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1 +1 @@ -supervisely==6.73.256 \ No newline at end of file +supervisely==6.73.257 \ No newline at end of file diff --git a/src/main.py b/src/main.py index 4598103..66d0dcb 100644 --- a/src/main.py +++ b/src/main.py @@ -153,8 +153,4 @@ def export_to_coco(api: sly.Api) -> None: if __name__ == "__main__": api = sly.Api.from_env() - if api.server_address == "https://app.supervisely.com": - semaphore = api.get_default_semaphore() - if semaphore._value == 10: - api.set_semaphore_size(7) export_to_coco(api)