diff --git a/.vscode/launch.json b/.vscode/launch.json index b16ca9b..3fb6275 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,29 +1,28 @@ { - "version": "0.2.0", - "configurations": [ - { - "name": "Debug", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "integratedTerminal", - "justMyCode": true, - "env": { - "PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}", - "ENV": "development" - } - }, - { - "name": "Advanced debug", - "type": "python", - "request": "launch", - "program": "${file}", - "console": "integratedTerminal", - "justMyCode": true, - "env": { - "PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}", - "ENV": "production" - } - } - ] + "version": "0.2.0", + "configurations": [ + { + "name": "Python: main.py", + "type": "debugpy", + "request": "launch", + "program": "src/main.py", + "console": "integratedTerminal", + "justMyCode": true, + "env": { + "PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}" + } + }, + { + "name": "Advanced debug", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": true, + "env": { + "PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}", + "ENV": "production" + } + } + ] } diff --git a/.vscode/settings.json b/.vscode/settings.json index 283a410..00d5e64 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,28 +1,24 @@ { - "python.defaultInterpreterPath": ".venv/bin/python", - "files.exclude": { - "**/__pycache__": true, - ".venv": true, - }, - "black-formatter.args": [ - "--line-length", - "100" - ], - "black-formatter.showNotification": "off", - "[python]": { - "editor.defaultFormatter": "ms-python.black-formatter", - "editor.formatOnSave": true, - "editor.formatOnPaste": false, - "editor.formatOnType": true, - }, - "python.formatting.provider": "none", - "editor.formatOnSave": true, - "editor.formatOnPaste": false, - "editor.formatOnType": true, - "[html]": { - "editor.defaultFormatter": "esbenp.prettier-vscode" - }, - "[json]": { - "editor.defaultFormatter": "esbenp.prettier-vscode" - }, + "python.defaultInterpreterPath": ".venv/bin/python", + "files.exclude": { + "**/__pycache__": true, + ".venv": true + }, + "black-formatter.args": ["--line-length", "100"], + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter", + "editor.formatOnSave": true, + "editor.formatOnPaste": false, + "editor.formatOnType": true + }, + "[json]": { + "editor.defaultFormatter": "esbenp.prettier-vscode", + "editor.tabSize": 4 + }, + "editor.formatOnSave": true, + "editor.formatOnPaste": false, + "editor.formatOnType": true, + "[html]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + } } diff --git a/config.json b/config.json index dc01017..88ef320 100644 --- a/config.json +++ b/config.json @@ -1,32 +1,26 @@ { - "name": "Export COCO Keypoints", - "type": "app", - "version": "2.0.0", - "description": "Converts Supervisely format to COCO Keypoints", - "categories": [ - "images", - "export" - ], - "main_script": "src/main.py", - "headless": true, - "icon": "https://github.com/supervisely-ecosystem/export-coco-keypoints/assets/119248312/c915fbea-a418-4e6b-ab74-899bcb6edd3b", - "icon_cover": true, - "poster": "https://github.com/supervisely-ecosystem/export-coco-keypoints/assets/119248312/5777a6fb-efe5-41c3-93b9-4abe92006b77", - "modal_template": "src/modal.html", - "docker_image": "supervisely/import-export:6.73.162", - "min_instance_version": "6.11.8", - "task_location": "workspace_tasks", - "modal_template_state": { - "allDatasets": true, - "datasets": [], - "selectedFilter": "all", - "selectedOutput": "images" - }, - "context_menu": { - "target": [ - "images_project", - "images_dataset" - ], - "context_root": "Download as" - } -} \ No newline at end of file + "name": "Export COCO Keypoints", + "type": "app", + "version": "2.0.0", + "description": "Converts Supervisely format to COCO Keypoints", + "categories": ["images", "export"], + "main_script": "src/main.py", + "headless": true, + "icon": "https://github.com/supervisely-ecosystem/export-coco-keypoints/assets/119248312/c915fbea-a418-4e6b-ab74-899bcb6edd3b", + "icon_cover": true, + "poster": "https://github.com/supervisely-ecosystem/export-coco-keypoints/assets/119248312/5777a6fb-efe5-41c3-93b9-4abe92006b77", + "modal_template": "src/modal.html", + "docker_image": "supervisely/import-export:6.73.259", + "min_instance_version": "6.12.12", + "task_location": "workspace_tasks", + "modal_template_state": { + "allDatasets": true, + "datasets": [], + "selectedFilter": "all", + "selectedOutput": "images" + }, + "context_menu": { + "target": ["images_project", "images_dataset"], + "context_root": "Download as" + } +} diff --git a/dev_requirements.txt b/dev_requirements.txt index 66733b0..5df0e17 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,5 +1,5 @@ python-dotenv -supervisely==6.73.162 +supervisely==6.73.259 # formatter black diff --git a/local.env b/local.env index cac8a5a..3696276 100644 --- a/local.env +++ b/local.env @@ -5,9 +5,7 @@ # DATASET_ID=75110 # ⬅️ specify when exporting from dataset SLY_APP_DATA_DIR="results/" # ⬅️ path to directory for local debugging -TEAM_ID = 431 -WORKSPACE_ID = 1019 -PROJECT_ID = 40721 +PROJECT_ID = 44123 # options: "images" "annotations" modal.state.selectedOutput="images" diff --git a/src/functions.py b/src/functions.py index 8abf1db..92104a0 100644 --- a/src/functions.py +++ b/src/functions.py @@ -1,8 +1,11 @@ import os -from typing import Dict +from typing import Dict, List, Optional, Union, Callable +import asyncio import numpy as np import supervisely as sly from supervisely.geometry import graph +from supervisely.api.annotation_api import AnnotationInfo +from tqdm import tqdm def create_project_dir(project): @@ -22,7 +25,7 @@ def create_coco_dataset(project_dir, dataset_name): return img_dir, ann_dir -def check_sly_annotations(ann_info, img_info, meta): +def check_sly_annotations(ann_info, img_info, meta, unsupported_anns: Dict): try: ann = sly.Annotation.from_json(ann_info.annotation, meta) except: @@ -36,9 +39,10 @@ def check_sly_annotations(ann_info, img_info, meta): bad_labels.append(lbl) if len(bad_labels) > 0: - sly.logger.warning( - f"{len(bad_labels)} objects with unsupported geometries in image [ID: {img_info.id}, NAME: {img_info.name}]" - ) + unsupported_anns[img_info.id] = {"name": img_info.name, "count": len(bad_labels)} + # sly.logger.warning( + # f"{len(bad_labels)} objects with unsupported geometries in image [ID: {img_info.id}, NAME: {img_info.name}]" + # ) return ann.clone(labels=new_labels) @@ -61,7 +65,7 @@ def get_keypoints_and_skeleton(obj_class): for edge in edges: skeleton.append( [ - list(nodes.keys()).index(edge["src"]) + 1, + list(nodes.keys()).index(edge["src"]) + 1, list(nodes.keys()).index(edge["dst"]) + 1, ] ) @@ -203,6 +207,32 @@ def create_coco_annotation( ), # int, indicates the number of labeled keypoints (v>0) for a given object ) ) - progress.iter_done_report() + progress(1) return coco_ann, label_id + + +def get_anns_list( + api: sly.Api, + dataset_id: int, + img_ids: List[int], + progress_cb: Optional[Union[tqdm, Callable]] = None, +) -> List[AnnotationInfo]: + async def fetch_annotations(): + tasks = [] + for batch in sly.batched(img_ids): + task = api.annotation.download_bulk_async( + dataset_id=dataset_id, image_ids=batch, progress_cb=progress_cb + ) + tasks.append(task) + ann_infos_lists = await asyncio.gather(*tasks) + return ann_infos_lists + + loop = sly.utils.get_or_create_event_loop() + if loop.is_running(): + future = asyncio.run_coroutine_threadsafe(fetch_annotations(), loop) + ann_infos_lists = future.result() + else: + ann_infos_lists = loop.run_until_complete(fetch_annotations()) + ann_infos = [ann_info for ann_infos in ann_infos_lists for ann_info in ann_infos] + return ann_infos diff --git a/src/main.py b/src/main.py index d2b62a3..1a381f7 100644 --- a/src/main.py +++ b/src/main.py @@ -11,7 +11,6 @@ import workflow as w import asyncio -from tinytimer import Timer if sly.is_development(): load_dotenv("local.env") @@ -26,6 +25,7 @@ api = sly.Api.from_env() + class MyExport(sly.app.Export): def process(self, context: sly.app.Export.Context): project = api.project.get_info_by_id(id=context.project_id) @@ -33,7 +33,7 @@ def process(self, context: sly.app.Export.Context): datasets = [api.dataset.get_info_by_id(context.dataset_id)] w.workflow_input(api, datasets[0].id, type="dataset") elif len(selected_datasets) > 0 and not all_datasets: - datasets = [api.dataset.get_info_by_id(dataset_id) for dataset_id in selected_datasets] + datasets = [api.dataset.get_info_by_id(dataset_id) for dataset_id in selected_datasets] if len(datasets) == 1: w.workflow_input(api, datasets[0].id, type="dataset") else: @@ -62,42 +62,40 @@ def process(self, context: sly.app.Export.Context): if selected_output == "images": image_ids = [image_info.id for image_info in images] paths = [os.path.join(img_dir, image_info.name) for image_info in images] - if api.server_address.startswith("https://"): - semaphore = asyncio.Semaphore(100) - else: - semaphore = None - - with Timer() as t: - coro = api.image.download_paths_async(image_ids, paths, semaphore) - loop = sly.utils.get_or_create_event_loop() - if loop.is_running(): - future = asyncio.run_coroutine_threadsafe(coro, loop) - future.result() - else: - loop.run_until_complete(coro) - sly.logger.info( - f"Downloading time: {t.elapsed:.4f} seconds per {len(image_ids)} images ({t.elapsed/len(image_ids):.4f} seconds per image)" - ) - pbar = sly.Progress(f"Converting dataset: {dataset.name}", total_cnt=len(images)) - for batch in sly.batched(images): - ann_infos = api.annotation.download_batch(dataset.id, image_ids) - anns = [] - for ann_info, img_info in zip(ann_infos, batch): - ann = f.check_sly_annotations(ann_info, img_info, project_meta) - anns.append(ann) - - coco_ann, label_id = f.create_coco_annotation( - coco_ann, - label_id, - project_meta, - dataset, - categories_mapping, - USER_NAME, - batch, - anns, - pbar, + di_pbar = sly.tqdm_sly(desc=f"Downloading images", total=len(image_ids)) + coro = api.image.download_paths_async(image_ids, paths, progress_cb=di_pbar) + loop = sly.utils.get_or_create_event_loop() + if loop.is_running(): + future = asyncio.run_coroutine_threadsafe(coro, loop) + future.result() + else: + loop.run_until_complete(coro) + + da_pbar = sly.tqdm_sly(desc=f"Downloading annotaions", total=len(image_ids)) + ann_infos = f.get_anns_list(api, dataset.id, image_ids, progress_cb=da_pbar) + anns = [] + + pbar = sly.tqdm_sly(desc=f"Converting dataset {dataset.name} items", total=len(images)) + unsupported_anns = {} + for ann_info, img_info in zip(ann_infos, images): + ann = f.check_sly_annotations(ann_info, img_info, project_meta, unsupported_anns) + anns.append(ann) + if unsupported_anns: + sly.logger.warning( + f"Objects with unsupported geometries for images were found: {json.dumps(unsupported_anns, indent=2)}" ) + coco_ann, label_id = f.create_coco_annotation( + coco_ann, + label_id, + project_meta, + dataset, + categories_mapping, + USER_NAME, + images, + anns, + pbar, + ) with open(os.path.join(ann_dir, "instances.json"), "w") as file: json.dump(coco_ann, file)