Update dev requirements and configuration files;

Refactor downloads and error message for unsupported geometries
supervisely-ecosystem · Dec 20, 2024 · 731fb7b · 731fb7b
1 parent 2883a04
commit 731fb7b
Show file tree

Hide file tree

Showing 7 changed files with 146 additions and 131 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -1,29 +1,28 @@
 {
-  "version": "0.2.0",
-  "configurations": [
-    {
-      "name": "Debug",
-      "type": "python",
-      "request": "launch",
-      "program": "${file}",
-      "console": "integratedTerminal",
-      "justMyCode": true,
-      "env": {
-        "PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}",
-        "ENV": "development"
-      }
-    },
-    {
-      "name": "Advanced debug",
-      "type": "python",
-      "request": "launch",
-      "program": "${file}",
-      "console": "integratedTerminal",
-      "justMyCode": true,
-      "env": {
-        "PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}",
-        "ENV": "production"
-      }
-    }
-  ]
+	"version": "0.2.0",
+	"configurations": [
+		{
+			"name": "Python: main.py",
+			"type": "debugpy",
+			"request": "launch",
+			"program": "src/main.py",
+			"console": "integratedTerminal",
+			"justMyCode": true,
+			"env": {
+				"PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}"
+			}
+		},
+		{
+			"name": "Advanced debug",
+			"type": "debugpy",
+			"request": "launch",
+			"program": "${file}",
+			"console": "integratedTerminal",
+			"justMyCode": true,
+			"env": {
+				"PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}",
+				"ENV": "production"
+			}
+		}
+	]
 }
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -1,28 +1,24 @@
 {
-  "python.defaultInterpreterPath": ".venv/bin/python",
-  "files.exclude": {
-    "**/__pycache__": true,
-    ".venv": true,
-  },
-  "black-formatter.args": [
-    "--line-length",
-    "100"
-  ],
-  "black-formatter.showNotification": "off",
-  "[python]": {
-    "editor.defaultFormatter": "ms-python.black-formatter",
-    "editor.formatOnSave": true,
-    "editor.formatOnPaste": false,
-    "editor.formatOnType": true,
-  },
-  "python.formatting.provider": "none",
-  "editor.formatOnSave": true,
-  "editor.formatOnPaste": false,
-  "editor.formatOnType": true,
-  "[html]": {
-    "editor.defaultFormatter": "esbenp.prettier-vscode"
-  },
-  "[json]": {
-    "editor.defaultFormatter": "esbenp.prettier-vscode"
-  },
+	"python.defaultInterpreterPath": ".venv/bin/python",
+	"files.exclude": {
+		"**/__pycache__": true,
+		".venv": true
+	},
+	"black-formatter.args": ["--line-length", "100"],
+	"[python]": {
+		"editor.defaultFormatter": "ms-python.black-formatter",
+		"editor.formatOnSave": true,
+		"editor.formatOnPaste": false,
+		"editor.formatOnType": true
+	},
+	"[json]": {
+		"editor.defaultFormatter": "esbenp.prettier-vscode",
+		"editor.tabSize": 4
+	},
+	"editor.formatOnSave": true,
+	"editor.formatOnPaste": false,
+	"editor.formatOnType": true,
+	"[html]": {
+		"editor.defaultFormatter": "esbenp.prettier-vscode"
+	}
 }
diff --git a/config.json b/config.json
@@ -1,32 +1,26 @@
 {
-  "name": "Export COCO Keypoints",
-  "type": "app",
-  "version": "2.0.0",
-  "description": "Converts Supervisely format to COCO Keypoints",
-  "categories": [
-    "images",
-    "export"
-  ],
-  "main_script": "src/main.py",
-  "headless": true,
-  "icon": "https://github.com/supervisely-ecosystem/export-coco-keypoints/assets/119248312/c915fbea-a418-4e6b-ab74-899bcb6edd3b",
-  "icon_cover": true,
-  "poster": "https://github.com/supervisely-ecosystem/export-coco-keypoints/assets/119248312/5777a6fb-efe5-41c3-93b9-4abe92006b77",
-  "modal_template": "src/modal.html",
-  "docker_image": "supervisely/import-export:6.73.162",
-  "min_instance_version": "6.11.8",
-  "task_location": "workspace_tasks",
-  "modal_template_state": {
-    "allDatasets": true,
-    "datasets": [],
-    "selectedFilter": "all",
-    "selectedOutput": "images"
-  },
-  "context_menu": {
-    "target": [
-      "images_project",
-      "images_dataset"
-    ],
-    "context_root": "Download as"
-  }
-}
+	"name": "Export COCO Keypoints",
+	"type": "app",
+	"version": "2.0.0",
+	"description": "Converts Supervisely format to COCO Keypoints",
+	"categories": ["images", "export"],
+	"main_script": "src/main.py",
+	"headless": true,
+	"icon": "https://github.com/supervisely-ecosystem/export-coco-keypoints/assets/119248312/c915fbea-a418-4e6b-ab74-899bcb6edd3b",
+	"icon_cover": true,
+	"poster": "https://github.com/supervisely-ecosystem/export-coco-keypoints/assets/119248312/5777a6fb-efe5-41c3-93b9-4abe92006b77",
+	"modal_template": "src/modal.html",
+	"docker_image": "supervisely/import-export:6.73.259",
+	"min_instance_version": "6.12.12",
+	"task_location": "workspace_tasks",
+	"modal_template_state": {
+		"allDatasets": true,
+		"datasets": [],
+		"selectedFilter": "all",
+		"selectedOutput": "images"
+	},
+	"context_menu": {
+		"target": ["images_project", "images_dataset"],
+		"context_root": "Download as"
+	}
+}
diff --git a/dev_requirements.txt b/dev_requirements.txt
@@ -1,5 +1,5 @@
 python-dotenv
-supervisely==6.73.162
+supervisely==6.73.259
 # formatter
 black
 

diff --git a/local.env b/local.env
@@ -5,9 +5,7 @@
 # DATASET_ID=75110              # ⬅️ specify when exporting from dataset
 SLY_APP_DATA_DIR="results/"   # ⬅️ path to directory for local debugging
 
-TEAM_ID = 431
-WORKSPACE_ID = 1019
-PROJECT_ID = 40721
+PROJECT_ID = 44123
 
 # options: "images" "annotations"
 modal.state.selectedOutput="images"

diff --git a/src/functions.py b/src/functions.py
@@ -1,8 +1,11 @@
 import os
-from typing import Dict
+from typing import Dict, List, Optional, Union, Callable
+import asyncio
 import numpy as np
 import supervisely as sly
 from supervisely.geometry import graph
+from supervisely.api.annotation_api import AnnotationInfo
+from tqdm import tqdm
 
 
 def create_project_dir(project):
@@ -22,7 +25,7 @@ def create_coco_dataset(project_dir, dataset_name):
     return img_dir, ann_dir
 
 
-def check_sly_annotations(ann_info, img_info, meta):
+def check_sly_annotations(ann_info, img_info, meta, unsupported_anns: Dict):
     try:
         ann = sly.Annotation.from_json(ann_info.annotation, meta)
     except:
@@ -36,9 +39,10 @@ def check_sly_annotations(ann_info, img_info, meta):
             bad_labels.append(lbl)
 
     if len(bad_labels) > 0:
-        sly.logger.warning(
-            f"{len(bad_labels)} objects with unsupported geometries in image [ID: {img_info.id}, NAME: {img_info.name}]"
-        )
+        unsupported_anns[img_info.id] = {"name": img_info.name, "count": len(bad_labels)}
+        # sly.logger.warning(
+        #     f"{len(bad_labels)} objects with unsupported geometries in image [ID: {img_info.id}, NAME: {img_info.name}]"
+        # )
     return ann.clone(labels=new_labels)
 
 
@@ -61,7 +65,7 @@ def get_keypoints_and_skeleton(obj_class):
     for edge in edges:
         skeleton.append(
             [
-                list(nodes.keys()).index(edge["src"]) + 1, 
+                list(nodes.keys()).index(edge["src"]) + 1,
                 list(nodes.keys()).index(edge["dst"]) + 1,
             ]
         )
@@ -203,6 +207,32 @@ def create_coco_annotation(
                         ),  # int, indicates the number of labeled keypoints (v>0) for a given object
                     )
                 )
-        progress.iter_done_report()
+        progress(1)
 
     return coco_ann, label_id
+
+
+def get_anns_list(
+    api: sly.Api,
+    dataset_id: int,
+    img_ids: List[int],
+    progress_cb: Optional[Union[tqdm, Callable]] = None,
+) -> List[AnnotationInfo]:
+    async def fetch_annotations():
+        tasks = []
+        for batch in sly.batched(img_ids):
+            task = api.annotation.download_bulk_async(
+                dataset_id=dataset_id, image_ids=batch, progress_cb=progress_cb
+            )
+            tasks.append(task)
+        ann_infos_lists = await asyncio.gather(*tasks)
+        return ann_infos_lists
+
+    loop = sly.utils.get_or_create_event_loop()
+    if loop.is_running():
+        future = asyncio.run_coroutine_threadsafe(fetch_annotations(), loop)
+        ann_infos_lists = future.result()
+    else:
+        ann_infos_lists = loop.run_until_complete(fetch_annotations())
+    ann_infos = [ann_info for ann_infos in ann_infos_lists for ann_info in ann_infos]
+    return ann_infos
diff --git a/src/main.py b/src/main.py
@@ -11,7 +11,6 @@
 import workflow as w
 
 import asyncio
-from tinytimer import Timer
 
 if sly.is_development():
     load_dotenv("local.env")
@@ -26,14 +25,15 @@
 
 api = sly.Api.from_env()
 
+
 class MyExport(sly.app.Export):
     def process(self, context: sly.app.Export.Context):
         project = api.project.get_info_by_id(id=context.project_id)
         if context.dataset_id is not None:
             datasets = [api.dataset.get_info_by_id(context.dataset_id)]
             w.workflow_input(api, datasets[0].id, type="dataset")
         elif len(selected_datasets) > 0 and not all_datasets:
-            datasets = [api.dataset.get_info_by_id(dataset_id) for dataset_id in selected_datasets]            
+            datasets = [api.dataset.get_info_by_id(dataset_id) for dataset_id in selected_datasets]
             if len(datasets) == 1:
                 w.workflow_input(api, datasets[0].id, type="dataset")
             else:
@@ -62,42 +62,40 @@ def process(self, context: sly.app.Export.Context):
             if selected_output == "images":
                 image_ids = [image_info.id for image_info in images]
                 paths = [os.path.join(img_dir, image_info.name) for image_info in images]
-                if api.server_address.startswith("https://"):
-                    semaphore = asyncio.Semaphore(100)
-                else:
-                    semaphore = None
-
-                with Timer() as t:
-                    coro = api.image.download_paths_async(image_ids, paths, semaphore)
-                    loop = sly.utils.get_or_create_event_loop()
-                    if loop.is_running():
-                        future = asyncio.run_coroutine_threadsafe(coro, loop)
-                        future.result()
-                    else:
-                        loop.run_until_complete(coro)
-                sly.logger.info(
-                    f"Downloading time: {t.elapsed:.4f} seconds per {len(image_ids)} images  ({t.elapsed/len(image_ids):.4f} seconds per image)"
-                )
 
-            pbar = sly.Progress(f"Converting dataset: {dataset.name}", total_cnt=len(images))
-            for batch in sly.batched(images):
-                ann_infos = api.annotation.download_batch(dataset.id, image_ids)
-                anns = []
-                for ann_info, img_info in zip(ann_infos, batch):
-                    ann = f.check_sly_annotations(ann_info, img_info, project_meta)
-                    anns.append(ann)
-
-                coco_ann, label_id = f.create_coco_annotation(
-                    coco_ann,
-                    label_id,
-                    project_meta,
-                    dataset,
-                    categories_mapping,
-                    USER_NAME,
-                    batch,
-                    anns,
-                    pbar,
+                di_pbar = sly.tqdm_sly(desc=f"Downloading images", total=len(image_ids))
+                coro = api.image.download_paths_async(image_ids, paths, progress_cb=di_pbar)
+                loop = sly.utils.get_or_create_event_loop()
+                if loop.is_running():
+                    future = asyncio.run_coroutine_threadsafe(coro, loop)
+                    future.result()
+                else:
+                    loop.run_until_complete(coro)
+
+            da_pbar = sly.tqdm_sly(desc=f"Downloading annotaions", total=len(image_ids))
+            ann_infos = f.get_anns_list(api, dataset.id, image_ids, progress_cb=da_pbar)
+            anns = []
+
+            pbar = sly.tqdm_sly(desc=f"Converting dataset {dataset.name} items", total=len(images))
+            unsupported_anns = {}
+            for ann_info, img_info in zip(ann_infos, images):
+                ann = f.check_sly_annotations(ann_info, img_info, project_meta, unsupported_anns)
+                anns.append(ann)
+            if unsupported_anns:
+                sly.logger.warning(
+                    f"Objects with unsupported geometries for images were found: {json.dumps(unsupported_anns, indent=2)}"
                 )
+            coco_ann, label_id = f.create_coco_annotation(
+                coco_ann,
+                label_id,
+                project_meta,
+                dataset,
+                categories_mapping,
+                USER_NAME,
+                images,
+                anns,
+                pbar,
+            )
 
             with open(os.path.join(ann_dir, "instances.json"), "w") as file:
                 json.dump(coco_ann, file)