Merge pull request #31 from supervisely-ecosystem/download_async

Improved Items Downloading

.vscode/launch.json

@@ -3,7 +3,7 @@
     "configurations": [
         {
             "name": "Python: Current File",
-            "type": "python",
+            "type": "debugpy",
             "request": "launch",
             "program": "src/main.py",
             "console": "integratedTerminal",

.vscode/settings.json

@@ -16,7 +16,9 @@
         "editor.defaultFormatter": "esbenp.prettier-vscode"
     },
     "[json]": {
-        "editor.defaultFormatter": "esbenp.prettier-vscode"
+        "editor.defaultFormatter": "esbenp.prettier-vscode",
+        "editor.formatOnSave": true,
+        "editor.tabSize": 4,
     },
     "[python]": {
         "editor.defaultFormatter": "ms-python.black-formatter",

config.json

@@ -1,32 +1,27 @@
 {
-  "name": "Export to COCO",
-  "type": "app",
-  "version": "2.0.0",
-  "categories": [
-    "images",
-    "export"
-  ],
-  "description": "Converts Supervisely to COCO format and prepares tar archive for download",
-  "docker_image": "supervisely/import-export:6.73.162",
-  "min_instance_version": "6.11.8",
-  "main_script": "src/main.py",
-  "modal_template": "src/modal.html",
-  "task_location": "workspace_tasks",
-  "icon": "https://i.imgur.com/9uY44aW.png",
-  "icon_background": "#FFFFFF",
-  "headless": true,
-  "modal_template_state": {
-    "allDatasets": true,
-    "datasets": [],
-    "selectedFilter": "all",
-    "selectedOutput": "images",
-    "captions": false
-  },
-  "context_menu": {
-    "target": [
-      "images_project"
-    ],
-    "context_root": "Download as"
-  },
-  "poster": "https://user-images.githubusercontent.com/48913536/183899083-64d7683d-57f9-4f7a-b5f4-bf9e7ffd3246.png"
-}
+	"name": "Export to COCO",
+	"type": "app",
+	"version": "2.0.0",
+	"categories": ["images", "export"],
+	"description": "Converts Supervisely to COCO format and prepares tar archive for download",
+	"docker_image": "supervisely/import-export:6.73.257",
+	"min_instance_version": "6.12.12",
+	"main_script": "src/main.py",
+	"modal_template": "src/modal.html",
+	"task_location": "workspace_tasks",
+	"icon": "https://i.imgur.com/9uY44aW.png",
+	"icon_background": "#FFFFFF",
+	"headless": true,
+	"modal_template_state": {
+		"allDatasets": true,
+		"datasets": [],
+		"selectedFilter": "all",
+		"selectedOutput": "images",
+		"captions": false
+	},
+	"context_menu": {
+		"target": ["images_project"],
+		"context_root": "Download as"
+	},
+	"poster": "https://user-images.githubusercontent.com/48913536/183899083-64d7683d-57f9-4f7a-b5f4-bf9e7ffd3246.png"
+}

dev_requirements.txt

@@ -1 +1 @@
-supervisely==6.73.162
+supervisely==6.73.257

local.env

@@ -1,6 +1,6 @@
 TEAM_ID = 431
 WORKSPACE_ID = 1019
-PROJECT_ID = 42759
+PROJECT_ID = 40721
 
 # options: "images" "annotations"
 modal.state.selectedOutput="images"
@@ -9,3 +9,5 @@ modal.state.selectedFilter="all"
 modal.state.allDatasets=True
 modal.state.datasets=[71754]
 modal.state.captions=False
+
+LOGLEVEL = "debug"

src/convert_geometry.py

@@ -5,6 +5,7 @@
 from typing import List
 import uuid
 
+
 def prepare_meta(meta: sly.ProjectMeta):
     new_classes = []
     for cls in meta.obj_classes:

src/main.py

@@ -2,6 +2,7 @@
 import json
 import os
 from distutils.util import strtobool
+import asyncio
 
 import supervisely as sly
 from dotenv import load_dotenv
@@ -25,15 +26,17 @@
 team_id = sly.env.team_id()
 workspace_id = sly.env.workspace_id()
 project_id = sly.env.project_id()
-selected_output = os.environ["modal.state.selectedOutput"]
+
+output_mode = os.environ["modal.state.selectedOutput"]
+export_images = True if output_mode == "images" else False
 selected_filter = os.environ["modal.state.selectedFilter"]
 all_datasets = bool(strtobool(os.getenv("modal.state.allDatasets")))
 selected_datasets = ast.literal_eval(os.environ["modal.state.datasets"])
 include_captions = bool(strtobool(os.getenv("modal.state.captions")))
 # endregion
 sly.logger.info(f"Team ID: {team_id}, Workspace ID: {workspace_id}, Project ID: {project_id}")
 sly.logger.info(
-    f"Selected output: {selected_output}, "
+    f"Selected output: {output_mode}, "
     f"Selected filter: {selected_filter}, "
     f"All datasets: {all_datasets}, "
     f"Selected datasets: {selected_datasets}, "
@@ -80,15 +83,34 @@ def export_to_coco(api: sly.Api) -> None:
             total_cnt=len(images),
             min_report_percent=5,
         )
-        for batch in sly.batched(images):
-            image_ids = [image_info.id for image_info in batch]
-            sly.logger.info(f"Working with batch of {len(batch)} images with ids: {image_ids}")
 
-            if selected_output == "images":
-                image_paths = [os.path.join(img_dir, image_info.name) for image_info in batch]
-                f.download_batch_with_retry(api, dataset.id, image_ids, image_paths)
+        # to move
+        dataset_path = os.path.join(coco_dataset_dir, project.name, dataset.name)
+        os.makedirs(dataset_path, exist_ok=True)
+
+        for batch in sly.batched(images):
+            batch_ids = [info.id for info in batch]
+            batch_paths = [os.path.join(dataset_path, image_info.name) for image_info in batch]
+
+            if export_images:
+                coro = api.image.download_paths_async(batch_ids, batch_paths)
+                loop = sly.utils.get_or_create_event_loop()
+                if loop.is_running():
+                    future = asyncio.run_coroutine_threadsafe(coro, loop)
+                    future.result()
+                else:
+                    loop.run_until_complete(coro)
+
+            ann_infos = []
+
+            coro = api.annotation.download_batch_async(dataset.id, batch_ids)
+            loop = sly.utils.get_or_create_event_loop()
+            if loop.is_running():
+                future = asyncio.run_coroutine_threadsafe(coro, loop)
+                ann_infos.extend(future.result())
+            else:
+                ann_infos.extend(loop.run_until_complete(coro))
 
-            ann_infos = api.annotation.download_batch(dataset.id, image_ids)
             anns = []
             sly.logger.info(f"Preparing to convert {len(ann_infos)} annotations...")
             for ann_info, img_info in zip(ann_infos, batch):
@@ -109,6 +131,7 @@ def export_to_coco(api: sly.Api) -> None:
                 include_captions,
                 RECTANGLE_MARK,
             )
+
         with open(os.path.join(ann_dir, "instances.json"), "w") as file:
             json.dump(coco_instances, file)
         if coco_captions is not None and include_captions: