Improved Images Downloading

.vscode/launch.json

@@ -3,7 +3,7 @@
   "configurations": [
     {
       "name": "Python: Current File",
-      "type": "python",
+      "type": "debugpy",
       "request": "launch",
       "program": "${file}",
       "console": "integratedTerminal",

.vscode/settings.json

@@ -14,7 +14,8 @@
     "editor.defaultFormatter": "esbenp.prettier-vscode"
   },
   "[json]": {
-    "editor.defaultFormatter": "esbenp.prettier-vscode"
+    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.tabSize": 4
   },
   "[python]": {
     "editor.defaultFormatter": "ms-python.black-formatter"

config.json

@@ -1,30 +1,25 @@
 {
-  "name": "Convert Supervisely to YOLO v5 format",
-  "type": "app",
-  "version": "2.0.0",
-  "categories": [
-    "images",
-    "export"
-  ],
-  "description": "Transform project to YOLO v5 format and prepares tar archive for download",
-  "docker_image": "supervisely/import-export:6.73.162",
-  "instance_version": "6.11.8",
-  "main_script": "src/convert_sly_to_yolov5.py",
-  "modal_template": "src/modal.html",
-  "modal_template_state": {
-    "processShapes": "skip"
-  },
-  "gui_template": "src/gui.html",
-  "task_location": "workspace_tasks",
-  "isolate": true,
-  "headless": true,
-  "icon": "https://i.imgur.com/pz3eSzx.png",
-  "icon_background": "#FFFFFF",
-  "context_menu": {
-    "target": [
-      "images_project"
-    ],
-    "context_root": "Download as"
-  },
-  "poster": "https://user-images.githubusercontent.com/106374579/183683758-89476d80-de3f-424f-9bfa-f1562703a168.png"
-}
+	"name": "Convert Supervisely to YOLO v5 format",
+	"type": "app",
+	"version": "2.0.0",
+	"categories": ["images", "export"],
+	"description": "Transform project to YOLO v5 format and prepares tar archive for download",
+	"docker_image": "supervisely/import-export:6.73.256",
+	"instance_version": "6.12.12",
+	"main_script": "src/convert_sly_to_yolov5.py",
+	"modal_template": "src/modal.html",
+	"modal_template_state": {
+		"processShapes": "skip"
+	},
+	"gui_template": "src/gui.html",
+	"task_location": "workspace_tasks",
+	"isolate": true,
+	"headless": true,
+	"icon": "https://i.imgur.com/pz3eSzx.png",
+	"icon_background": "#FFFFFF",
+	"context_menu": {
+		"target": ["images_project"],
+		"context_root": "Download as"
+	},
+	"poster": "https://user-images.githubusercontent.com/106374579/183683758-89476d80-de3f-424f-9bfa-f1562703a168.png"
+}

dev_requirements.txt

@@ -1 +1 @@
-supervisely==6.73.162
+supervisely==6.73.256

local.env

@@ -1,3 +1,7 @@
-TEAM_ID = 448
-WORKSPACE_ID = 690
-PROJECT_ID = 35637
+# TEAM_ID = 448
+# WORKSPACE_ID = 690
+# PROJECT_ID = 35637
+
+TEAM_ID = 431
+WORKSPACE_ID = 1019
+PROJECT_ID = 40721

src/convert_sly_to_yolov5.py

@@ -3,7 +3,8 @@
 from typing import List, Tuple
 from dotenv import load_dotenv
 import supervisely as sly
-from workflow import Workflow
+from workflow import add_input, add_output
+import asyncio
 
 # region constants
 TRAIN_TAG_NAME = "train"
@@ -86,7 +87,7 @@ def transform(api: sly.Api) -> None:
         missing_tags.append(VAL_TAG_NAME)
     if len(missing_tags) > 0:
         missing_tags_str = ", ".join([f'"{tag}"' for tag in missing_tags])
-        sly.logger.warn(
+        sly.logger.warning(
             f"Tag(s): {missing_tags_str} not found in project meta. Images without special tags will be marked as train"
         )
 
@@ -95,7 +96,7 @@ def transform(api: sly.Api) -> None:
         if obj_class.geometry_type != sly.Rectangle:
             error_classes.append(obj_class)
     if len(error_classes) > 0:
-        sly.logger.warn(
+        sly.logger.warning(
             f"Project has unsupported classes. "
             f"Objects with unsupported geometry types will be {process_shapes_message}: "
             f"{[obj_class.name for obj_class in error_classes]}"
@@ -115,7 +116,7 @@ def _add_to_split(image_id, img_name, split_ids, split_image_paths, labels_dir,
     train_count = 0
     val_count = 0
 
-    progress = sly.Progress("Transformation ...", api.project.get_images_count(project_id))
+    progress = sly.Progress("Processing project items...", api.project.get_images_count(project_id))
     for dataset in api.dataset.get_list(project_id, recursive=True):
         sly.logger.info(f"Working with dataset: {dataset.name}...")
         images = api.image.get_list(dataset.id)
@@ -127,6 +128,7 @@ def _add_to_split(image_id, img_name, split_ids, split_image_paths, labels_dir,
         val_ids = []
         val_image_paths = []
 
+        a_progress = sly.Progress("Transforming annotations...", total_cnt=len(images))
         for batch in sly.batched(images):
             image_ids = [image_info.id for image_info in batch]
             image_names = [f"{dataset.id}_{dataset.name}_{image_info.name}" for image_info in batch]
@@ -177,13 +179,25 @@ def _add_to_split(image_id, img_name, split_ids, split_image_paths, labels_dir,
                         train_images_dir,
                     )
                     train_count += 1
+            a_progress.iters_done_report(len(batch))
 
-        api.image.download_paths(dataset.id, train_ids, train_image_paths)
-        api.image.download_paths(dataset.id, val_ids, val_image_paths)
+        ids_to_download = train_ids + val_ids
+        paths_to_download = train_image_paths + val_image_paths
+        i_rogress = sly.Progress("Downloading images...", total_cnt=len(ids_to_download))
+        coro = api.image.download_paths_async(
+            ids_to_download, paths_to_download, progress_cb=i_rogress.iters_done_report
+        )
+        loop = sly.utils.get_or_create_event_loop()
+        if loop.is_running():
+            future = asyncio.run_coroutine_threadsafe(coro, loop)
+            future.result()
+        else:
+            loop.run_until_complete(coro)
+
+        progress.iters_done_report(len(ids_to_download))
 
-        progress.iters_done_report(len(batch))
         if unsupported_shapes > 0:
-            sly.logger.warn(
+            sly.logger.warning(
                 f"Dataset {dataset.name}: "
                 f"{unsupported_shapes} objects with unsupported geometry types have been {process_shapes_message}"
             )
@@ -205,12 +219,15 @@ def _add_to_split(image_id, img_name, split_ids, split_image_paths, labels_dir,
     file_info = sly.output.set_download(result_dir)
     sly.logger.info("File uploaded, app stopped.")
     # --------------------------------- Add Workflow Input And Output -------------------------------- #
-    workflow.add_input(project_id)
-    workflow.add_output(file_info)
+    add_input(api, project_id)
+    add_output(api, file_info)
     # --------------------------------- Add Workflow Input And Output -------------------------------- #
 
 
 if __name__ == "__main__":
     api = sly.Api.from_env()
-    workflow = Workflow(api)
+    if api.server_address == "https://app.supervisely.com":
+        semaphore = api.get_default_semaphore()
+        if semaphore._value == 10:
+            api.set_semaphore_size(7)
     transform(api)

src/modal.html

@@ -9,15 +9,13 @@
     title="Process shapes"
     description="Select how to process shapes that are not supported by the current application."
   >
-    <div>
+    <div style="margin-top: 5px;">
       <el-radio-group v-model="state.processShapes" size="normal">
-        <div>
-          <el-radio label="transform">
-            <span>transform to Rectangle (bbox)</span></el-radio
-          >
+        <div >
+          <el-radio label="skip"><span>Skip objects of unsupported geometries</span></el-radio>
         </div>
-        <div>
-          <el-radio label="skip"><span>skip</span></el-radio>
+        <div style="margin-top: 5px;">
+          <el-radio label="transform"><span>Transform shapes to Rectangle (bbox)</span></el-radio>
         </div>
       </el-radio-group>
     </div>

src/workflow.py

@@ -1,67 +1,28 @@
 import supervisely as sly
 
 
-def check_compatibility(func):
-    def wrapper(self, *args, **kwargs):
-        if self.is_compatible is None:
-            try:
-                self.is_compatible = self.check_instance_ver_compatibility()
-            except Exception as e:
-                sly.logger.error(
-                    "Can not check compatibility with Supervisely instance. "
-                    f"Workflow features will be disabled. Error: {repr(e)}"
-                )
-                self.is_compatible = False
-        if not self.is_compatible:
-            return
-        return func(self, *args, **kwargs)
-
-    return wrapper
-
-
-class Workflow:
-    def __init__(self, api: sly.Api, min_instance_version: str = None):
-        self.is_compatible = None
-        self.api = api
-        self._min_instance_version = (
-            "6.9.31" if min_instance_version is None else min_instance_version
-        )
-
-    def check_instance_ver_compatibility(self):
-        if not self.api.is_version_supported(self._min_instance_version):
-            sly.logger.info(
-                f"Supervisely instance version {self.api.instance_version} does not support workflow features."
-            )
-            if not sly.is_community():
-                sly.logger.info(
-                    f"To use them, please update your instance to version {self._min_instance_version} or higher."
-                )
-            return False
-        return True
-
-    @check_compatibility
-    def add_input(self, project_id: int):
-        self.api.app.workflow.add_input_project(project_id)
-        sly.logger.debug(f"Workflow: Input project - {project_id}")
-
-    @check_compatibility
-    def add_output(self, file_info: sly.api.file_api.FileInfo):
-        try:
-            meta = {
-                "customRelationSettings": {
-                    "icon": {
-                        "icon": "zmdi-archive",
-                        "color": "#33c94c",
-                        "backgroundColor": "#d9f7e4",
-                    },
-                    "title": f"<h4>{file_info.name}</h4>",
-                    "mainLink": {
-                        "url": f"/files/{file_info.id}/true/?teamId={file_info.team_id}",
-                        "title": "Download",
-                    },
-                }
+def add_input(api: sly.Api, project_id: int):
+    api.app.workflow.add_input_project(project_id)
+    sly.logger.debug(f"Workflow: Input project - {project_id}")
+
+
+def add_output(api: sly.Api, file_info: sly.api.file_api.FileInfo):
+    try:
+        meta = {
+            "customRelationSettings": {
+                "icon": {
+                    "icon": "zmdi-archive",
+                    "color": "#33c94c",
+                    "backgroundColor": "#d9f7e4",
+                },
+                "title": f"<h4>{file_info.name}</h4>",
+                "mainLink": {
+                    "url": f"/files/{file_info.id}/true/?teamId={file_info.team_id}",
+                    "title": "Download",
+                },
             }
-            self.api.app.workflow.add_output_file(file_info, meta=meta)
-            sly.logger.debug(f"Workflow: Output file - {file_info.id if file_info else None}")
-        except Exception as e:
-            sly.logger.debug(f"Workflow: Can not add output file. Error: {repr(e)}")
+        }
+        api.app.workflow.add_output_file(file_info, meta=meta)
+        sly.logger.debug(f"Workflow: Output file - {file_info.id if file_info else None}")
+    except Exception as e:
+        sly.logger.debug(f"Workflow: Can not add output file. Error: {repr(e)}")