add async

.vscode/launch.json

@@ -1,6 +1,17 @@
 {
     "version": "0.2.0",
     "configurations": [
+            {
+      "name": "Python: Current File",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${file}",
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}"
+      }
+    },
         {
             "name": "Launch app",
             "type": "python",

local.env

@@ -4,11 +4,15 @@ LOG_LEVEL="debug"
 
 TASK_ID=22046
 
-CONTEXT_TEAMID=8
-CONTEXT_WORKSPACEID=349
+#CONTEXT_TEAMID=8
+#CONTEXT_WORKSPACEID=349
+#
+#CONTEXT_PROJECTID=14278
+#CONTEXT_DATASETID=51995
 
-CONTEXT_PROJECTID=14278
-CONTEXT_DATASETID=51995
+TEAM_ID = 431
+WORKSPACE_ID = 1019
+PROJECT_ID = 40721
 
 DEBUG_APP_DIR="src/debug/app_debug_data"
 DEBUG_CACHE_DIR="src/debug/app_debug_cache"

src/main.py

@@ -12,6 +12,9 @@
 from src.globals import DATASET_ID, PROJECT_DIR, PROJECT_ID, api
 import src.workflow as w
 
+from tinytimer import Timer
+import asyncio
+
 @sly.handle_exceptions(has_ui=False)
 def main():
     project = api.project.get_info_by_id(id=PROJECT_ID)
@@ -25,6 +28,12 @@ def main():
         datasets = api.dataset.get_list(project_id=PROJECT_ID)
         w.workflow_input(api, PROJECT_ID, type="project")
     progress_ds = sly.Progress(message="Exporting datasets", total_cnt=len(datasets))
+
+    if api.server_address.startswith("https://"):
+        semaphore = asyncio.Semaphore(100)
+    else:
+        semaphore = None
+
     for dataset in datasets:
         dataset_dir = os.path.join(PROJECT_DIR, dataset.name)
         images_dir = os.path.join(dataset_dir, "images")
@@ -42,10 +51,20 @@ def main():
             for img_info in images_infos
         ]
 
+        with Timer() as t:
+            coro = api.image.download_paths_async(images_ids, images_paths, semaphore)
+            loop = sly.utils.get_or_create_event_loop()
+            if loop.is_running():
+                future = asyncio.run_coroutine_threadsafe(coro, loop)
+                future.result()
+            else:
+                loop.run_until_complete(coro)
+        sly.logger.info(
+            f"Downloading time: {t.elapsed:.4f} seconds per {len(images_ids)} images  ({t.elapsed/len(images_ids):.4f} seconds per image)"
+        )
+
         progress_img = sly.Progress(message="Processing images", total_cnt=len(images_ids))
-        for batch_img_ids, batch_img_paths, batch_anns, batch_anns_paths in zip(
-            sly.batched(images_ids),
-            sly.batched(images_paths),
+        for batch_anns, batch_anns_paths in zip(
             sly.batched(anns),
             sly.batched(anns_paths),
         ):
@@ -54,10 +73,7 @@ def main():
                 anns=batch_anns,
                 project_meta=project_meta,
             )
-            api.image.download_paths(
-                dataset_id=dataset.id, ids=batch_img_ids, paths=batch_img_paths
-            )
-            progress_img.iters_done_report(len(batch_img_ids))
+            progress_img.iters_done_report(len(batch_anns_paths))
         progress_ds.iter_done_report()
 
     file_info = upload_project_to_tf(api, project)