Merge branch 'main' into benchmark

supervisely-ecosystem · Dec 13, 2024 · 102ba38 · 102ba38
2 parents 1288e3a + 53c1eea
commit 102ba38
Show file tree

Hide file tree

Showing 8 changed files with 91 additions and 103 deletions.
diff --git a/dev_requirements.txt b/dev_requirements.txt
@@ -1,4 +1,4 @@
-supervisely==6.73.178
+supervisely==6.73.242
 
 openmim
 ffmpeg-python==0.2.0

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -16,15 +16,15 @@ RUN pip3 install yapf==0.40.1
 # COPY dev_requirements.txt dev_requirements.txt
 # RUN pip3 install -r dev_requirements.txt
 
-RUN pip3 install supervisely==6.73.242
 RUN pip3 install setuptools==69.5.1
 
 RUN pip3 install openmim
 RUN pip3 install ffmpeg-python==0.2.0
 RUN pip3 install pyyaml==6.0
 
 RUN mkdir -p /tmp/mmseg \
-    && wget https://github.com/open-mmlab/mmsegmentation/archive/refs/tags/v0.23.0.tar.gz -P /tmp/mmseg \
-    && tar -xvf /tmp/mmseg/v0.23.0.tar.gz -C /tmp/mmseg
+&& wget https://github.com/open-mmlab/mmsegmentation/archive/refs/tags/v0.23.0.tar.gz -P /tmp/mmseg \
+&& tar -xvf /tmp/mmseg/v0.23.0.tar.gz -C /tmp/mmseg
 
+RUN pip3 install supervisely==6.73.242
 LABEL python_sdk_version=6.73.242
diff --git a/serve/README.md b/serve/README.md
@@ -13,11 +13,11 @@
   <a href="#Acknowledgment">Acknowledgment</a>
 </p>
 
-[![](https://img.shields.io/badge/supervisely-ecosystem-brightgreen)](https://ecosystem.supervise.ly/apps/supervisely-ecosystem/mmsegmentation/serve)
-[![](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://supervise.ly/slack)
+[![](https://img.shields.io/badge/supervisely-ecosystem-brightgreen)](https://ecosystem.supervisely.com/apps/supervisely-ecosystem/mmsegmentation/serve)
+[![](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://supervisely.com/slack)
 ![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/supervisely-ecosystem/mmsegmentation)
-[![views](https://app.supervise.ly/img/badges/views/supervisely-ecosystem/mmsegmentation/serve.png)](https://supervise.ly)
-[![runs](https://app.supervise.ly/img/badges/runs/supervisely-ecosystem/mmsegmentation/serve.png)](https://supervise.ly)
+[![views](https://app.supervisely.com/img/badges/views/supervisely-ecosystem/mmsegmentation/serve.png)](https://supervisely.com)
+[![runs](https://app.supervisely.com/img/badges/runs/supervisely-ecosystem/mmsegmentation/serve.png)](https://supervisely.com)
 
 </div>
 
@@ -87,7 +87,7 @@ Supported methods:
 
 # How to Run
 
-**Step 1.** Add [Serve MMSegmentation](https://ecosystem.supervise.ly/apps/supervisely-ecosystem/mmsegmentation/serve) app to your team from Ecosystem
+**Step 1.** Add [Serve MMSegmentation](https://ecosystem.supervisely.com/apps/supervisely-ecosystem/mmsegmentation/serve) app to your team from Ecosystem
 
 <img data-key="sly-module-link" data-module-slug="supervisely-ecosystem/mmsegmentation/serve" src="https://i.imgur.com/e1SsMJh.png" width="350px" style='padding-bottom: 10px'/>
 
@@ -110,7 +110,7 @@ Supported methods:
 
 **Custom models**
 
-Model and directory structure must be acquired via [Train MMSegmentation](https://ecosystem.supervise.ly/apps/supervisely-ecosystem/mmsegmentation/train) app or manually created with the same directory structure
+Model and directory structure must be acquired via [Train MMSegmentation](https://ecosystem.supervisely.com/apps/supervisely-ecosystem/mmsegmentation/train) app or manually created with the same directory structure
 
 <img src="https://github.com/supervisely-ecosystem/mmsegmentation/releases/download/v0.0.1/custom_weights_guide-min.gif" style="width: 100%"/>
 
@@ -124,19 +124,19 @@ You can use your trained models outside Supervisely platform without any depende
 
 You can use served model in next Supervisely Applications ⬇️ 
 
-- [Train MMSegmentation](https://ecosystem.supervise.ly/apps/supervisely-ecosystem/mmsegmentation/train) - app allows to play with different inference options, monitor metrics charts in real time, and save training artifacts to Team Files.
+- [Train MMSegmentation](https://ecosystem.supervisely.com/apps/supervisely-ecosystem/mmsegmentation/train) - app allows to play with different inference options, monitor metrics charts in real time, and save training artifacts to Team Files.
 
     <img data-key="sly-module-link" data-module-slug="supervisely-ecosystem/mmsegmentation/train" src="https://i.imgur.com/e2r6ccw.png" width="350px"/>
 
-- [Apply NN to images project ](https://ecosystem.supervise.ly/apps/supervisely-ecosystem%252Fnn-image-labeling%252Fproject-dataset) - app allows to play with different inference options and visualize predictions in real time.  Once you choose inference settings you can apply model to all images in your project to visually analyse predictions and perform automatic data pre-labeling.   
+- [Apply NN to images project ](https://ecosystem.supervisely.com/apps/supervisely-ecosystem%252Fnn-image-labeling%252Fproject-dataset) - app allows to play with different inference options and visualize predictions in real time.  Once you choose inference settings you can apply model to all images in your project to visually analyse predictions and perform automatic data pre-labeling.   
 
     <img data-key="sly-module-link" data-module-slug="supervisely-ecosystem/nn-image-labeling/project-dataset" src="https://i.imgur.com/M2Tp8lE.png" width="350px"/> 
 
-- [Apply NN to Videos Project](https://ecosystem.supervise.ly/apps/apply-nn-to-videos-project) - app allows to label your videos using served Supervisely models.  
+- [Apply NN to Videos Project](https://ecosystem.supervisely.com/apps/apply-nn-to-videos-project) - app allows to label your videos using served Supervisely models.  
   <img data-key="sly-module-link" data-module-slug="supervisely-ecosystem/apply-nn-to-videos-project" src="https://imgur.com/LDo8K1A.png" height="54px" />
 
 
-- [NN Image Labeling](https://ecosystem.supervise.ly/apps/supervisely-ecosystem%252Fnn-image-labeling%252Fannotation-tool) - integrate any deployd NN to Supervisely Image Labeling UI. Configure inference settings and model output classes. Press `Apply` button (or use hotkey) and detections with their confidences will immediately appear on the image. 
+- [NN Image Labeling](https://ecosystem.supervisely.com/apps/supervisely-ecosystem%252Fnn-image-labeling%252Fannotation-tool) - integrate any deployd NN to Supervisely Image Labeling UI. Configure inference settings and model output classes. Press `Apply` button (or use hotkey) and detections with their confidences will immediately appear on the image. 
 
     <img data-key="sly-module-link" data-module-slug="supervisely-ecosystem/nn-image-labeling/annotation-tool" src="https://i.imgur.com/hYEucNt.png" width="350px"/>
 

diff --git a/serve/config.json b/serve/config.json
@@ -11,8 +11,8 @@
 		"serve"
 	],
 	"description": "Deploy model as REST API service",
-  	"docker_image": "supervisely/mmseg:1.3.18",
-  	"min_instance_version": "6.12.12",
+	"docker_image": "supervisely/mmseg:1.3.18",
+	"min_instance_version": "6.12.12",
 	"entrypoint": "python -m uvicorn main:m.app --app-dir ./serve/src --host 0.0.0.0 --port 8000 --ws websockets",
 	"port": 8000,
 	"task_location": "application_sessions",

diff --git a/train/README.md b/train/README.md
@@ -14,11 +14,11 @@
   <a href="#Acknowledgment">Acknowledgment</a>
 </p>
 
-[![](https://img.shields.io/badge/supervisely-ecosystem-brightgreen)](https://ecosystem.supervise.ly/apps/supervisely-ecosystem/mmsegmentation/train)
-[![](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://supervise.ly/slack)
+[![](https://img.shields.io/badge/supervisely-ecosystem-brightgreen)](https://ecosystem.supervisely.com/apps/supervisely-ecosystem/mmsegmentation/train)
+[![](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://supervisely.com/slack)
 ![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/supervisely-ecosystem/mmsegmentation)
-[![views](https://app.supervise.ly/img/badges/views/supervisely-ecosystem/mmsegmentation/train.png)](https://supervise.ly)
-[![runs](https://app.supervise.ly/img/badges/runs/supervisely-ecosystem/mmsegmentation/train.png)](https://supervise.ly)
+[![views](https://app.supervisely.com/img/badges/views/supervisely-ecosystem/mmsegmentation/train.png)](https://supervisely.com)
+[![runs](https://app.supervisely.com/img/badges/runs/supervisely-ecosystem/mmsegmentation/train.png)](https://supervisely.com)
 
 </div>
 
@@ -93,10 +93,10 @@ Supported methods:
 
 # How to Run
 
-### 1. Add [Train MMSegmentation](https://ecosystem.supervise.ly/apps/supervisely-ecosystem/mmsegmentation/train) app to your team from Ecosystem
+### 1. Add [Train MMSegmentation](https://ecosystem.supervisely.com/apps/supervisely-ecosystem/mmsegmentation/train) app to your team from Ecosystem
 <img data-key="sly-module-link" data-module-slug="supervisely-ecosystem/mmsegmentation/train" src="https://i.imgur.com/e2r6ccw.png" width="350px" style='padding-bottom: 10px'/>
 
-### 2. Run app from context menu of the project with annotations (polygon and bitmap only) [(example)](https://ecosystem.supervise.ly/projects/lemons-annotated)
+### 2. Run app from context menu of the project with annotations (polygon and bitmap only) [(example)](https://ecosystem.supervisely.com/projects/lemons-annotated)
 <img src="https://i.imgur.com/XczjaNy.png" width="100%" style='padding-top: 10px'>  
 
 # Demo

diff --git a/train/config.json b/train/config.json
@@ -1,33 +1,33 @@
 {
-  "name": "Train MMSegmentation",
-  "type": "app",
-  "categories": [
-    "neural network",
-    "images",
-    "videos",
-    "semantic segmentation",
-    "segmentation & tracking",
-    "train"
-  ],
-  "description": "Dashboard to configure, start and monitor training",
-  "docker_image": "supervisely/mmseg:1.3.18",
-  "min_instance_version": "6.12.12",
-  "main_script": "train/src/main.py",
-  "gui_template": "train/src/gui.html",
-  "task_location": "workspace_tasks",
-  "need_gpu": true,
-  "gpu": "required",
-  "isolate": true,
-  "icon": "https://i.imgur.com/GaEFmkH.png",
-  "icon_cover": true,
-  "context_menu": {
-    "target": ["images_project"],
-    "context_root": "Neural Networks",
-    "context_category": "MM Segmentation"
-  },
-  "poster": "https://user-images.githubusercontent.com/48245050/182847473-9a35f213-c27b-4abd-bd64-c73bf80fb056.jpg",
-  "community_agent": false,
-  "license": {
-    "type": "Apache-2.0"
-  }
+	"name": "Train MMSegmentation",
+	"type": "app",
+	"categories": [
+		"neural network",
+		"images",
+		"videos",
+		"semantic segmentation",
+		"segmentation & tracking",
+		"train"
+	],
+	"description": "Dashboard to configure, start and monitor training",
+	"docker_image": "supervisely/mmseg:1.3.18",
+	"min_instance_version": "6.12.12",
+	"main_script": "train/src/main.py",
+	"gui_template": "train/src/gui.html",
+	"task_location": "workspace_tasks",
+	"need_gpu": true,
+	"gpu": "required",
+	"isolate": true,
+	"icon": "https://i.imgur.com/GaEFmkH.png",
+	"icon_cover": true,
+	"context_menu": {
+		"target": ["images_project"],
+		"context_root": "Neural Networks",
+		"context_category": "MM Segmentation"
+	},
+	"poster": "https://user-images.githubusercontent.com/48245050/182847473-9a35f213-c27b-4abd-bd64-c73bf80fb056.jpg",
+	"community_agent": false,
+	"license": {
+		"type": "Apache-2.0"
+	}
 }
diff --git a/train/src/sly_project_cached.py b/train/src/sly_project_cached.py
@@ -4,24 +4,36 @@
 from supervisely.project.download import (
     download_to_cache,
     copy_from_cache,
-    is_cached,
     get_cache_size,
+    download_async
 )
 from sly_train_progress import get_progress_cb
 import sly_globals as g
 
 
 def _no_cache_download(api: sly.Api, project_info: sly.ProjectInfo, project_dir: str, progress_index: int):
     total = project_info.items_count
-    download_progress = get_progress_cb(progress_index, "Downloading input data...", total * 2)
-    sly.download(
-        api=api,
-        project_id=project_info.id,
-        dest_dir=project_dir,
-        dataset_ids=None,
-        log_progress=True,
-        progress_cb=download_progress,
-    )
+    try:
+        download_progress = get_progress_cb(progress_index, "Downloading input data...", total)
+        download_async(
+            api,
+            project_info.id,
+            project_dir,
+            progress_cb=download_progress
+        )
+    except Exception as e:
+        api.logger.warning(
+            "Failed to download project using async download. Trying sync download..."
+        )
+        download_progress = get_progress_cb(progress_index, "Downloading input data...", total)
+        sly.download(
+            api=api,
+            project_id=project_info.id,
+            dest_dir=project_dir,
+            dataset_ids=None,
+            log_progress=True,
+            progress_cb=download_progress,
+        )
 
 def download_project(
     api: sly.Api,
@@ -37,47 +49,23 @@ def download_project(
         return
 
     try:
-        # get datasets to download and cached
-        dataset_infos = api.dataset.get_list(project_info.id)
-        to_download = [info for info in dataset_infos if not is_cached(project_info.id, info.name)]
-        cached = [info for info in dataset_infos if is_cached(project_info.id, info.name)]
-        if len(cached) == 0:
-            log_msg = "No cached datasets found"
-        else:
-            log_msg = "Using cached datasets: " + ", ".join(
-                f"{ds_info.name} ({ds_info.id})" for ds_info in cached
-            )
-        sly.logger.info(log_msg)
-        if len(to_download) == 0:
-            log_msg = "All datasets are cached. No datasets to download"
-        else:
-            log_msg = "Downloading datasets: " + ", ".join(
-                f"{ds_info.name} ({ds_info.id})" for ds_info in to_download
-            )
-        sly.logger.info(log_msg)
-        # get images count
-        total = sum([ds_info.images_count for ds_info in dataset_infos])
+        total = project_info.items_count
         # download
-        if total > 0:
-            download_progress = get_progress_cb(progress_index, "Downloading input data...", total * 2)
-            download_to_cache(
-                api=api,
-                project_id=project_info.id,
-                dataset_infos=dataset_infos,
-                log_progress=True,
-                progress_cb=download_progress,
-            )
+        download_progress = get_progress_cb(progress_index, "Downloading input data...", total)
+        download_to_cache(
+            api=api,
+            project_id=project_info.id,
+            log_progress=True,
+            progress_cb=download_progress,
+        )
         # copy datasets from cache
-        total = sum([get_cache_size(project_info.id, ds.name) for ds in dataset_infos])
-        dataset_names = [ds_info.name for ds_info in dataset_infos]
-        if total > 0:
-            download_progress = get_progress_cb(progress_index, "Retreiving data from cache...", total, is_size=True)
-            copy_from_cache(
-                project_id=project_info.id,
-                dest_dir=project_dir,
-                dataset_names=dataset_names,
-                progress_cb=download_progress,
-            )
+        total = get_cache_size(project_info.id)
+        download_progress = get_progress_cb(progress_index, "Retreiving data from cache...", total, is_size=True)
+        copy_from_cache(
+            project_id=project_info.id,
+            dest_dir=project_dir,
+            progress_cb=download_progress,
+        )
     except Exception:
         sly.logger.warning(f"Failed to retreive project from cache. Downloading it...", exc_info=True)
         if os.path.exists(project_dir):

diff --git a/train/src/ui/monitoring.py b/train/src/ui/monitoring.py
@@ -107,7 +107,7 @@ def init_devices():
             "free": free_mem,
         }
         devices.append(device_info)
-    return devices
+    return sorted(devices, key=lambda x: x["free"], reverse=True)
 
 
 def init_chart(