diff --git a/dev_requirements.txt b/dev_requirements.txt index e347909..31bf5c9 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,4 +1,4 @@ -supervisely==6.73.178 +supervisely==6.73.242 openmim ffmpeg-python==0.2.0 diff --git a/docker/Dockerfile b/docker/Dockerfile index 2300617..6914ed7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,7 +16,6 @@ RUN pip3 install yapf==0.40.1 # COPY dev_requirements.txt dev_requirements.txt # RUN pip3 install -r dev_requirements.txt -RUN pip3 install supervisely==6.73.242 RUN pip3 install setuptools==69.5.1 RUN pip3 install openmim @@ -24,7 +23,8 @@ RUN pip3 install ffmpeg-python==0.2.0 RUN pip3 install pyyaml==6.0 RUN mkdir -p /tmp/mmseg \ - && wget https://github.com/open-mmlab/mmsegmentation/archive/refs/tags/v0.23.0.tar.gz -P /tmp/mmseg \ - && tar -xvf /tmp/mmseg/v0.23.0.tar.gz -C /tmp/mmseg +&& wget https://github.com/open-mmlab/mmsegmentation/archive/refs/tags/v0.23.0.tar.gz -P /tmp/mmseg \ +&& tar -xvf /tmp/mmseg/v0.23.0.tar.gz -C /tmp/mmseg +RUN pip3 install supervisely==6.73.242 LABEL python_sdk_version=6.73.242 diff --git a/serve/README.md b/serve/README.md index cf9b994..4d39b71 100644 --- a/serve/README.md +++ b/serve/README.md @@ -13,11 +13,11 @@ Acknowledgment

-[![](https://img.shields.io/badge/supervisely-ecosystem-brightgreen)](https://ecosystem.supervise.ly/apps/supervisely-ecosystem/mmsegmentation/serve) -[![](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://supervise.ly/slack) +[![](https://img.shields.io/badge/supervisely-ecosystem-brightgreen)](https://ecosystem.supervisely.com/apps/supervisely-ecosystem/mmsegmentation/serve) +[![](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://supervisely.com/slack) ![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/supervisely-ecosystem/mmsegmentation) -[![views](https://app.supervise.ly/img/badges/views/supervisely-ecosystem/mmsegmentation/serve.png)](https://supervise.ly) -[![runs](https://app.supervise.ly/img/badges/runs/supervisely-ecosystem/mmsegmentation/serve.png)](https://supervise.ly) +[![views](https://app.supervisely.com/img/badges/views/supervisely-ecosystem/mmsegmentation/serve.png)](https://supervisely.com) +[![runs](https://app.supervisely.com/img/badges/runs/supervisely-ecosystem/mmsegmentation/serve.png)](https://supervisely.com) @@ -87,7 +87,7 @@ Supported methods: # How to Run -**Step 1.** Add [Serve MMSegmentation](https://ecosystem.supervise.ly/apps/supervisely-ecosystem/mmsegmentation/serve) app to your team from Ecosystem +**Step 1.** Add [Serve MMSegmentation](https://ecosystem.supervisely.com/apps/supervisely-ecosystem/mmsegmentation/serve) app to your team from Ecosystem @@ -110,7 +110,7 @@ Supported methods: **Custom models** -Model and directory structure must be acquired via [Train MMSegmentation](https://ecosystem.supervise.ly/apps/supervisely-ecosystem/mmsegmentation/train) app or manually created with the same directory structure +Model and directory structure must be acquired via [Train MMSegmentation](https://ecosystem.supervisely.com/apps/supervisely-ecosystem/mmsegmentation/train) app or manually created with the same directory structure @@ -124,19 +124,19 @@ You can use your trained models outside Supervisely platform without any depende You can use served model in next Supervisely Applications ⬇️ -- [Train MMSegmentation](https://ecosystem.supervise.ly/apps/supervisely-ecosystem/mmsegmentation/train) - app allows to play with different inference options, monitor metrics charts in real time, and save training artifacts to Team Files. +- [Train MMSegmentation](https://ecosystem.supervisely.com/apps/supervisely-ecosystem/mmsegmentation/train) - app allows to play with different inference options, monitor metrics charts in real time, and save training artifacts to Team Files. -- [Apply NN to images project ](https://ecosystem.supervise.ly/apps/supervisely-ecosystem%252Fnn-image-labeling%252Fproject-dataset) - app allows to play with different inference options and visualize predictions in real time. Once you choose inference settings you can apply model to all images in your project to visually analyse predictions and perform automatic data pre-labeling. +- [Apply NN to images project ](https://ecosystem.supervisely.com/apps/supervisely-ecosystem%252Fnn-image-labeling%252Fproject-dataset) - app allows to play with different inference options and visualize predictions in real time. Once you choose inference settings you can apply model to all images in your project to visually analyse predictions and perform automatic data pre-labeling. -- [Apply NN to Videos Project](https://ecosystem.supervise.ly/apps/apply-nn-to-videos-project) - app allows to label your videos using served Supervisely models. +- [Apply NN to Videos Project](https://ecosystem.supervisely.com/apps/apply-nn-to-videos-project) - app allows to label your videos using served Supervisely models. -- [NN Image Labeling](https://ecosystem.supervise.ly/apps/supervisely-ecosystem%252Fnn-image-labeling%252Fannotation-tool) - integrate any deployd NN to Supervisely Image Labeling UI. Configure inference settings and model output classes. Press `Apply` button (or use hotkey) and detections with their confidences will immediately appear on the image. +- [NN Image Labeling](https://ecosystem.supervisely.com/apps/supervisely-ecosystem%252Fnn-image-labeling%252Fannotation-tool) - integrate any deployd NN to Supervisely Image Labeling UI. Configure inference settings and model output classes. Press `Apply` button (or use hotkey) and detections with their confidences will immediately appear on the image. diff --git a/serve/config.json b/serve/config.json index 72a60dc..f6b73b1 100644 --- a/serve/config.json +++ b/serve/config.json @@ -11,8 +11,8 @@ "serve" ], "description": "Deploy model as REST API service", - "docker_image": "supervisely/mmseg:1.3.18", - "min_instance_version": "6.12.12", + "docker_image": "supervisely/mmseg:1.3.18", + "min_instance_version": "6.12.12", "entrypoint": "python -m uvicorn main:m.app --app-dir ./serve/src --host 0.0.0.0 --port 8000 --ws websockets", "port": 8000, "task_location": "application_sessions", diff --git a/train/README.md b/train/README.md index 63516de..d2e8c44 100644 --- a/train/README.md +++ b/train/README.md @@ -14,11 +14,11 @@ Acknowledgment

-[![](https://img.shields.io/badge/supervisely-ecosystem-brightgreen)](https://ecosystem.supervise.ly/apps/supervisely-ecosystem/mmsegmentation/train) -[![](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://supervise.ly/slack) +[![](https://img.shields.io/badge/supervisely-ecosystem-brightgreen)](https://ecosystem.supervisely.com/apps/supervisely-ecosystem/mmsegmentation/train) +[![](https://img.shields.io/badge/slack-chat-green.svg?logo=slack)](https://supervisely.com/slack) ![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/supervisely-ecosystem/mmsegmentation) -[![views](https://app.supervise.ly/img/badges/views/supervisely-ecosystem/mmsegmentation/train.png)](https://supervise.ly) -[![runs](https://app.supervise.ly/img/badges/runs/supervisely-ecosystem/mmsegmentation/train.png)](https://supervise.ly) +[![views](https://app.supervisely.com/img/badges/views/supervisely-ecosystem/mmsegmentation/train.png)](https://supervisely.com) +[![runs](https://app.supervisely.com/img/badges/runs/supervisely-ecosystem/mmsegmentation/train.png)](https://supervisely.com) @@ -93,10 +93,10 @@ Supported methods: # How to Run -### 1. Add [Train MMSegmentation](https://ecosystem.supervise.ly/apps/supervisely-ecosystem/mmsegmentation/train) app to your team from Ecosystem +### 1. Add [Train MMSegmentation](https://ecosystem.supervisely.com/apps/supervisely-ecosystem/mmsegmentation/train) app to your team from Ecosystem -### 2. Run app from context menu of the project with annotations (polygon and bitmap only) [(example)](https://ecosystem.supervise.ly/projects/lemons-annotated) +### 2. Run app from context menu of the project with annotations (polygon and bitmap only) [(example)](https://ecosystem.supervisely.com/projects/lemons-annotated) # Demo diff --git a/train/config.json b/train/config.json index e8ca6c1..222b7d0 100644 --- a/train/config.json +++ b/train/config.json @@ -1,33 +1,33 @@ { - "name": "Train MMSegmentation", - "type": "app", - "categories": [ - "neural network", - "images", - "videos", - "semantic segmentation", - "segmentation & tracking", - "train" - ], - "description": "Dashboard to configure, start and monitor training", - "docker_image": "supervisely/mmseg:1.3.18", - "min_instance_version": "6.12.12", - "main_script": "train/src/main.py", - "gui_template": "train/src/gui.html", - "task_location": "workspace_tasks", - "need_gpu": true, - "gpu": "required", - "isolate": true, - "icon": "https://i.imgur.com/GaEFmkH.png", - "icon_cover": true, - "context_menu": { - "target": ["images_project"], - "context_root": "Neural Networks", - "context_category": "MM Segmentation" - }, - "poster": "https://user-images.githubusercontent.com/48245050/182847473-9a35f213-c27b-4abd-bd64-c73bf80fb056.jpg", - "community_agent": false, - "license": { - "type": "Apache-2.0" - } + "name": "Train MMSegmentation", + "type": "app", + "categories": [ + "neural network", + "images", + "videos", + "semantic segmentation", + "segmentation & tracking", + "train" + ], + "description": "Dashboard to configure, start and monitor training", + "docker_image": "supervisely/mmseg:1.3.18", + "min_instance_version": "6.12.12", + "main_script": "train/src/main.py", + "gui_template": "train/src/gui.html", + "task_location": "workspace_tasks", + "need_gpu": true, + "gpu": "required", + "isolate": true, + "icon": "https://i.imgur.com/GaEFmkH.png", + "icon_cover": true, + "context_menu": { + "target": ["images_project"], + "context_root": "Neural Networks", + "context_category": "MM Segmentation" + }, + "poster": "https://user-images.githubusercontent.com/48245050/182847473-9a35f213-c27b-4abd-bd64-c73bf80fb056.jpg", + "community_agent": false, + "license": { + "type": "Apache-2.0" + } } diff --git a/train/src/sly_project_cached.py b/train/src/sly_project_cached.py index 6ad0cdd..41a6ee8 100644 --- a/train/src/sly_project_cached.py +++ b/train/src/sly_project_cached.py @@ -4,8 +4,8 @@ from supervisely.project.download import ( download_to_cache, copy_from_cache, - is_cached, get_cache_size, + download_async ) from sly_train_progress import get_progress_cb import sly_globals as g @@ -13,15 +13,27 @@ def _no_cache_download(api: sly.Api, project_info: sly.ProjectInfo, project_dir: str, progress_index: int): total = project_info.items_count - download_progress = get_progress_cb(progress_index, "Downloading input data...", total * 2) - sly.download( - api=api, - project_id=project_info.id, - dest_dir=project_dir, - dataset_ids=None, - log_progress=True, - progress_cb=download_progress, - ) + try: + download_progress = get_progress_cb(progress_index, "Downloading input data...", total) + download_async( + api, + project_info.id, + project_dir, + progress_cb=download_progress + ) + except Exception as e: + api.logger.warning( + "Failed to download project using async download. Trying sync download..." + ) + download_progress = get_progress_cb(progress_index, "Downloading input data...", total) + sly.download( + api=api, + project_id=project_info.id, + dest_dir=project_dir, + dataset_ids=None, + log_progress=True, + progress_cb=download_progress, + ) def download_project( api: sly.Api, @@ -37,47 +49,23 @@ def download_project( return try: - # get datasets to download and cached - dataset_infos = api.dataset.get_list(project_info.id) - to_download = [info for info in dataset_infos if not is_cached(project_info.id, info.name)] - cached = [info for info in dataset_infos if is_cached(project_info.id, info.name)] - if len(cached) == 0: - log_msg = "No cached datasets found" - else: - log_msg = "Using cached datasets: " + ", ".join( - f"{ds_info.name} ({ds_info.id})" for ds_info in cached - ) - sly.logger.info(log_msg) - if len(to_download) == 0: - log_msg = "All datasets are cached. No datasets to download" - else: - log_msg = "Downloading datasets: " + ", ".join( - f"{ds_info.name} ({ds_info.id})" for ds_info in to_download - ) - sly.logger.info(log_msg) - # get images count - total = sum([ds_info.images_count for ds_info in dataset_infos]) + total = project_info.items_count # download - if total > 0: - download_progress = get_progress_cb(progress_index, "Downloading input data...", total * 2) - download_to_cache( - api=api, - project_id=project_info.id, - dataset_infos=dataset_infos, - log_progress=True, - progress_cb=download_progress, - ) + download_progress = get_progress_cb(progress_index, "Downloading input data...", total) + download_to_cache( + api=api, + project_id=project_info.id, + log_progress=True, + progress_cb=download_progress, + ) # copy datasets from cache - total = sum([get_cache_size(project_info.id, ds.name) for ds in dataset_infos]) - dataset_names = [ds_info.name for ds_info in dataset_infos] - if total > 0: - download_progress = get_progress_cb(progress_index, "Retreiving data from cache...", total, is_size=True) - copy_from_cache( - project_id=project_info.id, - dest_dir=project_dir, - dataset_names=dataset_names, - progress_cb=download_progress, - ) + total = get_cache_size(project_info.id) + download_progress = get_progress_cb(progress_index, "Retreiving data from cache...", total, is_size=True) + copy_from_cache( + project_id=project_info.id, + dest_dir=project_dir, + progress_cb=download_progress, + ) except Exception: sly.logger.warning(f"Failed to retreive project from cache. Downloading it...", exc_info=True) if os.path.exists(project_dir): diff --git a/train/src/ui/monitoring.py b/train/src/ui/monitoring.py index 3b041f9..70347dc 100644 --- a/train/src/ui/monitoring.py +++ b/train/src/ui/monitoring.py @@ -107,7 +107,7 @@ def init_devices(): "free": free_mem, } devices.append(device_info) - return devices + return sorted(devices, key=lambda x: x["free"], reverse=True) def init_chart(