Add Workflow

dev_requirements.txt

@@ -1,7 +1,7 @@
 # git+https://github.com/supervisely/supervisely.git@test-branch
 
 # pip install -r requirements.txt
-supervisely==6.73.122
+supervisely==6.73.123
 
 opencv-python-headless==4.5.5.62
 opencv-python==4.5.5.62

supervisely/serve/config.json

@@ -11,7 +11,7 @@
     "serve"
   ],
   "description": "Deploy model as REST API service",
-  "docker_image": "supervisely/yolov5:1.0.7",
+  "docker_image": "supervisely/yolov5:1.0.8",
   "instance_version": "6.8.88",
   "entrypoint": "python -m uvicorn main:m.app --app-dir ./supervisely/serve/src --host 0.0.0.0 --port 8000 --ws websockets",
   "port": 8000,

supervisely/serve/src/main.py

@@ -18,11 +18,14 @@
 from utils.datasets import letterbox
 from pathlib import Path
 
+
 root_source_path = str(Path(__file__).parents[3])
 app_source_path = str(Path(__file__).parents[1])
 load_dotenv(os.path.join(app_source_path, "local.env"))
 load_dotenv(os.path.expanduser("~/supervisely.env"))
 
+from workflow import Workflow
+
 model_weights_options = os.environ["modal.state.modelWeightsOptions"]
 pretrained_weights = os.environ["modal.state.selectedModel"].lower()
 custom_weights = os.environ["modal.state.weightsPath"]
@@ -45,6 +48,8 @@ def load_on_device(
             self.local_weights_path = self.download(custom_weights)
             cfg_path_in_teamfiles = os.path.join(Path(custom_weights).parents[1], "opt.yaml")
             configs_local_path = self.download(cfg_path_in_teamfiles)
+            workflow = Workflow(self.api)
+            workflow.add_input(custom_weights)
 
         self.device = select_device(device)
         self.half = self.device.type != "cpu"  # half precision only supported on CUDA

supervisely/serve/src/workflow.py

@@ -0,0 +1,46 @@
+import supervisely as sly
+
+
+def check_compatibility(func):
+    def wrapper(self, *args, **kwargs):
+        if self.is_compatible is None:
+            self.is_compatible = self.check_instance_ver_compatibility()
+        if not self.is_compatible:
+            return
+        return func(self, *args, **kwargs)
+
+    return wrapper
+
+
+class Workflow:
+    def __init__(self, api: sly.Api, min_instance_version: str = None):
+        self.is_compatible = None
+        self.api = api
+        self._min_instance_version = (
+            "6.9.31" if min_instance_version is None else min_instance_version
+        )
+
+    def check_instance_ver_compatibility(self):
+        if not self.api.is_version_supported(self._min_instance_version):
+            sly.logger.info(
+                f"Supervisely instance version {self.api.instance_version} does not support workflow features."
+            )
+            if not sly.is_community():
+                sly.logger.info(
+                    f"To use them, please update your instance to version {self._min_instance_version} or higher."
+                )
+            return False
+        return True
+
+    @check_compatibility
+    def add_input(self, checkpoint_url: str):
+        meta = {"customNodeSettings": {"title": "<h4>Serve Custom Model</h4>"}}
+        sly.logger.debug(f"Workflow Input: Checkpoint URL - {checkpoint_url}")
+        if checkpoint_url and self.api.file.exists(sly.env.team_id(), checkpoint_url):
+            self.api.app.workflow.add_input_file(checkpoint_url, model_weight=True, meta=meta)
+        else:
+            sly.logger.debug(f"Checkpoint {checkpoint_url} not found in Team Files. Cannot set workflow input")
+
+    @check_compatibility
+    def add_output(self):
+        raise NotImplementedError("add_output is not implemented in this workflow")

supervisely/train/config.json

@@ -10,7 +10,7 @@
     "train"
   ],
   "description": "Dashboard to configure and monitor training",
-  "docker_image": "supervisely/yolov5:1.0.7",
+  "docker_image": "supervisely/yolov5:1.0.8",
   "min_instance_version": "6.8.70",
   "main_script": "supervisely/train/src/sly_train.py",
   "gui_template": "supervisely/train/src/gui.html",

supervisely/train/src/sly_train.py

@@ -40,6 +40,10 @@ def train(api: sly.Api, task_id, context, state, app_logger):
         project_dir = os.path.join(my_app.data_dir, "sly_project")
         sly.fs.mkdir(project_dir, remove_content_if_exists=True)  # clean content for debug, has no effect in prod
 
+        # -------------------------------------- Add Workflow Input -------------------------------------- #
+        g.workflow.add_input(g.project_info, state)
+        # ----------------------------------------------- - ---------------------------------------------- #
+
         # download and preprocess Sypervisely project (using cache)
         try:
             download_project(
@@ -145,6 +149,7 @@ def train(api: sly.Api, task_id, context, state, app_logger):
         # upload artifacts directory to Team Files
         upload_artifacts(g.local_artifacts_dir, g.remote_artifacts_dir)
         set_task_output()
+        g.workflow.add_output(state, g.remote_artifacts_dir)
     except Exception as e:
         msg = f"Something went wrong. Find more info in the app logs."
         my_app.show_modal_window(f"{msg} {repr(e)}", level="error", log_message=False)

supervisely/train/src/sly_train_globals.py

@@ -6,6 +6,7 @@
 from supervisely.nn.artifacts.yolov5 import YOLOv5
 from supervisely.app.v1.app_service import AppService
 from dotenv import load_dotenv
+from workflow import Workflow
 
 root_source_dir = str(Path(sys.argv[0]).parents[3])
 sly.logger.info(f"Root source directory: {root_source_dir}")
@@ -35,6 +36,7 @@
 project_id = int(os.environ['modal.state.slyProjectId'])
 
 api: sly.Api = my_app.public_api
+workflow = Workflow(api)
 task_id = my_app.task_id
 
 local_artifacts_dir = None

supervisely/train/src/test.py

@@ -0,0 +1,15 @@
+import supervisely as sly
+import os
+
+api = sly.Api.from_env()
+api.task_id = 62688
+os.environ.setdefault("TEAM_ID", "451")
+from workflow import Workflow
+
+workflow = Workflow(api)
+
+state = {"weightsInitialization": "custom"}
+
+team_files_dir = "/yolov5_train/Train dataset - Eschikon Wheat Segmentation (EWS)/62688"
+
+workflow.add_output(state, team_files_dir)

supervisely/train/src/workflow.py

@@ -0,0 +1,90 @@
+# Description: This file contains versioning features and the Workflow class that is used to add input and output to the workflow.
+
+import supervisely as sly
+import os
+
+
+def check_compatibility(func):
+    def wrapper(self, *args, **kwargs):
+        if self.is_compatible is None:
+            self.is_compatible = self.check_instance_ver_compatibility()
+        if not self.is_compatible:
+            return
+        return func(self, *args, **kwargs)
+
+    return wrapper
+
+
+class Workflow:
+    def __init__(self, api: sly.Api, min_instance_version: str = None):
+        self.is_compatible = None
+        self.api = api
+        self._min_instance_version = (
+            "6.9.31" if min_instance_version is None else min_instance_version
+        )
+
+    def check_instance_ver_compatibility(self):
+        if not self.api.is_version_supported(self._min_instance_version):
+            sly.logger.info(
+                f"Supervisely instance version {self.api.instance_version} does not support workflow and versioning features."
+            )
+            if not sly.is_community():
+                sly.logger.info(
+                    f"To use them, please update your instance to version {self._min_instance_version} or higher."
+                )
+            return False
+        return True
+
+    @check_compatibility
+    def add_input(self, project_info: sly.ProjectInfo, state: dict):
+        project_version_id = self.api.project.version.create(
+            project_info, "Train YOLO v5", f"This backup was created automatically by Supervisely before the Train YOLO task with ID: {self.api.task_id}"
+        )
+        if project_version_id is None:
+            project_version_id = project_info.version.get("id", None) if project_info.version else None
+        self.api.app.workflow.add_input_project(project_info.id, version_id=project_version_id)
+        if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom":
+             file_info = self.api.file.get_info_by_path(sly.env.team_id(), state["_weightsPath"])
+             self.api.app.workflow.add_input_file(file_info, model_weight=True)
+        sly.logger.debug(f"Workflow Input: Project ID - {project_info.id}, Project Version ID - {project_version_id}, Input File - {True if file_info else False}")
+
+    @check_compatibility
+    def add_output(self, state: dict, team_files_dir: str):
+        weights_dir_in_team_files = os.path.join(team_files_dir, "weights")
+        files_info = self.api.file.list(sly.env.team_id(), weights_dir_in_team_files, return_type="fileinfo")
+        best_filename_info = None
+        for file_info in files_info:
+            if "best" in file_info.name:
+                best_filename_info = file_info
+                break
+        if best_filename_info:
+            module_id = self.api.task.get_info_by_id(self.api.task_id).get("meta", {}).get("app", {}).get("id")                       
+            if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom":
+                model_name = "Custom Model"
+            else:
+                model_name = "YOLOv5"
+
+            meta = {
+                "customNodeSettings": {
+                "title": f"<h4>Train {model_name}</h4>",
+                "mainLink": {
+                    "url": f"/apps/{module_id}/sessions/{self.api.task_id}" if module_id else f"apps/sessions/{self.api.task_id}",
+                    "title": "Show Results"
+                }
+            },
+            "customRelationSettings": {
+                "icon": {
+                    "icon": "zmdi-folder",
+                    "color": "#FFA500",
+                    "backgroundColor": "#FFE8BE"
+                },
+                "title": "<h4>Checkpoints</h4>",
+                "mainLink": {"url": f"/files/{best_filename_info.id}/true", "title": "Open Folder"}
+                }
+            }
+            sly.logger.debug(f"Workflow Output: Team Files dir - {team_files_dir}, Best filename - {best_filename_info.name}")
+            sly.logger.debug(f"Workflow Output: meta \n    {meta}")
+            self.api.app.workflow.add_output_file(best_filename_info, model_weight=True, meta=meta)
+        else:
+            sly.logger.debug(f"File with the best weighs not found in Team Files. Cannot set workflow output.")
+