diff --git a/dev_requirements.txt b/dev_requirements.txt index 71e97944a78f..e8b05cf33723 100755 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,7 +1,7 @@ # git+https://github.com/supervisely/supervisely.git@test-branch # pip install -r requirements.txt -supervisely==6.73.122 +supervisely==6.73.123 opencv-python-headless==4.5.5.62 opencv-python==4.5.5.62 diff --git a/supervisely/serve/config.json b/supervisely/serve/config.json index 10191023d6fb..c13346ee557f 100644 --- a/supervisely/serve/config.json +++ b/supervisely/serve/config.json @@ -11,7 +11,7 @@ "serve" ], "description": "Deploy model as REST API service", - "docker_image": "supervisely/yolov5:1.0.7", + "docker_image": "supervisely/yolov5:1.0.8", "instance_version": "6.8.88", "entrypoint": "python -m uvicorn main:m.app --app-dir ./supervisely/serve/src --host 0.0.0.0 --port 8000 --ws websockets", "port": 8000, diff --git a/supervisely/serve/src/main.py b/supervisely/serve/src/main.py index 920cd902678d..9b23ada39e72 100644 --- a/supervisely/serve/src/main.py +++ b/supervisely/serve/src/main.py @@ -18,11 +18,14 @@ from utils.datasets import letterbox from pathlib import Path + root_source_path = str(Path(__file__).parents[3]) app_source_path = str(Path(__file__).parents[1]) load_dotenv(os.path.join(app_source_path, "local.env")) load_dotenv(os.path.expanduser("~/supervisely.env")) +from workflow import Workflow + model_weights_options = os.environ["modal.state.modelWeightsOptions"] pretrained_weights = os.environ["modal.state.selectedModel"].lower() custom_weights = os.environ["modal.state.weightsPath"] @@ -45,6 +48,8 @@ def load_on_device( self.local_weights_path = self.download(custom_weights) cfg_path_in_teamfiles = os.path.join(Path(custom_weights).parents[1], "opt.yaml") configs_local_path = self.download(cfg_path_in_teamfiles) + workflow = Workflow(self.api) + workflow.add_input(custom_weights) self.device = select_device(device) self.half = self.device.type != "cpu" # half precision only supported on CUDA diff --git a/supervisely/serve/src/workflow.py b/supervisely/serve/src/workflow.py new file mode 100644 index 000000000000..d23b9d6ca8d4 --- /dev/null +++ b/supervisely/serve/src/workflow.py @@ -0,0 +1,46 @@ +import supervisely as sly + + +def check_compatibility(func): + def wrapper(self, *args, **kwargs): + if self.is_compatible is None: + self.is_compatible = self.check_instance_ver_compatibility() + if not self.is_compatible: + return + return func(self, *args, **kwargs) + + return wrapper + + +class Workflow: + def __init__(self, api: sly.Api, min_instance_version: str = None): + self.is_compatible = None + self.api = api + self._min_instance_version = ( + "6.9.31" if min_instance_version is None else min_instance_version + ) + + def check_instance_ver_compatibility(self): + if not self.api.is_version_supported(self._min_instance_version): + sly.logger.info( + f"Supervisely instance version {self.api.instance_version} does not support workflow features." + ) + if not sly.is_community(): + sly.logger.info( + f"To use them, please update your instance to version {self._min_instance_version} or higher." + ) + return False + return True + + @check_compatibility + def add_input(self, checkpoint_url: str): + meta = {"customNodeSettings": {"title": "

Serve Custom Model

"}} + sly.logger.debug(f"Workflow Input: Checkpoint URL - {checkpoint_url}") + if checkpoint_url and self.api.file.exists(sly.env.team_id(), checkpoint_url): + self.api.app.workflow.add_input_file(checkpoint_url, model_weight=True, meta=meta) + else: + sly.logger.debug(f"Checkpoint {checkpoint_url} not found in Team Files. Cannot set workflow input") + + @check_compatibility + def add_output(self): + raise NotImplementedError("add_output is not implemented in this workflow") diff --git a/supervisely/train/config.json b/supervisely/train/config.json index bf435d07dd1f..c5aa41d076cf 100644 --- a/supervisely/train/config.json +++ b/supervisely/train/config.json @@ -10,7 +10,7 @@ "train" ], "description": "Dashboard to configure and monitor training", - "docker_image": "supervisely/yolov5:1.0.7", + "docker_image": "supervisely/yolov5:1.0.8", "min_instance_version": "6.8.70", "main_script": "supervisely/train/src/sly_train.py", "gui_template": "supervisely/train/src/gui.html", diff --git a/supervisely/train/src/sly_train.py b/supervisely/train/src/sly_train.py index c182ef4f2661..1b2b301cdbfd 100644 --- a/supervisely/train/src/sly_train.py +++ b/supervisely/train/src/sly_train.py @@ -40,6 +40,10 @@ def train(api: sly.Api, task_id, context, state, app_logger): project_dir = os.path.join(my_app.data_dir, "sly_project") sly.fs.mkdir(project_dir, remove_content_if_exists=True) # clean content for debug, has no effect in prod + # -------------------------------------- Add Workflow Input -------------------------------------- # + g.workflow.add_input(g.project_info, state) + # ----------------------------------------------- - ---------------------------------------------- # + # download and preprocess Sypervisely project (using cache) try: download_project( @@ -145,6 +149,7 @@ def train(api: sly.Api, task_id, context, state, app_logger): # upload artifacts directory to Team Files upload_artifacts(g.local_artifacts_dir, g.remote_artifacts_dir) set_task_output() + g.workflow.add_output(state, g.remote_artifacts_dir) except Exception as e: msg = f"Something went wrong. Find more info in the app logs." my_app.show_modal_window(f"{msg} {repr(e)}", level="error", log_message=False) diff --git a/supervisely/train/src/sly_train_globals.py b/supervisely/train/src/sly_train_globals.py index 3e0e1d16d6e3..d02b37f4ed75 100644 --- a/supervisely/train/src/sly_train_globals.py +++ b/supervisely/train/src/sly_train_globals.py @@ -6,6 +6,7 @@ from supervisely.nn.artifacts.yolov5 import YOLOv5 from supervisely.app.v1.app_service import AppService from dotenv import load_dotenv +from workflow import Workflow root_source_dir = str(Path(sys.argv[0]).parents[3]) sly.logger.info(f"Root source directory: {root_source_dir}") @@ -35,6 +36,7 @@ project_id = int(os.environ['modal.state.slyProjectId']) api: sly.Api = my_app.public_api +workflow = Workflow(api) task_id = my_app.task_id local_artifacts_dir = None diff --git a/supervisely/train/src/test.py b/supervisely/train/src/test.py new file mode 100644 index 000000000000..9d319f71e11e --- /dev/null +++ b/supervisely/train/src/test.py @@ -0,0 +1,15 @@ +import supervisely as sly +import os + +api = sly.Api.from_env() +api.task_id = 62688 +os.environ.setdefault("TEAM_ID", "451") +from workflow import Workflow + +workflow = Workflow(api) + +state = {"weightsInitialization": "custom"} + +team_files_dir = "/yolov5_train/Train dataset - Eschikon Wheat Segmentation (EWS)/62688" + +workflow.add_output(state, team_files_dir) \ No newline at end of file diff --git a/supervisely/train/src/workflow.py b/supervisely/train/src/workflow.py new file mode 100644 index 000000000000..f8c00de7d456 --- /dev/null +++ b/supervisely/train/src/workflow.py @@ -0,0 +1,90 @@ +# Description: This file contains versioning features and the Workflow class that is used to add input and output to the workflow. + +import supervisely as sly +import os + + +def check_compatibility(func): + def wrapper(self, *args, **kwargs): + if self.is_compatible is None: + self.is_compatible = self.check_instance_ver_compatibility() + if not self.is_compatible: + return + return func(self, *args, **kwargs) + + return wrapper + + +class Workflow: + def __init__(self, api: sly.Api, min_instance_version: str = None): + self.is_compatible = None + self.api = api + self._min_instance_version = ( + "6.9.31" if min_instance_version is None else min_instance_version + ) + + def check_instance_ver_compatibility(self): + if not self.api.is_version_supported(self._min_instance_version): + sly.logger.info( + f"Supervisely instance version {self.api.instance_version} does not support workflow and versioning features." + ) + if not sly.is_community(): + sly.logger.info( + f"To use them, please update your instance to version {self._min_instance_version} or higher." + ) + return False + return True + + @check_compatibility + def add_input(self, project_info: sly.ProjectInfo, state: dict): + project_version_id = self.api.project.version.create( + project_info, "Train YOLO v5", f"This backup was created automatically by Supervisely before the Train YOLO task with ID: {self.api.task_id}" + ) + if project_version_id is None: + project_version_id = project_info.version.get("id", None) if project_info.version else None + self.api.app.workflow.add_input_project(project_info.id, version_id=project_version_id) + if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom": + file_info = self.api.file.get_info_by_path(sly.env.team_id(), state["_weightsPath"]) + self.api.app.workflow.add_input_file(file_info, model_weight=True) + sly.logger.debug(f"Workflow Input: Project ID - {project_info.id}, Project Version ID - {project_version_id}, Input File - {True if file_info else False}") + + @check_compatibility + def add_output(self, state: dict, team_files_dir: str): + weights_dir_in_team_files = os.path.join(team_files_dir, "weights") + files_info = self.api.file.list(sly.env.team_id(), weights_dir_in_team_files, return_type="fileinfo") + best_filename_info = None + for file_info in files_info: + if "best" in file_info.name: + best_filename_info = file_info + break + if best_filename_info: + module_id = self.api.task.get_info_by_id(self.api.task_id).get("meta", {}).get("app", {}).get("id") + if state["weightsInitialization"] is not None and state["weightsInitialization"] == "custom": + model_name = "Custom Model" + else: + model_name = "YOLOv5" + + meta = { + "customNodeSettings": { + "title": f"

Train {model_name}

", + "mainLink": { + "url": f"/apps/{module_id}/sessions/{self.api.task_id}" if module_id else f"apps/sessions/{self.api.task_id}", + "title": "Show Results" + } + }, + "customRelationSettings": { + "icon": { + "icon": "zmdi-folder", + "color": "#FFA500", + "backgroundColor": "#FFE8BE" + }, + "title": "

Checkpoints

", + "mainLink": {"url": f"/files/{best_filename_info.id}/true", "title": "Open Folder"} + } + } + sly.logger.debug(f"Workflow Output: Team Files dir - {team_files_dir}, Best filename - {best_filename_info.name}") + sly.logger.debug(f"Workflow Output: meta \n {meta}") + self.api.app.workflow.add_output_file(best_filename_info, model_weight=True, meta=meta) + else: + sly.logger.debug(f"File with the best weighs not found in Team Files. Cannot set workflow output.") +