Run benchmark after training. Other improvements (project conversion,…

… progress bar, search for bg class) (#61)

docker/Dockerfile

@@ -26,5 +26,5 @@ RUN mkdir -p /tmp/mmseg \
 && wget https://github.com/open-mmlab/mmsegmentation/archive/refs/tags/v0.23.0.tar.gz -P /tmp/mmseg \
 && tar -xvf /tmp/mmseg/v0.23.0.tar.gz -C /tmp/mmseg
 
-RUN pip3 install supervisely==6.73.242
-LABEL python_sdk_version=6.73.242
+RUN pip3 install supervisely[model-benchmark]==6.73.255
+LABEL python_sdk_version=6.73.255

serve/config.json

@@ -11,7 +11,7 @@
 		"serve"
 	],
 	"description": "Deploy model as REST API service",
-	"docker_image": "supervisely/mmseg:1.3.18",
+	"docker_image": "supervisely/mmseg:1.3.19",
 	"min_instance_version": "6.12.12",
 	"entrypoint": "python -m uvicorn main:m.app --app-dir ./serve/src --host 0.0.0.0 --port 8000 --ws websockets",
 	"port": 8000,

serve/dev_requirements.txt

@@ -1,6 +1,6 @@
 # git+https://github.com/supervisely/supervisely.git@some-test-branch
 
-supervisely==6.73.242
+supervisely==6.73.255
 
 openmim
 ffmpeg-python==0.2.0

serve/local.env

@@ -1,6 +1,6 @@
 PYTHONUNBUFFERED=1
 
-DEBUG_APP_DIR="/tmp/mmsegmentation"
-DEBUG_CACHE_DIR="/tmp/mmsegmentation/cache"
+DEBUG_APP_DIR="tmp/mmsegmentation"
+DEBUG_CACHE_DIR="tmp/mmsegmentation/cache"
 
 LOG_LEVEL="debug"

train/config.json

@@ -10,7 +10,7 @@
 		"train"
 	],
 	"description": "Dashboard to configure, start and monitor training",
-	"docker_image": "supervisely/mmseg:1.3.18",
+	"docker_image": "supervisely/mmseg:1.3.19",
 	"min_instance_version": "6.12.12",
 	"main_script": "train/src/main.py",
 	"gui_template": "train/src/gui.html",

train/dev_requirements.txt

@@ -1,6 +1,6 @@
 # git+https://github.com/supervisely/supervisely.git@some-test-branch
 
-supervisely==6.73.242
+supervisely==6.73.255
 
 openmim
 ffmpeg-python==0.2.0

train/local.env

@@ -0,0 +1,13 @@
+PYTHONUNBUFFERED=1
+
+modal.state.slyProjectId=43218
+
+context.teamId=447
+context.workspaceId=680
+
+AGENT_ID=341
+TASK_ID=68365
+DEBUG_APP_DIR="tmp/mmsegmentation"
+DEBUG_CACHE_DIR="tmp/mmsegmentation/cache"
+
+LOG_LEVEL="debug"

train/src/sly_functions.py

@@ -0,0 +1,17 @@
+import supervisely as sly
+
+
+def get_bg_class_name(class_names):
+    possible_bg_names = ["background", "bg", "unlabeled", "neutral", "__bg__"]
+    for name in class_names:
+        if name.lower() in possible_bg_names:
+            return name
+    return None
+
+
+def get_eval_results_dir_name(api, task_id, project_info):
+    task_info = api.task.get_info_by_id(task_id)
+    task_dir = f"{task_id}_{task_info['meta']['app']['name']}"
+    eval_res_dir = f"/model-benchmark/{project_info.id}_{project_info.name}/{task_dir}/"
+    eval_res_dir = api.file.get_free_dir_name(sly.env.team_id(), eval_res_dir)
+    return eval_res_dir

train/src/sly_logger_hook.py

@@ -4,6 +4,7 @@
 import supervisely as sly
 from sly_train_progress import get_progress_cb, set_progress, add_progress_to_request
 import sly_globals as g
+from sly_functions import get_bg_class_name
 import classes as cls
 import math
 
@@ -126,7 +127,11 @@ def _log_info(self, log_dict, runner):
             for metric_name, metrics in class_metrics.items():
                 if f"m{metric_name}" not in g.evalMetrics:
                     continue
-                classes = cls.selected_classes + ["__bg__"]
+                bg = get_bg_class_name(cls.selected_classes)
+                if bg is None:
+                    classes = cls.selected_classes + ["__bg__"]
+                else:
+                    classes = cls.selected_classes
                 for class_ind, class_name in enumerate(classes):
                     fields.extend(
                         [

train/src/sly_mmsegm.py

@@ -0,0 +1,9 @@
+import sys
+
+sys.path.insert(0, "../")
+
+from serve.src.mmsegm_model import MMSegmentationModel
+
+
+class MMSegmentationModelBench(MMSegmentationModel):
+    in_train = True

train/src/ui/classes.html

@@ -60,7 +60,7 @@
   </sly-el-table-selection-container>
   <sly-field
     title="INFO: Background Class"
-    description="background class (name: __bg__, color: [0, 0, 0]) will be added automatically"
+    description="if background class is not selected or not exists, it will be added automatically (by default: name: __bg__, color: [0, 0, 0])"
     style="padding-top: 0; padding-bottom: 0; margin: 10px 5px"
   >
     <sly-icon

train/src/ui/hyperparameters.html

@@ -45,6 +45,18 @@
                     <span class="ml5">iterations</span>
                 </div>
             </sly-field>
+            <sly-field title="Run Model Benchmark evaluation">
+                <el-switch v-model="state.runBenchmark"
+                           on-color="#13ce66" off-color="#B8B8B8"
+                           :disabled="data.done6">
+                </el-switch>
+            </sly-field>
+            <sly-field title="Run Speed test">
+                <el-switch v-model="state.runSpeedTest"
+                           on-color="#13ce66" off-color="#B8B8B8"
+                           :disabled="data.done6">
+                </el-switch>
+            </sly-field>
         </el-tab-pane>
         <el-tab-pane label="Checkpoints" name="checkpoints">
             <sly-field title="Checkpoints interval"

train/src/ui/hyperparameters.py

@@ -9,6 +9,8 @@ def init_general(state):
 
     state["valInterval"] = 1
     state["logConfigInterval"] = 5
+    state["runBenchmark"] = True
+    state["runSpeedTest"] = True
 
 def init_checkpoints(state):
     state["checkpointInterval"] = 1
@@ -99,4 +101,4 @@ def use_hyp(api: sly.Api, task_id, context, state, app_logger):
         {"field": "state.disabled7", "payload": False},
         {"field": "state.activeStep", "payload": 7},
     ]
-    g.api.app.set_fields(g.task_id, fields)
+    g.api.app.set_fields(g.task_id, fields)

train/src/ui/monitoring.html

@@ -79,6 +79,15 @@
 			>
 		</div>
 	</div>
+    <div v-if="data.benchmarkUrl">
+        <sly-field title="" description="Open the Model Benchmark evaluation report.">
+            <a slot="title" target="_blank" :href="data.benchmarkUrl">Evaluation Report</a>
+            <sly-icon
+                slot="icon"
+                :options="{ color: '#dcb0ff', bgColor: '#faebff', className: 'zmdi zmdi-assignment' }"
+            />
+        </sly-field>
+    </div>
 	<sly-field v-if="state.preparingData" class="mt10">
 		<b style="color: #20a0ff"
 			>Preparing segmentation data (it may take a few minutes)...</b
@@ -103,6 +112,26 @@
 		</div>
 		<el-progress :percentage="data.progressPercentUploadDir"></el-progress>
 	</div>
+    <div v-if="state.benchmarkInProgress">
+        <i class="zmdi zmdi-info mr5" style="color: #20a0ff"></i>
+        <span style="color: #5a6772"
+            >Model Benchmark evaluation is in progress...</span
+        >
+    </div>
+    <div v-if="state.benchmarkInProgress && data.progressBenchmark" class="mt10">
+		<div style="color: #20a0ff">
+			{{data.progressBenchmark}}: {{data.progressCurrentBenchmark}} /
+			{{data.progressTotalBenchmark}}
+		</div>
+		<el-progress :percentage="data.progressPercentBenchmark"></el-progress>
+	</div>
+    <div v-if="data.progressTqdm" class="mt10">
+		<div style="color: #20a0ff">
+			{{data.progressTqdm}}: {{data.progressCurrentTqdm}} /
+			{{data.progressTotalTqdm}}
+		</div>
+		<el-progress :percentage="data.progressPercentTqdm"></el-progress>
+	</div>
 	<div v-if="data.progressEpoch" class="mt10">
 		<div style="color: #20a0ff">
 			{{data.progressEpoch}}: {{data.progressCurrentEpoch}} /