Model comparison added. New endpoints to start evaluation/comparison from script

.vscode/launch.json

@@ -16,7 +16,7 @@
                 "websockets",
             ],
             "jinja": true,
-            "justMyCode": true,
+            "justMyCode": false,
             "env": {
                 "PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}",
                 "LOG_LEVEL": "DEBUG"

README.md

@@ -22,6 +22,15 @@
 
 The Evaluator for Model Benchmark is a versatile application designed to assess the performance of various machine learning models in a consistent and reliable manner. This app provides a streamlined process for evaluating models and generating comprehensive reports to help you learn different metrics and make informed decisions.
 
+The Evaluator app offers a range of evaluation metrics, including precision, recall, F1 score, mAP, and more. The app also includes a **Model Comparison** feature that allows you to compare the performance of multiple models side by side.
+
+**Changelog:**
+
+- **v0.1.0** – Public release (for object detection task type)
+- **v0.1.2** – Support for instance segmentation task type
+- **v0.1.4** – Speedtest benchmark added
+- **v0.1.15** – Model Comparison feature added
+
 ## Preparation
 
 Before running the Evaluator for Model Benchmark, please ensure that you have the following:
@@ -31,10 +40,20 @@ Before running the Evaluator for Model Benchmark, please ensure that you have th
 
 ## How To Run
 
-**Step 1:** Open the app from the Supervisely Ecosystem.
+**Step 1:** Open and launch the app from the Supervisely Ecosystem.
+
+**Step 2**:
+
+- _Model Evaluation_:
+
+  **Step 2.1:** Select the Ground Truth project and the model you want to evaluate.
+
+  **Step 2.2:** Press the “Evaluate” button to start the evaluation process. After the evaluation is complete, you can find a link to the report in the app’s interface.
+
+- _Model Comparison:_
 
-**Step 2:** Select the project you wish to evaluate.
+  **Step 2.1:** Select the folder with the Ground Truth project name.
 
-**Step 3:** Choose the model you want to evaluate from the list of served models.
+  **Step 2.1:** Select one or more evaluation folders with the model name.
 
-**Step 4:** Start the evaluation process by clicking the “Run” button. The app will process the data and evaluate the model(s) based on the selected benchmarks. You can monitor the progress in the app’s interface.
+  **Step 2.2:** Press the “Compare” button to start the comparison process. After the comparison is complete, you can find a link to the report in the app’s interface.

config.json

@@ -2,28 +2,15 @@
   "type": "app",
   "version": "2.0.0",
   "name": "Evaluator for Model Benchmark",
-  "description": "Evaluate the performance of the NN model",
-  "categories": ["neural network", "images", "object detection"],
+  "description": "Evaluate the performance of the NN model and compare it with the results of other models",
+  "categories": ["neural network", "images", "object detection", "instance segmentation"],
   "icon": "https://github.com/supervisely-ecosystem/model-benchmark/releases/download/v0.0.4/icon-mb.png",
   "icon_cover": true,
   "headless": false,
   "poster": "https://github.com/supervisely-ecosystem/model-benchmark/releases/download/v0.0.4/poster.jpg",
   "task_location": "workspace_tasks",
   "entrypoint": "python -m uvicorn src.main:app --host 0.0.0.0 --port 8000",
   "port": 8000,
-  "docker_image": "supervisely/model-benchmark:1.0.13",
-  "instance_version": "6.11.19",
-  "context_menu": {
-    "target": ["images_project"]
-  },
-  "modal_template": "src/modal.html",
-  "modal_template_state": {
-    "sessionId": null,
-    "autoStart": false,
-    "sessionOptions": {
-      "sessionTags": ["deployed_nn"],
-      "showLabel": false,
-      "size": "small"
-    }
-  }
+  "docker_image": "supervisely/model-benchmark:1.0.15",
+  "instance_version": "6.11.19"
 }

dev_requirements.txt

@@ -1,2 +1,2 @@
 # git+https://github.com/supervisely/supervisely.git@model-benchmark
-supervisely[model-benchmark]==6.73.208
+supervisely[model-benchmark]==6.73.215

docker/Dockerfile

@@ -1,5 +1,5 @@
 FROM supervisely/base-py-sdk:6.73.208
 
-RUN python3 -m pip install supervisely[model-benchmark]==6.73.208
+RUN python3 -m pip install supervisely[model-benchmark]==6.73.215
 
-LABEL python_sdk_version=6.73.208
+LABEL python_sdk_version=6.73.215

local.env

@@ -1,14 +1,7 @@
-# Ars
-# TEAM_ID = 449 # * Required! Copy your team id here: https://dev.supervisely.com/teams/my
-# WORKSPACE_ID = 691 # * Required! Copy your workspace id here: https://dev.supervisely.com/workspaces/
-# PROJECT_ID = 32796 # * Optional. Copy your project id here if needed: https://dev.supervisely.com/projects/
-# DATASET_ID = 81589 # * Optional. Copy your dataset id here if needed: https://dev.supervisely.com/projects/<project_id>/datasets/
-# SLY_APP_DATA_DIR = "/home/grokhi/supervisely/tasks/model-benchmark/APP_DATA" # * Optional. Path to the local folder for application data. Make sure the app will have rights to read/write from/to this folder.
-
-# PB
-TEAM_ID = 440
-WORKSPACE_ID = 1105
-PROJECT_ID = 40299
+TEAM_ID = 447
+WORKSPACE_ID = 680
+# PROJECT_ID = 41021
 SLY_APP_DATA_DIR = "APP_DATA"
 
-TASK_ID = 64030
+TASK_ID = 60447
+# modal.state.sessionId=66693

src/functions.py

@@ -1,3 +1,6 @@
+import os
+from typing import List, Tuple
+
 import src.globals as g
 import supervisely as sly
 from supervisely.nn import TaskType
@@ -47,3 +50,55 @@ def get_classes():
             not_matched_model_cls.append(obj_class)
 
     return (matched_proj_cls, matched_model_cls), (not_matched_proj_cls, not_matched_model_cls)
+
+
+def validate_paths(paths: List[str]):
+    if not paths:
+        raise ValueError("No paths selected")
+
+    split_paths = [path.strip("/").split(os.sep) for path in paths]
+    path_length = min(len(p) for p in split_paths)
+
+    if not all(len(p) == path_length for p in split_paths):
+        raise ValueError(f"Selected paths not on the correct level: {paths}")
+
+    if not all(p.startswith("/model-benchmark") for p in paths):
+        raise ValueError(f"Selected paths are not in the benchmark directory: {paths}")
+
+    if not all(p[1] == split_paths[0][1] for p in split_paths):
+        raise ValueError(f"Project names are different: {paths}")
+
+
+def get_parent_paths(paths: List[str]) -> Tuple[str, List[str]]:
+    split_paths = [path.strip("/").split(os.sep) for path in paths]
+    project_name = split_paths[0][1]
+    eval_dirs = [p[2] for p in split_paths]
+
+    return project_name, eval_dirs
+
+
+def get_res_dir(eval_dirs: List[str]) -> str:
+
+    res_dir = "/model-comparison"
+    project_name, eval_dirs = get_parent_paths(eval_dirs)
+    res_dir += "/" + project_name + "/"
+    res_dir += " vs ".join(eval_dirs)
+
+    res_dir = g.api.file.get_free_dir_name(g.team_id, res_dir)
+
+    return res_dir
+
+
+# ! temp fix (to allow the app to receive requests)
+def with_clean_up_progress(pbar):
+    def decorator(func):
+        def wrapper(*args, **kwargs):
+            try:
+                return func(*args, **kwargs)
+            finally:
+                with pbar(message="Application is started ...", total=1) as pb:
+                    pb.update(1)
+
+        return wrapper
+
+    return decorator

src/globals.py

@@ -15,7 +15,6 @@
 STORAGE_DIR = sly.app.get_data_dir()
 STATIC_DIR = os.path.join(STORAGE_DIR, "static")
 sly.fs.mkdir(STATIC_DIR)
-TF_RESULT_DIR = "/model-benchmark/layout"
 
 deployed_nn_tags = ["deployed_nn"]
 
@@ -27,6 +26,6 @@
 if session_id is not None:
     session_id = int(session_id)
 session = None
-autostart = bool(strtobool(os.environ.get("modal.state.autoStart", "false")))
 
-selected_classes = None
+selected_classes = None
+eval_dirs = None