Add support for running the app from selected datasets (#98)

* Add support for running the app from selected datasets * Fix annotation download to use image IDs from the images list * Update Docker image and instance version in config.json * Update dev_requirements.txt * Add Docker publish script for building and pushing data nodes image
supervisely-ecosystem · Jan 17, 2025 · fef3e92 · fef3e92
1 parent 866c7b3
commit fef3e92
Show file tree

Hide file tree

Showing 6 changed files with 86 additions and 41 deletions.
diff --git a/config.json b/config.json
@@ -4,7 +4,7 @@
 	"description": "[Beta] Drag and drop interface for building custom DataOps pipelines",
 	"entrypoint": "python -m uvicorn src.main:app --host 0.0.0.0 --port 8000",
 	"headless": false,
-	"docker_image": "supervisely/data-nodes:1.0.27",
+	"docker_image": "supervisely/data-nodes:1.0.28",
 	"modal_template": "src/modal.html",
 	"modal_template_state": {
 		"modalityType": "images"
@@ -23,5 +23,5 @@
 			"files_file"
 		]
 	},
-	"instance_version": "6.12.12"
+	"instance_version": "6.12.22"
 }
diff --git a/dev_requirements.txt b/dev_requirements.txt
@@ -1,6 +1,6 @@
 git+https://github.com/supervisely/supervisely.git@optimize-index
 
-# supervisely==6.73.203
+# supervisely==6.73.280
 jsonschema==4.19.2
 networkx==3.1
 scikit-image==0.21.0

diff --git a/docker/publish.sh b/docker/publish.sh
@@ -0,0 +1,2 @@
+docker build -t supervisely/data-nodes:1.0.28 . && \
+docker push supervisely/data-nodes:1.0.28
diff --git a/src/globals.py b/src/globals.py
@@ -1,19 +1,20 @@
-import os
 import ast
-
+import os
+from distutils.util import strtobool
 from queue import Queue
+
 from dotenv import load_dotenv
-from distutils.util import strtobool
+
 import supervisely as sly
 from supervisely.app.widgets import (
+    Button,
+    Checkbox,
+    Container,
     Dialog,
-    Text,
     Editor,
-    Container,
-    Button,
     Flexbox,
     NotificationBox,
-    Checkbox,
+    Text,
 )
 
 if sly.is_development():
@@ -90,6 +91,16 @@
             FILTERED_ENTITIES = [entity.id for entity in FILTERED_ENTITIES]
 
 
+FILTERED_DATASETS = []
+selected_datasets = os.getenv("modal.state.selectedDatasets", [])
+if selected_datasets != []:
+    selected_datasets = ast.literal_eval(selected_datasets)
+    nested_datasets = [
+        api.dataset.get_nested(PROJECT_ID, dataset_id) for dataset_id in selected_datasets
+    ]
+    unpacked_datasets = [dataset.id for sublist in nested_datasets for dataset in sublist]
+    FILTERED_DATASETS = selected_datasets + unpacked_datasets
+
 if MODALITY_TYPE == "images":
     BATCH_SIZE = 50
 else:

diff --git a/src/main.py b/src/main.py
@@ -1,29 +1,35 @@
 import threading
 import time
-from supervisely import Application, ProjectInfo, DatasetInfo, Annotation, ProjectMeta, logger
-from fastapi import Request, Response
-
-from src.ui.ui import layout, header
-from src.ui.tabs.configure import update_state, update_nodes, nodes_flow
-from src.ui.tabs.presets import load_json
-from src.ui.tabs.run import run_btn_clicked, error_notification, circle_progress
-
-from src.ui.dtl.Layer import Layer
-from src.ui.dtl.actions.input.images_project.images_project import ImagesProjectAction
-from src.ui.dtl.actions.input.videos_project.videos_project import VideosProjectAction
-from src.ui.dtl.actions.input.filtered_project.filtered_project import FilteredProjectAction
-
-from src.preconfigured.templates import templates
-from src.preconfigured.utils import load_template
 
-from src.compute.dtl_utils.item_descriptor import ImageDescriptor
-from src.utils import LegacyProjectItem
+from fastapi import Request, Response
 
 import src.globals as g
 import src.utils as u
+from src.compute.dtl_utils.item_descriptor import ImageDescriptor
+from src.preconfigured.templates import templates
+from src.preconfigured.utils import load_template
+from src.ui.dtl.actions.input.filtered_project.filtered_project import (
+    FilteredProjectAction,
+)
+from src.ui.dtl.actions.input.images_project.images_project import ImagesProjectAction
+from src.ui.dtl.actions.input.videos_project.videos_project import VideosProjectAction
+from src.ui.dtl.Layer import Layer
+from src.ui.tabs.configure import nodes_flow, update_nodes, update_state
+from src.ui.tabs.presets import load_json
+from src.ui.tabs.run import circle_progress, error_notification, run_btn_clicked
+from src.ui.ui import header, layout
 from src.ui.utils import create_new_layer
 from src.ui.widgets import ApplyCss
-from supervisely.app.widgets import ImageAnnotationPreview, FastTable
+from src.utils import LegacyProjectItem
+from supervisely import (
+    Annotation,
+    Application,
+    DatasetInfo,
+    ProjectInfo,
+    ProjectMeta,
+    logger,
+)
+from supervisely.app.widgets import FastTable, ImageAnnotationPreview
 
 # init widgets that use javascript
 ImageAnnotationPreview()
@@ -86,6 +92,10 @@ def generate_preview_for_project(layer: Layer):
         items = [g.api.image.get_info_by_id(g.FILTERED_ENTITIES[0])]
     elif g.DATASET_ID:
         items = g.api.image.get_list(g.DATASET_ID)
+    elif len(g.FILTERED_DATASETS) > 0:
+        items = []
+        for ds_id in g.FILTERED_DATASETS:
+            items.extend(g.api.image.get_list(ds_id))
     else:
         dss = g.api.dataset.get_list(g.PROJECT_ID, recursive=True)
         if len(dss) > 0:
@@ -151,6 +161,21 @@ def generate_preview_for_project(layer: Layer):
             pr: ProjectInfo = g.api.project.get_info_by_id(g.PROJECT_ID)
             src = [f"{pr.name}/{ds_name}"]
 
+elif g.PROJECT_ID and len(g.FILTERED_DATASETS) > 0:
+    pr: ProjectInfo = g.api.project.get_info_by_id(g.PROJECT_ID)
+    src = [f"{pr.name}/{ds_id}" for ds_id in g.FILTERED_DATASETS]
+    if pr.type == "images":
+        layer = create_new_layer(ImagesProjectAction.name)
+        layer.init_widgets()
+    elif pr.type == "videos":
+        layer = create_new_layer(VideosProjectAction.name)
+        layer.init_widgets()
+    else:
+        raise NotImplementedError(f"Project type {pr.type} is not supported")
+    layer.from_json({"src": src, "settings": {"classes_mapping": "default"}})
+    node = layer.create_node()
+    nodes_flow.add_node(node)
+
 elif g.PROJECT_ID and len(g.FILTERED_ENTITIES) == 0:
     ds_name = "*"
     if g.DATASET_ID:

diff --git a/src/ui/dtl/actions/input/images_project/images_project.py b/src/ui/dtl/actions/input/images_project/images_project.py
@@ -5,6 +5,19 @@
 import src.globals as g
 import src.utils as utils
 from src.ui.dtl import SourceAction
+from src.ui.dtl.actions.input.images_project.layout.src_classes import (
+    create_classes_selector_widgets,
+)
+from src.ui.dtl.actions.input.images_project.layout.src_input_data import (
+    create_input_data_selector_widgets,
+)
+from src.ui.dtl.actions.input.images_project.layout.src_layout import (
+    create_settings_options,
+    create_src_options,
+)
+from src.ui.dtl.actions.input.images_project.layout.src_tags import (
+    create_tags_selector_widgets,
+)
 from src.ui.dtl.Layer import Layer
 from src.ui.dtl.utils import (
     classes_list_settings_changed_meta,
@@ -21,19 +34,8 @@
     tags_list_settings_changed_meta,
     tags_list_to_mapping,
 )
-from supervisely.app.content import StateJson
-from src.ui.dtl.actions.input.images_project.layout.src_classes import (
-    create_classes_selector_widgets,
-)
-from src.ui.dtl.actions.input.images_project.layout.src_input_data import (
-    create_input_data_selector_widgets,
-)
-from src.ui.dtl.actions.input.images_project.layout.src_layout import (
-    create_settings_options,
-    create_src_options,
-)
-from src.ui.dtl.actions.input.images_project.layout.src_tags import create_tags_selector_widgets
 from supervisely import ProjectMeta
+from supervisely.app.content import StateJson
 from supervisely.app.widgets import Button
 
 
@@ -440,7 +442,12 @@ def _set_src_from_json(srcs: List[str]):
                     if dataset_name == "*":
                         datasets.extend(utils.get_all_datasets(project_info.id))
                     else:
-                        datasets.append(utils.get_dataset_by_name(dataset_name, project_info.id))
+                        if dataset_name.isdigit():
+                            datasets.append(utils.get_dataset_by_id(int(dataset_name)))
+                        else:
+                            datasets.append(
+                                utils.get_dataset_by_name(dataset_name, project_info.id)
+                            )
                 if project_not_found is False:
                     # set datasets to widget
                     src_input_data_sidebar_dataset_selector.set_project_id(project_info.id)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		docker build -t supervisely/data-nodes:1.0.28 . && \
		docker push supervisely/data-nodes:1.0.28