From fef3e92232b87831597c65247e987467eb8413c2 Mon Sep 17 00:00:00 2001 From: Pavel Bartsits <48913536+cxnt@users.noreply.github.com> Date: Fri, 17 Jan 2025 13:00:44 +0400 Subject: [PATCH] Add support for running the app from selected datasets (#98) * Add support for running the app from selected datasets * Fix annotation download to use image IDs from the images list * Update Docker image and instance version in config.json * Update dev_requirements.txt * Add Docker publish script for building and pushing data nodes image --- config.json | 4 +- dev_requirements.txt | 2 +- docker/publish.sh | 2 + src/globals.py | 25 +++++--- src/main.py | 61 +++++++++++++------ .../input/images_project/images_project.py | 33 ++++++---- 6 files changed, 86 insertions(+), 41 deletions(-) create mode 100644 docker/publish.sh diff --git a/config.json b/config.json index 8f2f268a..27131f67 100644 --- a/config.json +++ b/config.json @@ -4,7 +4,7 @@ "description": "[Beta] Drag and drop interface for building custom DataOps pipelines", "entrypoint": "python -m uvicorn src.main:app --host 0.0.0.0 --port 8000", "headless": false, - "docker_image": "supervisely/data-nodes:1.0.27", + "docker_image": "supervisely/data-nodes:1.0.28", "modal_template": "src/modal.html", "modal_template_state": { "modalityType": "images" @@ -23,5 +23,5 @@ "files_file" ] }, - "instance_version": "6.12.12" + "instance_version": "6.12.22" } diff --git a/dev_requirements.txt b/dev_requirements.txt index 657c26c0..5d79883c 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,6 +1,6 @@ git+https://github.com/supervisely/supervisely.git@optimize-index -# supervisely==6.73.203 +# supervisely==6.73.280 jsonschema==4.19.2 networkx==3.1 scikit-image==0.21.0 diff --git a/docker/publish.sh b/docker/publish.sh new file mode 100644 index 00000000..ecb6d249 --- /dev/null +++ b/docker/publish.sh @@ -0,0 +1,2 @@ +docker build -t supervisely/data-nodes:1.0.28 . && \ +docker push supervisely/data-nodes:1.0.28 diff --git a/src/globals.py b/src/globals.py index bb2cf685..22cbda12 100644 --- a/src/globals.py +++ b/src/globals.py @@ -1,19 +1,20 @@ -import os import ast - +import os +from distutils.util import strtobool from queue import Queue + from dotenv import load_dotenv -from distutils.util import strtobool + import supervisely as sly from supervisely.app.widgets import ( + Button, + Checkbox, + Container, Dialog, - Text, Editor, - Container, - Button, Flexbox, NotificationBox, - Checkbox, + Text, ) if sly.is_development(): @@ -90,6 +91,16 @@ FILTERED_ENTITIES = [entity.id for entity in FILTERED_ENTITIES] +FILTERED_DATASETS = [] +selected_datasets = os.getenv("modal.state.selectedDatasets", []) +if selected_datasets != []: + selected_datasets = ast.literal_eval(selected_datasets) + nested_datasets = [ + api.dataset.get_nested(PROJECT_ID, dataset_id) for dataset_id in selected_datasets + ] + unpacked_datasets = [dataset.id for sublist in nested_datasets for dataset in sublist] + FILTERED_DATASETS = selected_datasets + unpacked_datasets + if MODALITY_TYPE == "images": BATCH_SIZE = 50 else: diff --git a/src/main.py b/src/main.py index 1b60700a..536edbd6 100644 --- a/src/main.py +++ b/src/main.py @@ -1,29 +1,35 @@ import threading import time -from supervisely import Application, ProjectInfo, DatasetInfo, Annotation, ProjectMeta, logger -from fastapi import Request, Response - -from src.ui.ui import layout, header -from src.ui.tabs.configure import update_state, update_nodes, nodes_flow -from src.ui.tabs.presets import load_json -from src.ui.tabs.run import run_btn_clicked, error_notification, circle_progress - -from src.ui.dtl.Layer import Layer -from src.ui.dtl.actions.input.images_project.images_project import ImagesProjectAction -from src.ui.dtl.actions.input.videos_project.videos_project import VideosProjectAction -from src.ui.dtl.actions.input.filtered_project.filtered_project import FilteredProjectAction - -from src.preconfigured.templates import templates -from src.preconfigured.utils import load_template -from src.compute.dtl_utils.item_descriptor import ImageDescriptor -from src.utils import LegacyProjectItem +from fastapi import Request, Response import src.globals as g import src.utils as u +from src.compute.dtl_utils.item_descriptor import ImageDescriptor +from src.preconfigured.templates import templates +from src.preconfigured.utils import load_template +from src.ui.dtl.actions.input.filtered_project.filtered_project import ( + FilteredProjectAction, +) +from src.ui.dtl.actions.input.images_project.images_project import ImagesProjectAction +from src.ui.dtl.actions.input.videos_project.videos_project import VideosProjectAction +from src.ui.dtl.Layer import Layer +from src.ui.tabs.configure import nodes_flow, update_nodes, update_state +from src.ui.tabs.presets import load_json +from src.ui.tabs.run import circle_progress, error_notification, run_btn_clicked +from src.ui.ui import header, layout from src.ui.utils import create_new_layer from src.ui.widgets import ApplyCss -from supervisely.app.widgets import ImageAnnotationPreview, FastTable +from src.utils import LegacyProjectItem +from supervisely import ( + Annotation, + Application, + DatasetInfo, + ProjectInfo, + ProjectMeta, + logger, +) +from supervisely.app.widgets import FastTable, ImageAnnotationPreview # init widgets that use javascript ImageAnnotationPreview() @@ -86,6 +92,10 @@ def generate_preview_for_project(layer: Layer): items = [g.api.image.get_info_by_id(g.FILTERED_ENTITIES[0])] elif g.DATASET_ID: items = g.api.image.get_list(g.DATASET_ID) + elif len(g.FILTERED_DATASETS) > 0: + items = [] + for ds_id in g.FILTERED_DATASETS: + items.extend(g.api.image.get_list(ds_id)) else: dss = g.api.dataset.get_list(g.PROJECT_ID, recursive=True) if len(dss) > 0: @@ -151,6 +161,21 @@ def generate_preview_for_project(layer: Layer): pr: ProjectInfo = g.api.project.get_info_by_id(g.PROJECT_ID) src = [f"{pr.name}/{ds_name}"] +elif g.PROJECT_ID and len(g.FILTERED_DATASETS) > 0: + pr: ProjectInfo = g.api.project.get_info_by_id(g.PROJECT_ID) + src = [f"{pr.name}/{ds_id}" for ds_id in g.FILTERED_DATASETS] + if pr.type == "images": + layer = create_new_layer(ImagesProjectAction.name) + layer.init_widgets() + elif pr.type == "videos": + layer = create_new_layer(VideosProjectAction.name) + layer.init_widgets() + else: + raise NotImplementedError(f"Project type {pr.type} is not supported") + layer.from_json({"src": src, "settings": {"classes_mapping": "default"}}) + node = layer.create_node() + nodes_flow.add_node(node) + elif g.PROJECT_ID and len(g.FILTERED_ENTITIES) == 0: ds_name = "*" if g.DATASET_ID: diff --git a/src/ui/dtl/actions/input/images_project/images_project.py b/src/ui/dtl/actions/input/images_project/images_project.py index f9f89fa0..c4430c91 100644 --- a/src/ui/dtl/actions/input/images_project/images_project.py +++ b/src/ui/dtl/actions/input/images_project/images_project.py @@ -5,6 +5,19 @@ import src.globals as g import src.utils as utils from src.ui.dtl import SourceAction +from src.ui.dtl.actions.input.images_project.layout.src_classes import ( + create_classes_selector_widgets, +) +from src.ui.dtl.actions.input.images_project.layout.src_input_data import ( + create_input_data_selector_widgets, +) +from src.ui.dtl.actions.input.images_project.layout.src_layout import ( + create_settings_options, + create_src_options, +) +from src.ui.dtl.actions.input.images_project.layout.src_tags import ( + create_tags_selector_widgets, +) from src.ui.dtl.Layer import Layer from src.ui.dtl.utils import ( classes_list_settings_changed_meta, @@ -21,19 +34,8 @@ tags_list_settings_changed_meta, tags_list_to_mapping, ) -from supervisely.app.content import StateJson -from src.ui.dtl.actions.input.images_project.layout.src_classes import ( - create_classes_selector_widgets, -) -from src.ui.dtl.actions.input.images_project.layout.src_input_data import ( - create_input_data_selector_widgets, -) -from src.ui.dtl.actions.input.images_project.layout.src_layout import ( - create_settings_options, - create_src_options, -) -from src.ui.dtl.actions.input.images_project.layout.src_tags import create_tags_selector_widgets from supervisely import ProjectMeta +from supervisely.app.content import StateJson from supervisely.app.widgets import Button @@ -440,7 +442,12 @@ def _set_src_from_json(srcs: List[str]): if dataset_name == "*": datasets.extend(utils.get_all_datasets(project_info.id)) else: - datasets.append(utils.get_dataset_by_name(dataset_name, project_info.id)) + if dataset_name.isdigit(): + datasets.append(utils.get_dataset_by_id(int(dataset_name))) + else: + datasets.append( + utils.get_dataset_by_name(dataset_name, project_info.id) + ) if project_not_found is False: # set datasets to widget src_input_data_sidebar_dataset_selector.set_project_id(project_info.id)