Add Workflow features and various error fixes (#84)

* Implement workflows Co-authored-by: Sergey <[email protected]> * Refactor workflow * Refactor workflow * Fix isinstance for data_layers * Fix created_labeling_jobs items extraction * Improve error handling in workflow output functions * Update workflow icon in workflow_output function * Refactor workflow and improve error handling in workflow functions * Upload archive to TASKID folder * Save archive name to preset * Add workflow loggers * Update workflow loggers * Add refresh project button * Add refresh project button for videos modality * add create_refresh_projects_btn func * Refactor workflow loggers * Add input labeling job to workflow * Fix typo in property name * Fix all datasets checkbox in the input project layers and save project name in settings in OutputProject Layer * Update min instance version and docker image tag * Refactor workflow to include new project from jobs into output * Fix job_project_infos list generator * Refactor workflow to remove duplicate project_infos * update docker version --------- Co-authored-by: cxnt <[email protected]> Co-authored-by: Sergey <[email protected]>
supervisely-ecosystem · Aug 29, 2024 · 8ad95a6 · 8ad95a6
1 parent 819d5c6
commit 8ad95a6
Show file tree

Hide file tree

Showing 18 changed files with 345 additions and 20 deletions.
diff --git a/config.json b/config.json
@@ -4,7 +4,7 @@
 	"description": "[Beta] Drag and drop interface for building custom DataOps pipelines",
 	"entrypoint": "python -m uvicorn src.main:app --host 0.0.0.0 --port 8000",
 	"headless": false,
-	"docker_image": "supervisely/data-nodes:1.0.16",
+	"docker_image": "supervisely/data-nodes:1.0.18",
 	"modal_template": "src/modal.html",
 	"modal_template_state": {
 		"modalityType": "images"
@@ -23,5 +23,5 @@
 			"files_file"
 		]
 	},
-	"instance_version": "6.11.8"
+	"instance_version": "6.11.10"
 }
diff --git a/src/compute/layers/save/ExportArchiveLayer.py b/src/compute/layers/save/ExportArchiveLayer.py
@@ -43,6 +43,7 @@ class ExportArchiveLayer(Layer):
             "settings": {
                 "type": "object",
                 "properties": {
+                    "archive_name": {"type": "string"},
                     "images": {"type": "boolean"},  # Deprecated
                     "annotations": {"type": "boolean"},  # Deprecated
                     "visualize": {"type": "boolean"},

diff --git a/src/compute/layers/save/ExportArchiveWithMasksLayer.py b/src/compute/layers/save/ExportArchiveWithMasksLayer.py
@@ -32,8 +32,9 @@ class ExportArchiveWithMasksLayer(Layer):
         "properties": {
             "settings": {
                 "type": "object",
-                "required": ["masks_machine", "masks_human"],
+                "required": ["archive_name", "masks_machine", "masks_human"],
                 "properties": {
+                    "archive_name": {"type": "string"},
                     "gt_machine_color": {
                         "type": "object",
                         "patternProperties": {".*": {"$ref": "#/definitions/color"}},

diff --git a/src/compute/main.py b/src/compute/main.py
@@ -73,7 +73,7 @@ def main(
     if not g.pipeline_running:
         return
 
-    logger.info("DTL started")
+    logger.info("Pipeline started")
     helper = DtlHelper()
 
     try:
@@ -113,12 +113,12 @@ def main(
 
     except CustomException as e:
         circle_progress.hide()
-        # logger.error("Error occurred on DTL-graph initialization step!")
+        # logger.error("Error occurred on Pipeline graph initialization step!")
         # e.log()
         raise e
     except Exception as e:
         circle_progress.hide()
-        # logger.error("Error occurred on DTL-graph initialization step!", exc_info=str(e))
+        # logger.error("Error occurred on Pipeline graph initialization step!", exc_info=str(e))
         raise e
 
     total = net.get_total_elements()
@@ -187,7 +187,7 @@ def main(
         return
 
     logger.info(
-        "DTL finished",
+        "Pipeline finished",
         extra={"event_type": EventType.DTL_APPLIED, "new_proj_size": results_counter},
     )
     total_pipeline_time_end = time()
@@ -200,4 +200,4 @@ def main(
 if __name__ == "__main__":
     if os.getenv("DEBUG_LOG_TO_FILE", None):
         sly_logger.add_default_logging_into_file(logger, DtlPaths().debug_dir)
-    logging_utils.main_wrapper("DTL", main)
+    logging_utils.main_wrapper("Pipeline", main)
diff --git a/src/globals.py b/src/globals.py
@@ -23,14 +23,23 @@
 
 api = sly.Api()
 
+TASK_ID = sly.env.task_id(raise_not_found=False)
 TEAM_ID = sly.env.team_id()
 WORKSPACE_ID = sly.env.workspace_id()
 USER_ID = sly.env.user_id()
 DATA_DIR = "sly_task_data/data"
 RESULTS_DIR = "sly_task_data/results"
 PREVIEW_DIR = "sly_task_data/preview"
+WORKFLOW_DIR = "sly_task_data/workflow"
 STATIC_DIR = "static"
 
+if TASK_ID is not None:
+    OFFLINE_SESSION_PATH = f"/offline-sessions/{TASK_ID}/workflows"
+else:
+    OFFLINE_SESSION_PATH = f"/TEST_WORKFLOW/task_id/workflows"
+WORKFLOW_ID = 1
+
+
 sly.fs.mkdir(DATA_DIR, True)
 sly.fs.mkdir(RESULTS_DIR, True)
 sly.fs.mkdir(PREVIEW_DIR, True)

diff --git a/src/ui/dtl/actions/input/images_project/images_project.py b/src/ui/dtl/actions/input/images_project/images_project.py
@@ -70,6 +70,7 @@ def create_new_layer(cls, layer_id: Optional[str] = None):
             # Sidebar
             src_input_data_sidebar_dataset_selector,
             src_input_data_sidebar_save_btn,
+            src_input_data_sidebar_refresh_btn,
             src_input_data_sidebar_empty_ds_notification,
             src_input_data_sidebar_widgets_container,
             # Preview
@@ -128,6 +129,18 @@ def create_new_layer(cls, layer_id: Optional[str] = None):
         )
         update_preview_btn.disable()
 
+        # Refresh project on SELECT
+        @src_input_data_sidebar_refresh_btn.click
+        def src_input_data_sidebar_refresh_btn_btn_cb():
+            # Get currently selected project and datasets
+            current_project = src_input_data_sidebar_dataset_selector.get_selected_project_id()
+            current_datasets = src_input_data_sidebar_dataset_selector.get_selected_ids()
+
+            # Refresh project list and set current project and datasets if any
+            src_input_data_sidebar_dataset_selector.set_workspace_id(g.WORKSPACE_ID)
+            src_input_data_sidebar_dataset_selector.set_project_id(current_project)
+            src_input_data_sidebar_dataset_selector.set_dataset_ids(current_datasets)
+
         def _set_src_preview():
             src_preview_text = ""
             if _current_info is not None:

diff --git a/src/ui/dtl/actions/input/images_project/layout/src_input_data.py b/src/ui/dtl/actions/input/images_project/layout/src_input_data.py
@@ -1,6 +1,7 @@
 import src.globals as g
 from src.ui.dtl.utils import (
     create_save_btn,
+    create_refresh_projects_btn,
     get_text_font_size,
     get_set_settings_button_style,
     get_set_settings_container,
@@ -11,7 +12,7 @@
     Button,
     Container,
     NotificationBox,
-    SelectDataset,
+    Flexbox,
     SelectDatasetTree,
     Text,
     ProjectThumbnail,
@@ -24,16 +25,18 @@ def create_input_data_selector_widgets():
     src_input_data_sidebar_dataset_selector = SelectDatasetTree(
         multiselect=True,
         flat=True,
-        select_all_datasets=True,
+        select_all_datasets=False,
         allowed_project_types=[ProjectType.IMAGES],
         always_open=False,
         compact=False,
         team_is_selectable=False,
         workspace_is_selectable=False,
         append_to_body=False,
+        show_select_all_datasets_checkbox=True,
     )
 
     src_input_data_sidebar_save_btn = create_save_btn()
+    src_input_data_sidebar_refresh_btn = create_refresh_projects_btn()
     src_input_data_sidebar_save_btn.disable()
     src_input_data_sidebar_empty_ds_notification = NotificationBox(
         title="No datasets selected", description="Select at lease one dataset"
@@ -42,7 +45,13 @@ def create_input_data_selector_widgets():
         widgets=[
             src_input_data_sidebar_dataset_selector,
             src_input_data_sidebar_empty_ds_notification,
-            src_input_data_sidebar_save_btn,
+            Flexbox(
+                widgets=[
+                    src_input_data_sidebar_save_btn,
+                    src_input_data_sidebar_refresh_btn,
+                ],
+                gap=10,
+            ),
         ]
     )
     # Preview
@@ -87,6 +96,7 @@ def create_input_data_selector_widgets():
         # Sidebar
         src_input_data_sidebar_dataset_selector,
         src_input_data_sidebar_save_btn,
+        src_input_data_sidebar_refresh_btn,
         src_input_data_sidebar_empty_ds_notification,
         src_input_data_sidebar_widgets_container,
         # Preview

diff --git a/src/ui/dtl/actions/input/images_project/layout/src_layout.py b/src/ui/dtl/actions/input/images_project/layout/src_layout.py
@@ -14,7 +14,7 @@ def create_src_options(
                 sidebar_component=NodesFlow.WidgetOptionComponent(
                     src_input_data_sidebar_widgets_container
                 ),
-                sidebar_width=300,
+                sidebar_width=325,
             ),
         ),
         NodesFlow.Node.Option(

diff --git a/src/ui/dtl/actions/input/videos_project/layout/src_input_data.py b/src/ui/dtl/actions/input/videos_project/layout/src_input_data.py
@@ -1,6 +1,7 @@
 import src.globals as g
 from src.ui.dtl.utils import (
     create_save_btn,
+    create_refresh_projects_btn,
     get_text_font_size,
     get_set_settings_button_style,
     get_set_settings_container,
@@ -11,7 +12,7 @@
     Button,
     Container,
     NotificationBox,
-    SelectDataset,
+    Flexbox,
     SelectDatasetTree,
     Text,
     ProjectThumbnail,
@@ -24,16 +25,18 @@ def create_input_data_selector_widgets():
     src_input_data_sidebar_dataset_selector = SelectDatasetTree(
         multiselect=True,
         flat=True,
-        select_all_datasets=True,
+        select_all_datasets=False,
         allowed_project_types=[ProjectType.VIDEOS],
         always_open=False,
         compact=False,
         team_is_selectable=False,
         workspace_is_selectable=False,
         append_to_body=False,
+        show_select_all_datasets_checkbox=True,
     )
 
     src_input_data_sidebar_save_btn = create_save_btn()
+    src_input_data_sidebar_refresh_btn = create_refresh_projects_btn()
     src_input_data_sidebar_save_btn.disable()
     src_input_data_sidebar_empty_ds_notification = NotificationBox(
         title="No datasets selected", description="Select at lease one dataset"
@@ -42,7 +45,13 @@ def create_input_data_selector_widgets():
         widgets=[
             src_input_data_sidebar_dataset_selector,
             src_input_data_sidebar_empty_ds_notification,
-            src_input_data_sidebar_save_btn,
+            Flexbox(
+                widgets=[
+                    src_input_data_sidebar_save_btn,
+                    src_input_data_sidebar_refresh_btn,
+                ],
+                gap=10,
+            ),
         ]
     )
     # Preview
@@ -87,6 +96,7 @@ def create_input_data_selector_widgets():
         # Sidebar
         src_input_data_sidebar_dataset_selector,
         src_input_data_sidebar_save_btn,
+        src_input_data_sidebar_refresh_btn,
         src_input_data_sidebar_empty_ds_notification,
         src_input_data_sidebar_widgets_container,
         # Preview

diff --git a/src/ui/dtl/actions/input/videos_project/layout/src_layout.py b/src/ui/dtl/actions/input/videos_project/layout/src_layout.py
@@ -14,7 +14,7 @@ def create_src_options(
                 sidebar_component=NodesFlow.WidgetOptionComponent(
                     src_input_data_sidebar_widgets_container
                 ),
-                sidebar_width=300,
+                sidebar_width=325,
             ),
         ),
         NodesFlow.Node.Option(

diff --git a/src/ui/dtl/actions/input/videos_project/videos_project.py b/src/ui/dtl/actions/input/videos_project/videos_project.py
@@ -70,6 +70,7 @@ def create_new_layer(cls, layer_id: Optional[str] = None):
             # Sidebar
             src_input_data_sidebar_dataset_selector,
             src_input_data_sidebar_save_btn,
+            src_input_data_sidebar_refresh_btn,
             src_input_data_sidebar_empty_ds_notification,
             src_input_data_sidebar_widgets_container,
             # Preview
@@ -118,6 +119,18 @@ def create_new_layer(cls, layer_id: Optional[str] = None):
         ) = create_tags_selector_widgets()
         # ----------------------------
 
+        # Refresh project on SELECT
+        @src_input_data_sidebar_refresh_btn.click
+        def src_input_data_sidebar_refresh_btn_btn_cb():
+            # Get currently selected project and datasets
+            current_project = src_input_data_sidebar_dataset_selector.get_selected_project_id()
+            current_datasets = src_input_data_sidebar_dataset_selector.get_selected_ids()
+
+            # Refresh project list and set current project and datasets if any
+            src_input_data_sidebar_dataset_selector.set_workspace_id(g.WORKSPACE_ID)
+            src_input_data_sidebar_dataset_selector.set_project_id(current_project)
+            src_input_data_sidebar_dataset_selector.set_dataset_ids(current_datasets)
+
         def _set_src_preview():
             src_preview_text = ""
             if _current_info is not None:
@@ -568,5 +581,5 @@ def create_options(src: List[str], dst: List[str], settings: dict) -> dict:
             get_settings=get_settings,
             data_changed_cb=data_changed_cb,
             postprocess_cb=postprocess_cb,
-            need_preview=False
+            need_preview=False,
         )
diff --git a/src/ui/dtl/actions/output/export_archive/export_archive.py b/src/ui/dtl/actions/output/export_archive/export_archive.py
@@ -20,14 +20,17 @@ class ExportArchiveAction(OutputAction):
     @classmethod
     def create_new_layer(cls, layer_id: Optional[str] = None) -> Layer:
         save_path_text = Text("Archive name", status="text", font_size=get_text_font_size())
-        save_path_input = Input(value="", placeholder="Enter archive name", size="small")
+        save_path_input = Input(
+            value="", placeholder="Enter archive name (without extension)", size="small"
+        )
         visualize_checkbox = Checkbox("Visualize")
         if g.MODALITY_TYPE == "videos":
             visualize_checkbox.hide()
 
         def get_settings(options_json: dict) -> dict:
             """This function is used to get settings from options json we get from NodesFlow widget"""
             return {
+                "archive_name": save_path_input.get_value(),
                 "visualize": visualize_checkbox.is_checked(),
             }
 
@@ -49,6 +52,9 @@ def _set_settings_from_json(settings: dict):
             else:
                 visualize_checkbox.uncheck()
 
+            archive_name = settings.get("archive_name", "")
+            save_path_input.set_value(archive_name)
+
         def create_options(src: list, dst: list, settings: dict) -> dict:
             _set_settings_from_json(settings)
             settings_options = [

diff --git a/src/ui/dtl/actions/output/export_archive_with_masks/export_archive_with_masks.py b/src/ui/dtl/actions/output/export_archive_with_masks/export_archive_with_masks.py
@@ -31,7 +31,9 @@ def create_new_layer(cls, layer_id: Optional[str] = None) -> Layer:
         _current_meta = ProjectMeta()
 
         destination_text = Text("Destination", status="text", font_size=get_text_font_size())
-        destination_input = Input(value="", placeholder="Enter Team Files path", size="small")
+        destination_input = Input(
+            value="", placeholder="Enter archive name (without extension)", size="small"
+        )
 
         add_human_masks_checkbox = Checkbox("Add human masks")
         add_machine_masks_checkbox = Checkbox("Add machine masks")
@@ -177,6 +179,7 @@ def get_settings(options_json: dict) -> dict:
                 gt_machine_color = saved_machine_classes_colors_settings
 
             return {
+                "archive_name": destination_input.get_value(),
                 "masks_human": masks_human,
                 "masks_machine": masks_machine,
                 "gt_human_color": gt_human_color,
@@ -253,6 +256,9 @@ def _set_settings_from_json(settings):
                 add_machine_masks_checkbox.uncheck()
                 machine_masks_edit_container.hide()
 
+            archive_name = settings.get("archive_name", "")
+            destination_input.set_value(archive_name)
+
         @human_classes_colors_save_btn.click
         def human_classes_saved():
             _save_human_classes_colors()

diff --git a/src/ui/dtl/actions/output/output_project/output_project.py b/src/ui/dtl/actions/output/output_project/output_project.py
@@ -122,6 +122,15 @@ def _update_preview():
 
         def get_settings(options_json: dict):
             nonlocal saved_settings
+            if not is_existing_project.is_checked():
+                settings = {
+                    "project_name": new_project_name_input.get_value(),
+                    "dataset_option": None,
+                    "dataset_name": None,
+                    "dataset_id": None,
+                    "merge_different_meta": False,
+                }
+                saved_settings = settings
             return {
                 **saved_settings,
                 "is_existing_project": is_existing_project.is_checked(),

diff --git a/src/ui/dtl/utils.py b/src/ui/dtl/utils.py
@@ -638,6 +638,10 @@ def create_set_default_btn() -> Button:
     return Button("Set Default", button_type="info", plain=True, icon="zmdi zmdi-refresh")
 
 
+def create_refresh_projects_btn() -> Button:
+    return Button("Refresh Projects", button_type="info", plain=True, icon="zmdi zmdi-refresh")
+
+
 def create_sidebar_btn_container(
     save_btn: Button, default_btn: Button = None, need_separator: bool = True
 ):