Shifted to engine 2.0, added support for nested datasets

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 100
+ignore = E203, E501, W503, E722, W605, E721

.gitignore

@@ -5,4 +5,5 @@ debug
 .venv
 supervisely
 __pycache__
-.vscode
+storage/
+.DS_Store

.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python: Current File",
+            "type": "python",
+            "request": "launch",
+            "program": "src/main.py",
+            "console": "integratedTerminal",
+            "justMyCode": true
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,35 @@
+{
+  "files.exclude": {
+    "**/__pycache__": true,
+    "build": true,
+    "supervisely.egg-info": true,
+    // ".venv": true
+  },
+  "python.defaultInterpreterPath": ".venv/bin/python",
+  "editor.formatOnSave": true,
+  "editor.formatOnPaste": true,
+  "editor.formatOnType": true,
+  "black-formatter.args": ["--line-length", "100"],
+  "[html]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[json]": {
+    "editor.defaultFormatter": "esbenp.prettier-vscode"
+  },
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter",
+    "editor.formatOnSave": true,
+    "editor.codeActionsOnSave": {
+      "source.organizeImports": "explicit"
+    }
+  },
+  "isort.args": ["--profile", "black"],
+  "debug.inlineValues": "off",
+  "python.analysis.typeCheckingMode": "off",
+  "python.analysis.autoImportCompletions": false,
+  "autoDocstring.docstringFormat": "sphinx",
+  "autoDocstring.customTemplatePath": "docs/.mustache",
+  "python.testing.pytestArgs": ["tests/inference_cache"],
+  "python.testing.unittestEnabled": false,
+  "python.testing.pytestEnabled": true
+}

config.json

@@ -1,12 +1,13 @@
 {
     "name": "Export as masks",
     "type": "app",
+    "version": "2.0.0",
     "categories": [
         "images",
         "export"
     ],
     "description": "For semantic and instance segmentation tasks",
-    "docker_image": "supervisely/import-export:6.73.22",
+    "docker_image": "supervisely/import-export:6.73.48",
     "instance_version": "6.8.73",
     "main_script": "src/main.py",
     "modal_template": "src/modal.html",

local.env

@@ -1,17 +1,14 @@
 PYTHONUNBUFFERED=1
 
-LOG_LEVEL="debug"
-
-TASK_ID=52548
-context.teamId=506
-context.workspaceId=942
-modal.state.slyProjectId=34036
+context.teamId=448
+context.workspaceId=690
+modal.state.slyProjectId=35637
 
 modal.state.humanMasks="True"
 modal.state.machineMasks="True"
 modal.state.instanceMasks="True"
 modal.state.thickness=3
 
-DEBUG_APP_DIR="debug/app_debug_data"
-DEBUG_CACHE_DIR="debug/app_debug_cache"
+# DEBUG_APP_DIR="debug/app_debug_data"
+# DEBUG_CACHE_DIR="debug/app_debug_cache"
 

src/globals.py

@@ -4,12 +4,10 @@
 
 import supervisely as sly
 from dotenv import load_dotenv
-
-
 from PIL import Image
-from supervisely.app.v1.app_service import AppService
 
-app_root_directory = os.path.dirname(os.getcwd())
+cwd = os.getcwd()
+app_root_directory = os.path.dirname(cwd)
 sys.path.append(app_root_directory)
 sys.path.append(os.path.join(app_root_directory, "src"))
 print(f"App root directory: {app_root_directory}")
@@ -19,25 +17,31 @@
     load_dotenv("local.env")
     load_dotenv(os.path.expanduser("~/supervisely.env"))
 
-TASK_ID = os.environ["TASK_ID"]
 TEAM_ID = int(os.environ["context.teamId"])
 WORKSPACE_ID = int(os.environ["context.workspaceId"])
 PROJECT_ID = int(os.environ["modal.state.slyProjectId"])
 
-my_app = AppService()
+sly.logger.info(f"Team ID: {TEAM_ID}, Workspace ID: {WORKSPACE_ID}, Project ID: {PROJECT_ID}")
+
+api = sly.Api.from_env()
 
 Image.MAX_IMAGE_PIXELS = None
 HUMAN_MASKS = bool(strtobool(os.environ["modal.state.humanMasks"]))
 MACHINE_MASKS = bool(strtobool(os.environ["modal.state.machineMasks"]))
 INSTANCE_MASKS = bool(strtobool(os.environ["modal.state.instanceMasks"]))
 THICKNESS = int(os.environ["modal.state.thickness"])
 
+sly.logger.info(
+    f"Export human masks: {HUMAN_MASKS}, machine masks: {MACHINE_MASKS}, "
+    f"instance masks: {INSTANCE_MASKS}, thickness: {THICKNESS}"
+)
+
 SIZE_LIMIT = 10 if sly.is_community() else 100
 SIZE_LIMIT_BYTES = SIZE_LIMIT * 1024 * 1024 * 1024
 SPLIT_MODE = "MB"
-SPLIT_SIZE = 500 # do not increase this value (memory issues)
+SPLIT_SIZE = 500  # do not increase this value (memory issues)
 # SPLIT_SIZE_BYTES = SPLIT_SIZE * 1024 * 1024 * 1024
 
-STORAGE_DIR = sly.app.get_data_dir()
-if sly.fs.dir_exists(STORAGE_DIR):
-    sly.fs.clean_dir(STORAGE_DIR)
+STORAGE_DIR = os.path.join(cwd, "storage")
+sly.fs.mkdir(STORAGE_DIR, remove_content_if_exists=True)
+sly.logger.debug(f"Storage directory: {STORAGE_DIR}")

src/main.py

@@ -1,7 +1,5 @@
 import os
 
-import cv2
-
 import numpy as np
 import supervisely as sly
 from supervisely._utils import generate_free_name
@@ -11,14 +9,17 @@
 import globals as g
 
 
-@g.my_app.callback("export_as_masks")
-@sly.timeit
-def export_as_masks(api: sly.Api, task_id, context, state, app_logger):
+def export_as_masks(api: sly.Api):
     project_info = api.project.get_info_by_id(g.PROJECT_ID)
-    project_dir = f.download_project(api, project_info.name, project_info.id)
+    project_dir = os.path.join(g.STORAGE_DIR, f"{project_info.id}_{project_info.name}")
+    sly.logger.debug(f"Project will be saved to: {project_dir}")
+    sly.Project.download(api, g.PROJECT_ID, project_dir)
+    sly.logger.debug("Project downloaded.")
 
     if g.MACHINE_MASKS or g.HUMAN_MASKS or g.INSTANCE_MASKS:
+        sly.logger.debug("Started mask creation...")
         project = sly.Project(directory=project_dir, mode=sly.OpenMode.READ)
+        sly.logger.debug(f"Readed local project: {project.name}")
         machine_colors = None
         if g.MACHINE_MASKS:
             machine_colors = {
@@ -32,6 +33,7 @@ def export_as_masks(api: sly.Api, task_id, context, state, app_logger):
             )
 
         for dataset in project:
+            sly.logger.info(f"Working with dataset {dataset.name}...")
             ds_progress = sly.Progress(
                 "Processing dataset: {!r}/{!r}".format(project.name, dataset.name),
                 total_cnt=len(dataset),
@@ -56,6 +58,7 @@ def export_as_masks(api: sly.Api, task_id, context, state, app_logger):
                 mask_img_name = f"{os.path.splitext(item_name)[0]}.png"
 
                 if g.HUMAN_MASKS:
+                    sly.logger.debug("Creating human masks...")
                     raw_img = sly.image.read(item_paths.img_path)
                     overlay = raw_img.copy()
 
@@ -78,11 +81,10 @@ def export_as_masks(api: sly.Api, task_id, context, state, app_logger):
                     )
 
                 if g.MACHINE_MASKS:
+                    sly.logger.debug("Creating macnhine masks...")
                     machine_mask = np.zeros(shape=ann.img_size + (3,), dtype=np.uint8)
 
-                    sorted_labels = sorted(
-                        ann.labels, key=lambda x: x.area, reverse=True
-                    )
+                    sorted_labels = sorted(ann.labels, key=lambda x: x.area, reverse=True)
 
                     for label in sorted_labels:
                         label.geometry.draw(
@@ -94,6 +96,7 @@ def export_as_masks(api: sly.Api, task_id, context, state, app_logger):
                     f.convert2gray_and_save(machine_mask_path, machine_mask)
 
                 if g.INSTANCE_MASKS:
+                    sly.logger.debug("Creating instance masks...")
                     used_names = []
                     for label in ann.labels:
                         mask_name = generate_free_name(
@@ -109,42 +112,19 @@ def export_as_masks(api: sly.Api, task_id, context, state, app_logger):
                                 mask_name,
                             )
                         )
-                        instance_mask = np.zeros(
-                            shape=ann.img_size + (3,), dtype=np.uint8
-                        )
+                        instance_mask = np.zeros(shape=ann.img_size + (3,), dtype=np.uint8)
                         label.geometry.draw(
                             instance_mask, color=[255, 255, 255], thickness=g.THICKNESS
                         )
                         f.convert2gray_and_save(instance_mask_path, instance_mask)
 
                     ds_progress.iter_done_report()
-        sly.logger.info("Finished masks rendering.".format(project_info.name))
-
-    f.upload_result_archive(
-        api=api,
-        task_id=task_id,
-        project_id=project_info.id,
-        project_name=project_info.name,
-        project_dir=project_dir,
-        app_logger=app_logger,
-    )
-
-    g.my_app.stop()
-
-
-def main():
-    sly.logger.info(
-        "Input arguments",
-        extra={
-            "TASK_ID": g.TASK_ID,
-            "context.teamId": g.TEAM_ID,
-            "context.workspaceId": g.WORKSPACE_ID,
-            "modal.state.slyProjectId": g.PROJECT_ID,
-        },
-    )
+                sly.logger.debug(f"Finished processing dataset {dataset.name}.")
+        sly.logger.info(f"Finished processing project {project.name}.")
 
-    g.my_app.run(initial_events=[{"command": "export_as_masks"}])
+    sly.output.set_download(project_dir)
+    sly.logger.debug("Application finished, output set.")
 
 
 if __name__ == "__main__":
-    sly.main_wrapper("main", main, log_for_agent=False)
+    export_as_masks(g.api)