Prepared for nested datasets.

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 100
+ignore = E203, E501, W503, E722, W605

.gitignore

@@ -1,4 +1,7 @@
 .idea
 .venv
 src/debug
-secret_debug.env
+secret_debug.env
+data/
+.DS_Store
+__pycache__/

.vscode/settings.json

@@ -1,8 +1,7 @@
 {
     "files.exclude": {
         "**/__pycache__": true,
-        "build": true,
-        "supervisely.egg-info": true,
+        "venv": true,
         ".venv": true
     },
     "python.defaultInterpreterPath": ".venv/bin/python",
@@ -21,15 +20,12 @@
     },
     "[python]": {
         "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.formatOnSave": true,
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": "explicit"
+        },
     },
     "debug.inlineValues": "off",
     "python.analysis.typeCheckingMode": "off",
-    "python.analysis.autoImportCompletions": false,
-    "autoDocstring.docstringFormat": "sphinx",
-    "autoDocstring.customTemplatePath": "docs/.mustache",
-    "python.testing.pytestArgs": [
-        "tests/inference_cache"
-    ],
-    "python.testing.unittestEnabled": false,
-    "python.testing.pytestEnabled": true
+    "python.analysis.autoImportCompletions": false
 }

config.json

@@ -6,7 +6,7 @@
         "export"
     ],
     "description": "Converts Supervisely Project to Pascal VOC format",
-    "docker_image": "supervisely/import-export:6.73.22",
+    "docker_image": "supervisely/import-export:6.73.48",
     "instance_version": "6.8.73",
     "main_script": "src/main.py",
     "modal_template": "src/modal.html",

create_venv.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+# learn more in documentation
+# Official python docs: https://docs.python.org/3/library/venv.html
+# Superviely developer portal: https://developer.supervise.ly/getting-started/installation#venv
+
+if [ -d ".venv" ]; then
+    echo "VENV already exists, will be removed"
+    rm -rf .venv
+fi
+
+echo "VENV will be created" && \
+python3 -m venv .venv && \
+source .venv/bin/activate && \
+
+echo "Install dev_requirements..." && \
+pip3 install -r dev_requirements.txt && \
+echo "Requirements have been successfully installed" && \
+deactivate

dev_requirements.txt

@@ -1,4 +1,4 @@
-supervisely==6.73.22
-lxml==4.6.3
+supervisely==6.73.48
+lxml
 numpy>=1.19.4
 Pillow>=8.0.1

local.env

@@ -1,15 +1,6 @@
-PYTHONUNBUFFERED=1
-
-LOG_LEVEL="debug"
-
-TASK_ID=44728
-
-context.teamId=8
-context.workspaceId=349
-modal.state.slyProjectId=22451
+TEAM_ID=448
+WORKSPACE_ID=690
+PROJECT_ID=35637
 
 modal.state.pascalContourThickness=3
-modal.state.trainSplitCoef=0.8
-
-DEBUG_APP_DIR="src/debug/app_debug_data"
-DEBUG_CACHE_DIR="src/debug/app_debug_cache"
+modal.state.trainSplitCoef=0.8

src/globals.py

@@ -1,25 +1,29 @@
 import os
-from dotenv import load_dotenv
+
 import supervisely as sly
-from supervisely.app.v1.app_service import AppService
+from dotenv import load_dotenv
 
 if sly.is_development():
     load_dotenv("local.env")
     load_dotenv(os.path.expanduser("~/supervisely.env"))
 
+# region constants
+ARCHIVE_NAME_ENDING = "_pascal_voc.tar.gz"
+RESULT_DIR_NAME_ENDING = "_pascal_voc"
+RESULT_SUBDIR_NAME = "VOCdevkit/VOC"
+DATA_DIR = os.path.join(os.getcwd(), "data")
+# endregion
+sly.fs.mkdir(DATA_DIR, remove_content_if_exists=True)
 
-my_app = AppService()
-
-TEAM_ID = int(os.environ["context.teamId"])
-WORKSPACE_ID = int(os.environ["context.workspaceId"])
-PROJECT_ID = int(os.environ["modal.state.slyProjectId"])
+# region envvars
+team_id = sly.env.team_id()
+workspace_id = sly.env.workspace_id()
+project_id = sly.env.project_id()
 
 PASCAL_CONTOUR_THICKNESS = int(os.environ["modal.state.pascalContourThickness"])
 TRAIN_VAL_SPLIT_COEF = float(os.environ["modal.state.trainSplitCoef"])
+# endregion
 
-ARCHIVE_NAME_ENDING = "_pascal_voc.tar.gz"
-RESULT_DIR_NAME_ENDING = "_pascal_voc"
-RESULT_SUBDIR_NAME = "VOCdevkit/VOC"
 
 images_dir_name = "JPEGImages"
 ann_dir_name = "Annotations"

src/main.py

@@ -1,23 +1,21 @@
 import os
-import supervisely as sly
 from collections import OrderedDict
+
+import supervisely as sly
+
 import globals as g
 import utils
 
 
-@g.my_app.callback("from_sly_to_pascal")
-@sly.timeit
-def from_sly_to_pascal(api: sly.Api, task_id, context, state, app_logger):
-    project_info = api.project.get_info_by_id(g.PROJECT_ID)
-    meta_json = api.project.get_meta(g.PROJECT_ID)
+def from_sly_to_pascal(api: sly.Api):
+    project_info = api.project.get_info_by_id(g.project_id)
+    meta_json = api.project.get_meta(g.project_id)
     meta = sly.ProjectMeta.from_json(meta_json)
-    app_logger.info("Palette has been created")
+    sly.logger.info("Palette has been created")
 
-    full_archive_name = f"{str(project_info.id)}_{project_info.name}{g.ARCHIVE_NAME_ENDING}"
     full_result_dir_name = f"{str(project_info.id)}_{project_info.name}{g.RESULT_DIR_NAME_ENDING}"
 
-    result_archive = os.path.join(g.my_app.data_dir, full_archive_name)
-    result_dir = os.path.join(g.my_app.data_dir, full_result_dir_name)
+    result_dir = os.path.join(g.DATA_DIR, full_result_dir_name)
     result_subdir = os.path.join(result_dir, g.RESULT_SUBDIR_NAME)
 
     result_ann_dir = os.path.join(result_subdir, g.ann_dir_name)
@@ -32,15 +30,15 @@ def from_sly_to_pascal(api: sly.Api, task_id, context, state, app_logger):
     sly.fs.mkdir(result_class_dir_name)
     sly.fs.mkdir(result_obj_dir)
 
-    app_logger.info("Pascal VOC directories have been created")
+    sly.logger.info("Pascal VOC directories have been created")
 
     images_stats = []
     classes_colors = {}
 
-    datasets = api.dataset.get_list(g.PROJECT_ID)
+    datasets = api.dataset.get_list(g.project_id, recursive=True)
     dataset_names = ["trainval", "val", "train"]
     progress = sly.Progress(
-        "Preparing images for export", api.project.get_images_count(g.PROJECT_ID), app_logger
+        "Preparing images for export", api.project.get_images_count(g.project_id), sly.logger
     )
     for dataset in datasets:
         if dataset.name in dataset_names:
@@ -86,7 +84,7 @@ def from_sly_to_pascal(api: sly.Api, task_id, context, state, app_logger):
                     if type(label.geometry) in g.SUPPORTED_GEOMETRY_TYPES:
                         valid_labels.append(label)
                     else:
-                        app_logger.warn(
+                        sly.logger.warn(
                             f"Label has unsupported geometry type ({type(label.geometry)}) and will be skipped."
                         )
 
@@ -136,49 +134,9 @@ def from_sly_to_pascal(api: sly.Api, task_id, context, state, app_logger):
     utils.write_segm_set(is_trainval, images_stats, result_imgsets_dir)
     utils.write_main_set(is_trainval, images_stats, meta, result_imgsets_dir)
 
-    sly.fs.archive_directory(result_dir, result_archive)
-    app_logger.info("Result directory is archived")
-
-    upload_progress = []
-    remote_archive_path = os.path.join(
-        sly.team_files.RECOMMENDED_EXPORT_PATH,
-        "export-to-Pascal-VOC/{}/{}".format(task_id, full_archive_name),
-    )
-
-    def _print_progress(monitor, upload_progress):
-        if len(upload_progress) == 0:
-            upload_progress.append(
-                sly.Progress(
-                    message="Upload {!r}".format(full_archive_name),
-                    total_cnt=monitor.len,
-                    ext_logger=app_logger,
-                    is_size=True,
-                )
-            )
-        upload_progress[0].set_current_value(monitor.bytes_read)
-
-    file_info = api.file.upload(
-        g.TEAM_ID,
-        result_archive,
-        remote_archive_path,
-        lambda m: _print_progress(m, upload_progress),
-    )
-    app_logger.info("Uploaded to Team-Files: {!r}".format(file_info.storage_path))
-    api.task.set_output_archive(
-        task_id, file_info.id, full_archive_name, file_url=file_info.storage_path
-    )
-
-    g.my_app.stop()
-
-
-def main():
-    sly.logger.info(
-        "Script arguments",
-        extra={"TEAM_ID": g.TEAM_ID, "WORKSPACE_ID": g.WORKSPACE_ID, "PROJECT_ID": g.PROJECT_ID},
-    )
-
-    g.my_app.run(initial_events=[{"command": "from_sly_to_pascal"}])
+    sly.output.set_download(result_dir)
 
 
 if __name__ == "__main__":
-    sly.main_wrapper("main", main, log_for_agent=False)
+    api = sly.Api.from_env()
+    from_sly_to_pascal(api)