Nested datasets and shift to new engine.

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 100
+ignore = E203, E501, W503, E722, W605

.gitignore

@@ -4,4 +4,6 @@ venv
 debug
 /src/debug
 secret_debug.env
-__pycache__
+__pycache__
+.DS_Store
+storage_dir/

.vscode/settings.json

@@ -20,6 +20,10 @@
     },
     "[python]": {
         "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.formatOnSave": true,
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": "explicit"
+        },
     },
     "debug.inlineValues": "off",
     "python.analysis.typeCheckingMode": "off",

config.json

@@ -1,9 +1,10 @@
 {
   "name": "Export to COCO",
   "type": "app",
+  "version": "2.0.0",
   "categories": ["images", "export"],
   "description": "Converts Supervisely to COCO format and prepares tar archive for download",
-  "docker_image": "supervisely/import-export:6.72.196",
+  "docker_image": "supervisely/import-export:6.73.48",
   "min_instance_version": "6.8.48",
   "main_script": "src/main.py",
   "modal_template": "src/modal.html",

create_venv.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+# learn more in documentation
+# Official python docs: https://docs.python.org/3/library/venv.html
+# Superviely developer portal: https://developer.supervise.ly/getting-started/installation#venv
+
+if [ -d ".venv" ]; then
+    echo "VENV already exists, will be removed"
+    rm -rf .venv
+fi
+
+echo "VENV will be created" && \
+python3 -m venv .venv && \
+source .venv/bin/activate && \
+
+echo "Install requirements..." && \
+pip3 install -r dev_requirements.txt && \
+echo "Requirements have been successfully installed" && \
+echo "Testing imports, please wait a minute ..." && \
+python3 -c "import supervisely as sly" && \
+echo "Success!" && \
+deactivate

dev_requirements.txt

@@ -1 +1 @@
-supervisely==6.72.147
+supervisely==6.73.48

local.env

@@ -1,22 +1,11 @@
-PYTHONUNBUFFERED=1
-
-DEBUG_APP_DIR="src/debug/app_debug_data"
-DEBUG_CACHE_DIR="src/debug/app_debug_cache"
-
-LOG_LEVEL="debug"
-
-TASK_ID=44849
-
-context.userId=7
-context.teamId=8
-context.workspaceId=349
-modal.state.slyProjectId=29064
+TEAM_ID = 448
+WORKSPACE_ID = 690
+PROJECT_ID = 35637
 
 # options: "images" "annotations"
 modal.state.selectedOutput="images"
 # options: "all" "annotated"
 modal.state.selectedFilter="all"
 modal.state.allDatasets=True
-# 54530 54531 54532 54533 54534 
 modal.state.datasets=[71754]
 modal.state.captions=False

src/convert_geometry.py

@@ -1,10 +1,7 @@
 import supervisely as sly
 from supervisely.annotation.json_geometries_map import GET_GEOMETRY_FROM_STR
-from supervisely.geometry import rectangle, polyline, bitmap, image_rotator
-from globals import rectangle_mark
+from supervisely.geometry import polyline, rectangle
 from supervisely.sly_logger import logger
-import numpy as np
-import cv2
 
 
 def prepare_meta(meta: sly.ProjectMeta):
@@ -24,11 +21,12 @@ def prepare_meta(meta: sly.ProjectMeta):
     return meta
 
 
-def convert_annotation(ann_info, img_info, src_meta, dst_meta):
+def convert_annotation(ann_info, img_info, src_meta, dst_meta, rectangle_mark):
     try:
         ann = sly.Annotation.from_json(ann_info.annotation, src_meta)
-    except:
-        return sly.Annotation(img_info.height, img_info.width)
+    except Exception as e:
+        sly.logger.debug(f"Exception while creating sly.Annotation from JSON: {e}")
+        return sly.Annotation((img_info.height, img_info.width))
     new_labels = []
     for lbl in ann.labels:
         try:
@@ -45,7 +43,7 @@ def convert_annotation(ann_info, img_info, src_meta, dst_meta):
                     converted_label.pop()
                     converted_label.append(new_label)
                 new_labels.extend(converted_label)
-        except NotImplementedError as e:
+        except NotImplementedError:
             logger.warning(
                 f"Unsupported conversion of annotation '{lbl.obj_class.geometry_type.name()}' type to '{new_cls.geometry_type.name()}'. Skipping annotation with [ID: {lbl.to_json()['id']}]",
                 exc_info=False,

src/functions.py

@@ -1,13 +1,11 @@
 import os
 import time
+from itertools import groupby
+
 import numpy as np
 import supervisely as sly
-
-from supervisely.io.fs import mkdir
-from itertools import groupby
 from supervisely.geometry import bitmap
-
-import globals as g
+from supervisely.io.fs import mkdir
 
 
 def get_categories_map_from_meta(meta):
@@ -86,6 +84,8 @@ def create_coco_annotation(
     caption_id,
     coco_captions,
     progress,
+    include_captions,
+    rectangle_mark,
 ):
     for image_info, ann in zip(image_infos, anns):
         image_coco_ann = dict(
@@ -98,11 +98,11 @@ def create_coco_annotation(
             id=image_info.id,
         )
         coco_ann["images"].append(image_coco_ann)
-        if coco_captions is not None and g.include_captions:
+        if coco_captions is not None and include_captions:
             coco_captions["images"].append(image_coco_ann)
 
         for label in ann.labels:
-            if g.rectangle_mark in label.description:
+            if rectangle_mark in label.description:
                 segmentation = []
             elif label.geometry.name() == bitmap.Bitmap.name():
                 segmentation = extend_mask_up_to_image(
@@ -122,7 +122,7 @@ def create_coco_annotation(
             coco_ann["annotations"].append(
                 dict(
                     segmentation=segmentation,  # a list of polygon vertices around the object, but can also be a run-length-encoded (RLE) bit mask
-                    area=label.geometry.area,  # Area is measured in pixels (e.g. a 10px by 20px box would have an area of 200)
+                    area=label.geometry.area,  # Area is measured in pixels (e. a 10px by 20px box would have an area of 200)
                     iscrowd=0,  # Is Crowd specifies whether the segmentation is for a single object or for a group/cluster of objects
                     image_id=image_info.id,  # The image id corresponds to a specific image in the dataset
                     bbox=bbox,  # he COCO bounding box format is [top left x position, top left y position, width, height]
@@ -132,7 +132,7 @@ def create_coco_annotation(
                     id=label_id,  # Each annotation also has an id (unique to all other annotations in the dataset)
                 )
             )
-        if coco_captions is not None and g.include_captions:
+        if coco_captions is not None and include_captions:
             for tag in ann.img_tags:
                 if (
                     tag.meta.name == "caption"
@@ -150,45 +150,45 @@ def create_coco_annotation(
     return coco_ann, label_id, coco_captions, caption_id
 
 
-def upload_coco_project(full_archive_name, result_archive, app_logger):
-    sly.fs.archive_directory(g.storage_dir, result_archive)
-
-    archive_size = sly.fs.get_file_size(result_archive) / (1024 * 1024)
-    archive_size = f"{archive_size:.2f} MB"
-    sly.logger.info(f"Total archive size: {archive_size}")
-
-    app_logger.info("Result directory is archived")
-
-    upload_progress = []
-    remote_archive_path = os.path.join(
-        sly.team_files.RECOMMENDED_EXPORT_PATH, f"export-to-COCO/{full_archive_name}"
-    )
-
-    def _print_progress(monitor, upload_progress):
-        if len(upload_progress) == 0:
-            upload_progress.append(
-                sly.Progress(
-                    message="Upload {!r}".format(full_archive_name),
-                    total_cnt=monitor.len,
-                    ext_logger=app_logger,
-                    is_size=True,
-                )
-            )
-        upload_progress[0].set_current_value(monitor.bytes_read)
-
-    file_info = g.api.file.upload(
-        g.team_id,
-        result_archive,
-        remote_archive_path,
-        lambda m: _print_progress(m, upload_progress),
-    )
-    app_logger.info("Uploaded to Team Files: {!r}".format(file_info.path))
-    g.api.task.set_output_archive(
-        g.task_id, file_info.id, full_archive_name, file_url=file_info.storage_path
-    )
-
-
-def create_coco_ann_templates(dataset, user_name, meta: sly.ProjectMeta):
+# def upload_coco_project(full_archive_name, result_archive, app_logger):
+#     sly.fs.archive_directory(storage_dir, result_archive)
+
+#     archive_size = sly.fs.get_file_size(result_archive) / (1024 * 1024)
+#     archive_size = f"{archive_size:.2f} MB"
+#     sly.logger.info(f"Total archive size: {archive_size}")
+
+#     app_logger.info("Result directory is archived")
+
+#     upload_progress = []
+#     remote_archive_path = os.path.join(
+#         sly.team_files.RECOMMENDED_EXPORT_PATH, f"export-to-COCO/{full_archive_name}"
+#     )
+
+#     def _print_progress(monitor, upload_progress):
+#         if len(upload_progress) == 0:
+#             upload_progress.append(
+#                 sly.Progress(
+#                     message="Upload {!r}".format(full_archive_name),
+#                     total_cnt=monitor.len,
+#                     ext_logger=app_logger,
+#                     is_size=True,
+#                 )
+#             )
+#         upload_progress[0].set_current_value(monitor.bytes_read)
+
+#     file_info = api.file.upload(
+#         team_id,
+#         result_archive,
+#         remote_archive_path,
+#         lambda m: _print_progress(m, upload_progress),
+#     )
+#     app_logger.info("Uploaded to Team Files: {!r}".format(file_info.path))
+#     api.task.set_output_archive(
+#         task_id, file_info.id, full_archive_name, file_url=file_info.storage_path
+#     )
+
+
+def create_coco_ann_templates(dataset, user_name, meta: sly.ProjectMeta, include_captions):
     coco_ann = dict(
         info=dict(
             description=dataset.description,
@@ -209,7 +209,7 @@ def create_coco_ann_templates(dataset, user_name, meta: sly.ProjectMeta):
         categories=get_categories_from_meta(meta),  # supercategory, id, name
     )
 
-    if not g.include_captions:
+    if not include_captions:
         return coco_ann, None
     captions_tag_meta = meta.get_tag_meta("caption")
     if captions_tag_meta is None or captions_tag_meta.value_type != sly.TagValueType.ANY_STRING:

src/globals.py

@@ -1,39 +1,39 @@
-import os
-import ast
-from distutils.util import strtobool
-import supervisely as sly
-from supervisely.io.fs import mkdir
-from supervisely.app.v1.app_service import AppService
-from dotenv import load_dotenv
+# import os
+# import ast
+# from distutils.util import strtobool
+# import supervisely as sly
+# from supervisely.io.fs import mkdir
+# from supervisely.app.v1.app_service import AppService
+# from dotenv import load_dotenv
 
-if sly.is_development():
-    load_dotenv("local.env")
-    load_dotenv(os.path.expanduser("~/supervisely.env"))
+# if sly.is_development():
+#     load_dotenv("local.env")
+#     load_dotenv(os.path.expanduser("~/supervisely.env"))
 
-my_app = AppService()
-api: sly.Api = my_app.public_api
+# my_app = AppService()
+# api: sly.Api = my_app.public_api
 
-selected_output = os.environ["modal.state.selectedOutput"]
-selected_filter = os.environ["modal.state.selectedFilter"]
-all_datasets = bool(strtobool(os.getenv("modal.state.allDatasets")))
-selected_datasets = ast.literal_eval(os.environ["modal.state.datasets"])
-include_captions = bool(strtobool(os.getenv("modal.state.captions")))
+# selected_output = os.environ["modal.state.selectedOutput"]
+# selected_filter = os.environ["modal.state.selectedFilter"]
+# all_datasets = bool(strtobool(os.getenv("modal.state.allDatasets")))
+# selected_datasets = ast.literal_eval(os.environ["modal.state.datasets"])
+# include_captions = bool(strtobool(os.getenv("modal.state.captions")))
 
-task_id = os.environ["TASK_ID"]
-user_id = os.environ["context.userId"]
-team_id = int(os.environ["context.teamId"])
-workspace_id = int(os.environ["context.workspaceId"])
-project_id = int(os.environ["modal.state.slyProjectId"])
+# task_id = os.environ["TASK_ID"]
+# user_id = os.environ["context.userId"]
+# team_id = int(os.environ["context.teamId"])
+# workspace_id = int(os.environ["context.workspaceId"])
+# project_id = int(os.environ["modal.state.slyProjectId"])
 
 # user = api.user.get_info_by_id(user_id)
-user_name = "Supervisely"
-rectangle_mark = "converted_from_rectangle"
-project = api.project.get_info_by_id(project_id)
-meta_json = api.project.get_meta(project_id)
-meta = sly.ProjectMeta.from_json(meta_json)
+# user_name = "Supervisely"
+# rectangle_mark = "converted_from_rectangle"
+# project = api.project.get_info_by_id(project_id)
+# meta_json = api.project.get_meta(project_id)
+# meta = sly.ProjectMeta.from_json(meta_json)
 
-storage_dir = os.path.join(my_app.data_dir, "storage_dir")
-mkdir(storage_dir, True)
+# storage_dir = os.path.join(my_app.data_dir, "storage_dir")
+# mkdir(storage_dir, True)
 
-coco_base_dir = os.path.join(storage_dir, project.name)
-mkdir(coco_base_dir)
+# coco_base_dir = os.path.join(storage_dir, project.name)
+# mkdir(coco_base_dir)