Implement run app from preset, update SDK branch, update format settings

.vscode/settings.json

@@ -17,14 +17,27 @@
 		"editor.defaultFormatter": "esbenp.prettier-vscode"
 	},
 	"[python]": {
-		"editor.defaultFormatter": "ms-python.black-formatter"
+		"editor.defaultFormatter": "ms-python.black-formatter",
+		"editor.formatOnSave": true,
+		"editor.codeActionsOnSave": {
+			"source.organizeImports": "explicit"
+		}
 	},
+	"isort.args": ["--profile", "black"],
 	"debug.inlineValues": "off",
 	"python.analysis.typeCheckingMode": "off",
 	"python.analysis.autoImportCompletions": false,
 	"autoDocstring.docstringFormat": "sphinx",
 	"autoDocstring.customTemplatePath": "docs/.mustache",
 	"python.testing.pytestArgs": ["tests/inference_cache"],
 	"python.testing.unittestEnabled": false,
-	"python.testing.pytestEnabled": true
+	"python.testing.pytestEnabled": true,
+	"workbench.colorCustomizations": {
+		"minimap.errorHighlight": "#ff0000",
+		"editorOverviewRuler.errorForeground": "#ff0000",
+		"editorOverviewRuler.warningForeground": "#ff9900",
+		"minimap.warningHighlight": "#ff9900",
+		"minimap.infoHighlight": "#00a8aa",
+		"editorOverviewRuler.infoForeground": "#00a8aa"
+	}
 }

config.json

@@ -19,7 +19,8 @@
 			"images_project",
 			"images_dataset",
 			"videos_project",
-			"videos_dataset"
+			"videos_dataset",
+			"files_file"
 		]
 	},
 	"instance_version": "6.11.8"

dev_requirements.txt

@@ -1,6 +1,6 @@
 git+https://github.com/supervisely/supervisely.git@optimize-index
 
-# supervisely==6.73.161
+# supervisely==6.73.164
 jsonschema==4.19.2
 networkx==3.1
 scikit-image==0.21.0

requirements.txt

@@ -0,0 +1,12 @@
+git+https://github.com/supervisely/supervisely.git@optimize-index-2
+
+# supervisely==6.73.164
+# jsonschema==4.19.2
+# networkx==3.1
+# scikit-image==0.21.0
+# cacheout==0.14.1
+# markdown==3.5.1
+# json2html==1.3.0
+# moviepy==1.0.3
+# imgaug==0.4.0
+# imagecorruptions==1.1.2

src/compute/layers/save/CopyAnnotationsLayer.py

@@ -248,7 +248,7 @@ def preprocess(self):
         is_single_input_ds = False
         input_projects_map = _get_source_projects_ids_from_dtl()
         if len(input_projects_map) == 0:
-            raise ValueError("No source projects found in the DTL")
+            raise ValueError("No source projects found in the Pipeline")
         if len(input_projects_map) == 1:
             input_datasets_count = len(input_projects_map[list(input_projects_map.keys())[0]])
             if input_datasets_count == 1:

src/compute/main.py

@@ -73,7 +73,7 @@ def main(
     if not g.pipeline_running:
         return
 
-    logger.info("DTL started")
+    logger.info("Pipeline started")
     helper = DtlHelper()
 
     try:
@@ -187,7 +187,7 @@ def main(
         return
 
     logger.info(
-        "DTL finished",
+        "Pipeline finished",
         extra={"event_type": EventType.DTL_APPLIED, "new_proj_size": results_counter},
     )
     total_pipeline_time_end = time()
@@ -200,4 +200,4 @@ def main(
 if __name__ == "__main__":
     if os.getenv("DEBUG_LOG_TO_FILE", None):
         sly_logger.add_default_logging_into_file(logger, DtlPaths().debug_dir)
-    logging_utils.main_wrapper("DTL", main)
+    logging_utils.main_wrapper("Pipeline", main)

src/compute/tasks/progress_counter.py

@@ -12,8 +12,9 @@ def epoch_float(epoch, train_it, train_its):
 
 
 class ProgressCounter:
-    def __init__(self, subtask_name, total_cnt,
-                 report_limit=100, ext_logger=None, report_divisor=1):
+    def __init__(
+        self, subtask_name, total_cnt, report_limit=100, ext_logger=None, report_divisor=1
+    ):
         self.subtask = subtask_name
         self.total = total_cnt
         self.current = 0
@@ -39,18 +40,23 @@ def iters_done(self, count):
         self.current += count
 
     def report_progress(self):
-        self.logger.info('progress', extra={
-            'event_type': EventType.PROGRESS,
-            'subtask': self.subtask,
-            'current': math.ceil(self.current / self.report_divisor),
-            'total': math.ceil(self.total / self.report_divisor),
-        })
+        self.logger.info(
+            "progress",
+            extra={
+                "event_type": EventType.PROGRESS,
+                "subtask": self.subtask,
+                "current": math.ceil(self.current / self.report_divisor),
+                "total": math.ceil(self.total / self.report_divisor),
+            },
+        )
         self.reported_cnt += 1
 
     def report_if_needed(self):
-        if (self.current == self.total) \
-                or (self.current % self.report_every == 0) \
-                or ((self.reported_cnt - 1) < (self.current // self.report_every)):
+        if (
+            (self.current == self.total)
+            or (self.current % self.report_every == 0)
+            or ((self.reported_cnt - 1) < (self.current // self.report_every))
+        ):
             self.report_progress()
 
     def iter_done_report(self):  # finish & report
@@ -64,83 +70,92 @@ def iters_done_report(self, count):  # finish & report
 
 def progress_counter_train(total_epochs, cnt_iters_per_epoch, ext_logger=None):
     total_cnt = total_epochs * cnt_iters_per_epoch
-    ctr = ProgressCounter('model training', total_cnt, ext_logger=ext_logger)
+    ctr = ProgressCounter("model training", total_cnt, ext_logger=ext_logger)
     return ctr
 
 
 def progress_counter_dtl(pr_name, cnt_images, ext_logger=None):
     total_cnt = cnt_images
-    ctr = ProgressCounter('DTL: {}'.format(pr_name), total_cnt, ext_logger=ext_logger)
+    ctr = ProgressCounter("Pipeline: {}".format(pr_name), total_cnt, ext_logger=ext_logger)
+    # ctr = ProgressCounter('DTL: {}'.format(pr_name), total_cnt, ext_logger=ext_logger)
     return ctr
 
 
 def progress_counter_inference(cnt_imgs, ext_logger=None):
-    ctr = ProgressCounter('model applying', cnt_imgs, ext_logger=ext_logger)
+    ctr = ProgressCounter("model applying", cnt_imgs, ext_logger=ext_logger)
     return ctr
 
 
 def progress_download_project(total_cnt, pr_name, ext_logger=None):
-    subtask_name = 'Downloading: {}'.format(pr_name)
+    subtask_name = "Downloading: {}".format(pr_name)
     ctr = ProgressCounter(subtask_name=subtask_name, total_cnt=total_cnt, ext_logger=ext_logger)
     return ctr
 
 
 def progress_upload_project(total_cnt, pr_name, ext_logger=None):
-    subtask_name = 'Uploading: {}'.format(pr_name)
+    subtask_name = "Uploading: {}".format(pr_name)
     ctr = ProgressCounter(subtask_name=subtask_name, total_cnt=total_cnt, ext_logger=ext_logger)
     return ctr
 
 
 def progress_download_nn(total_cnt, ext_logger=None):
-    subtask_name = 'Download NN, MB:'
-    ctr = ProgressCounter(subtask_name=subtask_name, total_cnt=total_cnt, ext_logger=ext_logger,
-                          report_divisor=2**20)  # by MBs
+    subtask_name = "Download NN, MB:"
+    ctr = ProgressCounter(
+        subtask_name=subtask_name, total_cnt=total_cnt, ext_logger=ext_logger, report_divisor=2**20
+    )  # by MBs
     return ctr
 
 
 def progress_counter_import(pr_name, cnt_images, ext_logger=None):
     total_cnt = cnt_images
-    ctr = ProgressCounter('Import: {}'.format(pr_name), total_cnt, ext_logger=ext_logger)
+    ctr = ProgressCounter("Import: {}".format(pr_name), total_cnt, ext_logger=ext_logger)
     return ctr
 
 
 def progress_counter_build_model(cnt_models=1, ext_logger=None):
     total_cnt = cnt_models
-    ctr = ProgressCounter('Building model:', total_cnt, ext_logger=ext_logger)
+    ctr = ProgressCounter("Building model:", total_cnt, ext_logger=ext_logger)
     return ctr
 
 
 # metrics as dct {string_name: float_value}
 def _report_metrics(m_type, epoch, metrics):
-    logger.info('metrics', extra={
-        'event_type': EventType.METRICS,
-        'type': m_type,
-        'epoch': epoch,
-        'metrics': metrics
-    })
+    logger.info(
+        "metrics",
+        extra={"event_type": EventType.METRICS, "type": m_type, "epoch": epoch, "metrics": metrics},
+    )
 
 
 def report_metrics_training(epoch, metrics):
-    _report_metrics('train', epoch, metrics)
+    _report_metrics("train", epoch, metrics)
 
 
 def report_metrics_validation(epoch, metrics):
-    _report_metrics('val', epoch, metrics)
+    _report_metrics("val", epoch, metrics)
 
 
 def report_inference_finished():
-    logger.info('model applied', extra={
-        'event_type': EventType.MODEL_APPLIED,
-    })
+    logger.info(
+        "model applied",
+        extra={
+            "event_type": EventType.MODEL_APPLIED,
+        },
+    )
 
 
 def report_import_finished():
-    logger.info('import finished', extra={
-        'event_type': EventType.IMPORT_APPLIED,
-    })
+    logger.info(
+        "import finished",
+        extra={
+            "event_type": EventType.IMPORT_APPLIED,
+        },
+    )
 
 
 def report_agent_rpc_ready():
-    logger.info('Ready to get events', extra={
-        'event_type': EventType.TASK_DEPLOYED,
-    })
+    logger.info(
+        "Ready to get events",
+        extra={
+            "event_type": EventType.TASK_DEPLOYED,
+        },
+    )

src/globals.py

@@ -38,7 +38,7 @@
 TEAM_FILES_PATH = "data-nodes"
 PROJECT_ID = sly.env.project_id(raise_not_found=False)
 DATASET_ID = sly.env.dataset_id(raise_not_found=False)
-# FILE = sly.env.team_files_file(raise_not_found=False)
+FILE = sly.env.team_files_file(raise_not_found=False)
 SUPPORTED_MODALITIES = ["images", "videos"]
 
 SUPPORTED_MODALITIES_MAP = {

src/main.py

@@ -121,7 +121,9 @@ def generate_preview_for_project(layer: Layer):
 
 
 layer = None
-if g.PIPELINE_TEMPLATE is not None:
+if g.FILE:
+    g.updater("load_json")
+elif g.PIPELINE_TEMPLATE is not None:
     template = templates[g.MODALITY_TYPE].get(g.PIPELINE_TEMPLATE, None)
     if template is not None:
         load_template(template)
@@ -164,9 +166,6 @@ def generate_preview_for_project(layer: Layer):
 
 update_loop.start()
 
-# if g.FILE:
-#     g.updater("load_json")
-
 app.call_before_shutdown(u.on_app_shutdown)
 if layer is not None:
     if g.MODALITY_TYPE == "images":