Skip to content

Commit

Permalink
Merge pull request #17 from supervisely-ecosystem/download-async
Browse files Browse the repository at this point in the history
Improved Images Downloading
  • Loading branch information
GoldenAnpu authored Dec 17, 2024
2 parents 29c900d + 8fa2a16 commit 1182476
Show file tree
Hide file tree
Showing 8 changed files with 92 additions and 116 deletions.
2 changes: 1 addition & 1 deletion .vscode/launch.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
"configurations": [
{
"name": "Python: Current File",
"type": "python",
"type": "debugpy",
"request": "launch",
"program": "${file}",
"console": "integratedTerminal",
Expand Down
3 changes: 2 additions & 1 deletion .vscode/settings.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@
"editor.defaultFormatter": "esbenp.prettier-vscode"
},
"[json]": {
"editor.defaultFormatter": "esbenp.prettier-vscode"
"editor.defaultFormatter": "esbenp.prettier-vscode",
"editor.tabSize": 4
},
"[python]": {
"editor.defaultFormatter": "ms-python.black-formatter"
Expand Down
53 changes: 24 additions & 29 deletions config.json
Original file line number Diff line number Diff line change
@@ -1,30 +1,25 @@
{
"name": "Convert Supervisely to YOLO v5 format",
"type": "app",
"version": "2.0.0",
"categories": [
"images",
"export"
],
"description": "Transform project to YOLO v5 format and prepares tar archive for download",
"docker_image": "supervisely/import-export:6.73.162",
"instance_version": "6.11.8",
"main_script": "src/convert_sly_to_yolov5.py",
"modal_template": "src/modal.html",
"modal_template_state": {
"processShapes": "skip"
},
"gui_template": "src/gui.html",
"task_location": "workspace_tasks",
"isolate": true,
"headless": true,
"icon": "https://i.imgur.com/pz3eSzx.png",
"icon_background": "#FFFFFF",
"context_menu": {
"target": [
"images_project"
],
"context_root": "Download as"
},
"poster": "https://user-images.githubusercontent.com/106374579/183683758-89476d80-de3f-424f-9bfa-f1562703a168.png"
}
"name": "Convert Supervisely to YOLO v5 format",
"type": "app",
"version": "2.0.0",
"categories": ["images", "export"],
"description": "Transform project to YOLO v5 format and prepares tar archive for download",
"docker_image": "supervisely/import-export:6.73.256",
"instance_version": "6.12.12",
"main_script": "src/convert_sly_to_yolov5.py",
"modal_template": "src/modal.html",
"modal_template_state": {
"processShapes": "skip"
},
"gui_template": "src/gui.html",
"task_location": "workspace_tasks",
"isolate": true,
"headless": true,
"icon": "https://i.imgur.com/pz3eSzx.png",
"icon_background": "#FFFFFF",
"context_menu": {
"target": ["images_project"],
"context_root": "Download as"
},
"poster": "https://user-images.githubusercontent.com/106374579/183683758-89476d80-de3f-424f-9bfa-f1562703a168.png"
}
2 changes: 1 addition & 1 deletion dev_requirements.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
supervisely==6.73.162
supervisely==6.73.256
10 changes: 7 additions & 3 deletions local.env
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
TEAM_ID = 448
WORKSPACE_ID = 690
PROJECT_ID = 35637
# TEAM_ID = 448
# WORKSPACE_ID = 690
# PROJECT_ID = 35637

TEAM_ID = 431
WORKSPACE_ID = 1019
PROJECT_ID = 40721
39 changes: 28 additions & 11 deletions src/convert_sly_to_yolov5.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
from typing import List, Tuple
from dotenv import load_dotenv
import supervisely as sly
from workflow import Workflow
from workflow import add_input, add_output
import asyncio

# region constants
TRAIN_TAG_NAME = "train"
Expand Down Expand Up @@ -86,7 +87,7 @@ def transform(api: sly.Api) -> None:
missing_tags.append(VAL_TAG_NAME)
if len(missing_tags) > 0:
missing_tags_str = ", ".join([f'"{tag}"' for tag in missing_tags])
sly.logger.warn(
sly.logger.warning(
f"Tag(s): {missing_tags_str} not found in project meta. Images without special tags will be marked as train"
)

Expand All @@ -95,7 +96,7 @@ def transform(api: sly.Api) -> None:
if obj_class.geometry_type != sly.Rectangle:
error_classes.append(obj_class)
if len(error_classes) > 0:
sly.logger.warn(
sly.logger.warning(
f"Project has unsupported classes. "
f"Objects with unsupported geometry types will be {process_shapes_message}: "
f"{[obj_class.name for obj_class in error_classes]}"
Expand All @@ -115,7 +116,7 @@ def _add_to_split(image_id, img_name, split_ids, split_image_paths, labels_dir,
train_count = 0
val_count = 0

progress = sly.Progress("Transformation ...", api.project.get_images_count(project_id))
progress = sly.Progress("Processing project items...", api.project.get_images_count(project_id))
for dataset in api.dataset.get_list(project_id, recursive=True):
sly.logger.info(f"Working with dataset: {dataset.name}...")
images = api.image.get_list(dataset.id)
Expand All @@ -127,6 +128,7 @@ def _add_to_split(image_id, img_name, split_ids, split_image_paths, labels_dir,
val_ids = []
val_image_paths = []

a_progress = sly.Progress("Transforming annotations...", total_cnt=len(images))
for batch in sly.batched(images):
image_ids = [image_info.id for image_info in batch]
image_names = [f"{dataset.id}_{dataset.name}_{image_info.name}" for image_info in batch]
Expand Down Expand Up @@ -177,13 +179,25 @@ def _add_to_split(image_id, img_name, split_ids, split_image_paths, labels_dir,
train_images_dir,
)
train_count += 1
a_progress.iters_done_report(len(batch))

api.image.download_paths(dataset.id, train_ids, train_image_paths)
api.image.download_paths(dataset.id, val_ids, val_image_paths)
ids_to_download = train_ids + val_ids
paths_to_download = train_image_paths + val_image_paths
i_rogress = sly.Progress("Downloading images...", total_cnt=len(ids_to_download))
coro = api.image.download_paths_async(
ids_to_download, paths_to_download, progress_cb=i_rogress.iters_done_report
)
loop = sly.utils.get_or_create_event_loop()
if loop.is_running():
future = asyncio.run_coroutine_threadsafe(coro, loop)
future.result()
else:
loop.run_until_complete(coro)

progress.iters_done_report(len(ids_to_download))

progress.iters_done_report(len(batch))
if unsupported_shapes > 0:
sly.logger.warn(
sly.logger.warning(
f"Dataset {dataset.name}: "
f"{unsupported_shapes} objects with unsupported geometry types have been {process_shapes_message}"
)
Expand All @@ -205,12 +219,15 @@ def _add_to_split(image_id, img_name, split_ids, split_image_paths, labels_dir,
file_info = sly.output.set_download(result_dir)
sly.logger.info("File uploaded, app stopped.")
# --------------------------------- Add Workflow Input And Output -------------------------------- #
workflow.add_input(project_id)
workflow.add_output(file_info)
add_input(api, project_id)
add_output(api, file_info)
# --------------------------------- Add Workflow Input And Output -------------------------------- #


if __name__ == "__main__":
api = sly.Api.from_env()
workflow = Workflow(api)
if api.server_address == "https://app.supervisely.com":
semaphore = api.get_default_semaphore()
if semaphore._value == 10:
api.set_semaphore_size(7)
transform(api)
12 changes: 5 additions & 7 deletions src/modal.html
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,13 @@
title="Process shapes"
description="Select how to process shapes that are not supported by the current application."
>
<div>
<div style="margin-top: 5px;">
<el-radio-group v-model="state.processShapes" size="normal">
<div>
<el-radio label="transform">
<span>transform to Rectangle (bbox)</span></el-radio
>
<div >
<el-radio label="skip"><span>Skip objects of unsupported geometries</span></el-radio>
</div>
<div>
<el-radio label="skip"><span>skip</span></el-radio>
<div style="margin-top: 5px;">
<el-radio label="transform"><span>Transform shapes to Rectangle (bbox)</span></el-radio>
</div>
</el-radio-group>
</div>
Expand Down
87 changes: 24 additions & 63 deletions src/workflow.py
Original file line number Diff line number Diff line change
@@ -1,67 +1,28 @@
import supervisely as sly


def check_compatibility(func):
def wrapper(self, *args, **kwargs):
if self.is_compatible is None:
try:
self.is_compatible = self.check_instance_ver_compatibility()
except Exception as e:
sly.logger.error(
"Can not check compatibility with Supervisely instance. "
f"Workflow features will be disabled. Error: {repr(e)}"
)
self.is_compatible = False
if not self.is_compatible:
return
return func(self, *args, **kwargs)

return wrapper


class Workflow:
def __init__(self, api: sly.Api, min_instance_version: str = None):
self.is_compatible = None
self.api = api
self._min_instance_version = (
"6.9.31" if min_instance_version is None else min_instance_version
)

def check_instance_ver_compatibility(self):
if not self.api.is_version_supported(self._min_instance_version):
sly.logger.info(
f"Supervisely instance version {self.api.instance_version} does not support workflow features."
)
if not sly.is_community():
sly.logger.info(
f"To use them, please update your instance to version {self._min_instance_version} or higher."
)
return False
return True

@check_compatibility
def add_input(self, project_id: int):
self.api.app.workflow.add_input_project(project_id)
sly.logger.debug(f"Workflow: Input project - {project_id}")

@check_compatibility
def add_output(self, file_info: sly.api.file_api.FileInfo):
try:
meta = {
"customRelationSettings": {
"icon": {
"icon": "zmdi-archive",
"color": "#33c94c",
"backgroundColor": "#d9f7e4",
},
"title": f"<h4>{file_info.name}</h4>",
"mainLink": {
"url": f"/files/{file_info.id}/true/?teamId={file_info.team_id}",
"title": "Download",
},
}
def add_input(api: sly.Api, project_id: int):
api.app.workflow.add_input_project(project_id)
sly.logger.debug(f"Workflow: Input project - {project_id}")


def add_output(api: sly.Api, file_info: sly.api.file_api.FileInfo):
try:
meta = {
"customRelationSettings": {
"icon": {
"icon": "zmdi-archive",
"color": "#33c94c",
"backgroundColor": "#d9f7e4",
},
"title": f"<h4>{file_info.name}</h4>",
"mainLink": {
"url": f"/files/{file_info.id}/true/?teamId={file_info.team_id}",
"title": "Download",
},
}
self.api.app.workflow.add_output_file(file_info, meta=meta)
sly.logger.debug(f"Workflow: Output file - {file_info.id if file_info else None}")
except Exception as e:
sly.logger.debug(f"Workflow: Can not add output file. Error: {repr(e)}")
}
api.app.workflow.add_output_file(file_info, meta=meta)
sly.logger.debug(f"Workflow: Output file - {file_info.id if file_info else None}")
except Exception as e:
sly.logger.debug(f"Workflow: Can not add output file. Error: {repr(e)}")

0 comments on commit 1182476

Please sign in to comment.