From cd409e1cc6279a77ed1aa2c3bee67c00ea63ca46 Mon Sep 17 00:00:00 2001 From: peixed351 Date: Sat, 21 Jan 2023 13:04:19 +0300 Subject: [PATCH 1/3] add export template --- .gitignore | 7 +++- config.json | 11 ++--- create_venv.sh | 19 +++++++++ debug.env | 20 --------- local.env | 12 ++++++ requirements.txt | 2 +- src/main.py | 103 ++++++++++++++++------------------------------- 7 files changed, 76 insertions(+), 98 deletions(-) create mode 100755 create_venv.sh delete mode 100644 debug.env create mode 100644 local.env diff --git a/.gitignore b/.gitignore index 5c94b7c..44dfca6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ secret_debug.env +src/debug +debug .idea -venv \ No newline at end of file +.venv +supervisely +__pycache__ +.vscode \ No newline at end of file diff --git a/config.json b/config.json index bb5e2e9..87c856b 100644 --- a/config.json +++ b/config.json @@ -1,12 +1,9 @@ { "name": "Export to Supervisely format", "type": "app", - "categories": [ - "images", - "export" - ], + "categories": ["images", "export"], "description": "images and JSON annotations", - "docker_image": "supervisely/base-py-sdk:6.68.109", + "docker_image": "supervisely/base-py-sdk:6.69.8", "instance_version": "6.5.1", "main_script": "src/main.py", "modal_template": "src/modal.html", @@ -20,9 +17,7 @@ "icon": "https://i.imgur.com/1hqGMyg.png", "icon_background": "#FFFFFF", "context_menu": { - "target": [ - "images_project" - ], + "target": ["images_project"], "context_root": "Download as" }, "poster": "https://user-images.githubusercontent.com/106374579/186665737-ec3da9cc-193f-43ee-85db-a6f802b2dfe4.png" diff --git a/create_venv.sh b/create_venv.sh new file mode 100755 index 0000000..6df47f9 --- /dev/null +++ b/create_venv.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +# learn more in documentation +# Official python docs: https://docs.python.org/3/library/venv.html +# Superviely developer portal: https://developer.supervise.ly/getting-started/installation#venv + +if [ -d ".venv" ]; then + echo "VENV already exists, will be removed" + rm -rf .venv +fi + +echo "VENV will be created" && \ +python3 -m venv .venv && \ +source .venv/bin/activate && \ + +echo "Install requirements..." && \ +pip3 install -r requirements.txt && \ +echo "Requirements have been successfully installed" && \ +deactivate \ No newline at end of file diff --git a/debug.env b/debug.env deleted file mode 100644 index f5198b4..0000000 --- a/debug.env +++ /dev/null @@ -1,20 +0,0 @@ -PYTHONUNBUFFERED=1 -TASK_ID=14507 - -context.teamId=8 -context.workspaceId=349 -modal.state.slyProjectId=9771 -modal.state.download=all -modal.state.fixExtension=true -DEBUG_APP_DIR="/home/paul/Documents/Work/Applications/export-to-supervisely-format/debug/app_debug_data" -DEBUG_CACHE_DIR="/home/paul/Documents/Work/Applications/export-to-supervisely-format/debug/app_debug_cache" - -LOG_LEVEL="debug" -#DEBUG_APP_DIR="/Users/maxim/work/app_debug_data" -#DEBUG_CACHE_DIR="/Users/maxim/work/app_debug_cache" - -SERVER_ADDRESS="put your value here in secret_debug.env" -API_TOKEN="put your value here in secret_debug.env" -AGENT_TOKEN="put your value here in secret_debug.env" - - diff --git a/local.env b/local.env new file mode 100644 index 0000000..6c07fcf --- /dev/null +++ b/local.env @@ -0,0 +1,12 @@ +PYTHONUNBUFFERED=1 + +TASK_ID = 24870 +context.teamId=439 +context.workspaceId=658 +modal.state.slyProjectId=14957 +modal.state.download=all +modal.state.fixExtension=true + +DEBUG_APP_DIR="/home/alex/alex_work/app_data/" +DEBUG_CACHE_DIR="/home/alex/alex_work/app_cache" +LOG_LEVEL="debug" diff --git a/requirements.txt b/requirements.txt index c95b0df..bca2ec4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -supervisely==6.68.109 +supervisely==6.69.8 diff --git a/src/main.py b/src/main.py index 10aa77f..4df2e2c 100644 --- a/src/main.py +++ b/src/main.py @@ -2,19 +2,20 @@ import supervisely as sly from supervisely.api.module_api import ApiField from supervisely.io.fs import get_file_ext -from supervisely.app.v1.app_service import AppService from distutils import util -api: sly.Api = sly.Api.from_env() -my_app: AppService = AppService() +from dotenv import load_dotenv + + +if sly.is_development(): + load_dotenv("local.env") + load_dotenv(os.path.expanduser("~/supervisely.env")) + -TEAM_ID = int(os.environ['context.teamId']) -WORKSPACE_ID = int(os.environ['context.workspaceId']) -PROJECT_ID = int(os.environ['modal.state.slyProjectId']) -task_id = int(os.environ["TASK_ID"]) mode = os.environ['modal.state.download'] replace_method = bool(util.strtobool(os.environ['modal.state.fixExtension'])) batch_size = 10 +STORAGE_DIR = sly.app.get_data_dir() def ours_convert_json_info(self, info: dict, skip_missing=True): @@ -47,62 +48,22 @@ def ours_convert_json_info(self, info: dict, skip_missing=True): return self.InfoType(*field_values) -def init(data, state): - state['download'] = mode - state['fixExtension'] = replace_method - - if replace_method: sly.logger.debug('change SDK method') sly.api.image_api.ImageApi._convert_json_info = ours_convert_json_info -@my_app.callback("download_as_sly") -@sly.timeit -def download_as_sly(api: sly.Api, task_id, context, state, app_logger): - project = api.project.get_info_by_id(PROJECT_ID) - datasets = api.dataset.get_list(project.id) - dataset_ids = [dataset.id for dataset in datasets] - if mode == 'all': - download_json_plus_images(api, project, dataset_ids) - else: - download_only_json(api, project, dataset_ids) - - download_dir = os.path.join(my_app.data_dir, f'{project.id}_{project.name}') - full_archive_name = str(project.id) + '_' + project.name + '.tar' - result_archive = os.path.join(my_app.data_dir, full_archive_name) - sly.fs.archive_directory(download_dir, result_archive) - app_logger.info("Result directory is archived") - upload_progress = [] - remote_archive_path = os.path.join( - sly.team_files.RECOMMENDED_EXPORT_PATH, "export-to-supervisely-format/{}_{}".format(task_id, full_archive_name)) - - def _print_progress(monitor, upload_progress): - if len(upload_progress) == 0: - upload_progress.append(sly.Progress(message="Upload {!r}".format(full_archive_name), - total_cnt=monitor.len, - ext_logger=app_logger, - is_size=True)) - upload_progress[0].set_current_value(monitor.bytes_read) - - file_info = api.file.upload(TEAM_ID, result_archive, remote_archive_path, - lambda m: _print_progress(m, upload_progress)) - app_logger.info("Uploaded to Team-Files: {!r}".format(file_info.storage_path)) - api.task.set_output_archive(task_id, file_info.id, full_archive_name, file_url=file_info.storage_path) - my_app.stop() - - def download_json_plus_images(api, project, dataset_ids): - sly.logger.info('DOWNLOAD_PROJECT', extra={'title': project.name}) - download_dir = os.path.join(my_app.data_dir, f'{project.id}_{project.name}') - sly.download_project(api, project.id, download_dir, dataset_ids=dataset_ids, - log_progress=True, batch_size=batch_size) - sly.logger.info('Project {!r} has been successfully downloaded.'.format(project.name)) + sly.logger.info('DOWNLOAD_PROJECT', extra={'title': project.name}) + download_dir = os.path.join(STORAGE_DIR, f'{project.id}_{project.name}') + sly.download_project(api, project.id, download_dir, dataset_ids=dataset_ids, + log_progress=True, batch_size=batch_size) + sly.logger.info('Project {!r} has been successfully downloaded.'.format(project.name)) def download_only_json(api, project, dataset_ids): sly.logger.info('DOWNLOAD_PROJECT', extra={'title': project.name}) - download_dir = os.path.join(my_app.data_dir, f'{project.id}_{project.name}') + download_dir = os.path.join(STORAGE_DIR, f'{project.id}_{project.name}') sly.fs.mkdir(download_dir) meta_json = api.project.get_meta(project.id) sly.io.json.dump_json_file(meta_json, os.path.join(download_dir, 'meta.json')) @@ -135,21 +96,27 @@ def download_only_json(api, project, dataset_ids): sly.logger.info('Total number of images: {!r}'.format(total_images)) -def main(): - sly.logger.info( - "Script arguments", - extra={ - "TEAM_ID": TEAM_ID, - "WORKSPACE_ID": WORKSPACE_ID, - "PROJECT_ID": PROJECT_ID - } - ) +class MyExport(sly.app.Export): + def process(self, context: sly.app.Export.Context): + + api = sly.Api.from_env() + + project = api.project.get_info_by_id(id=context.project_id) + datasets = api.dataset.get_list(project.id) + dataset_ids = [dataset.id for dataset in datasets] + if mode == 'all': + download_json_plus_images(api, project, dataset_ids) + else: + download_only_json(api, project, dataset_ids) - data = {} - state = {} - init(data, state) - my_app.run(initial_events=[{"command": "download_as_sly"}]) + download_dir = os.path.join(STORAGE_DIR, f'{project.id}_{project.name}') + full_archive_name = str(project.id) + '_' + project.name + '.tar' + result_archive = os.path.join(STORAGE_DIR, full_archive_name) + sly.fs.archive_directory(download_dir, result_archive) + sly.logger.info("Result directory is archived") + return result_archive + -if __name__ == "__main__": - sly.main_wrapper("main", main) +app = MyExport() +app.run() From 0f9e9fb051e03f5ae1b853584aed9ceb118a3901 Mon Sep 17 00:00:00 2001 From: peixed351 Date: Tue, 31 Jan 2023 16:40:09 +0300 Subject: [PATCH 2/3] add ds context --- config.json | 4 +-- requirements.txt | 2 +- src/main.py | 64 ++++++++++++++++++++++++++++++------------------ 3 files changed, 43 insertions(+), 27 deletions(-) diff --git a/config.json b/config.json index 87c856b..11f34ca 100644 --- a/config.json +++ b/config.json @@ -3,7 +3,7 @@ "type": "app", "categories": ["images", "export"], "description": "images and JSON annotations", - "docker_image": "supervisely/base-py-sdk:6.69.8", + "docker_image": "supervisely/base-py-sdk:6.69.19", "instance_version": "6.5.1", "main_script": "src/main.py", "modal_template": "src/modal.html", @@ -17,7 +17,7 @@ "icon": "https://i.imgur.com/1hqGMyg.png", "icon_background": "#FFFFFF", "context_menu": { - "target": ["images_project"], + "target": ["images_project", "images_dataset"], "context_root": "Download as" }, "poster": "https://user-images.githubusercontent.com/106374579/186665737-ec3da9cc-193f-43ee-85db-a6f802b2dfe4.png" diff --git a/requirements.txt b/requirements.txt index bca2ec4..556c6ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -supervisely==6.69.8 +supervisely==6.69.19 diff --git a/src/main.py b/src/main.py index 4df2e2c..7ca5bb7 100644 --- a/src/main.py +++ b/src/main.py @@ -12,8 +12,8 @@ load_dotenv(os.path.expanduser("~/supervisely.env")) -mode = os.environ['modal.state.download'] -replace_method = bool(util.strtobool(os.environ['modal.state.fixExtension'])) +mode = os.environ["modal.state.download"] +replace_method = bool(util.strtobool(os.environ["modal.state.fixExtension"])) batch_size = 10 STORAGE_DIR = sly.app.get_data_dir() @@ -32,7 +32,7 @@ def ours_convert_json_info(self, info: dict, skip_missing=True): val = info[field_name] field_values.append(val) if field_name == ApiField.MIME: - temp_ext = val.split('/')[1] + temp_ext = val.split("/")[1] field_values.append(temp_ext) for idx, field_name in enumerate(self.info_sequence()): if field_name == ApiField.NAME: @@ -40,7 +40,7 @@ def ours_convert_json_info(self, info: dict, skip_missing=True): if not cur_ext: field_values[idx] = "{}.{}".format(field_values[idx], temp_ext) break - if temp_ext == 'jpeg' and cur_ext in ['jpg', 'jpeg', 'mpo']: + if temp_ext == "jpeg" and cur_ext in ["jpg", "jpeg", "mpo"]: break if temp_ext != cur_ext and cur_ext is not None: pass @@ -49,37 +49,47 @@ def ours_convert_json_info(self, info: dict, skip_missing=True): if replace_method: - sly.logger.debug('change SDK method') + sly.logger.debug("change SDK method") sly.api.image_api.ImageApi._convert_json_info = ours_convert_json_info def download_json_plus_images(api, project, dataset_ids): - sly.logger.info('DOWNLOAD_PROJECT', extra={'title': project.name}) - download_dir = os.path.join(STORAGE_DIR, f'{project.id}_{project.name}') - sly.download_project(api, project.id, download_dir, dataset_ids=dataset_ids, - log_progress=True, batch_size=batch_size) - sly.logger.info('Project {!r} has been successfully downloaded.'.format(project.name)) + sly.logger.info("DOWNLOAD_PROJECT", extra={"title": project.name}) + download_dir = os.path.join(STORAGE_DIR, f"{project.id}_{project.name}") + sly.download_project( + api, + project.id, + download_dir, + dataset_ids=dataset_ids, + log_progress=True, + batch_size=batch_size, + ) + sly.logger.info("Project {!r} has been successfully downloaded.".format(project.name)) def download_only_json(api, project, dataset_ids): - sly.logger.info('DOWNLOAD_PROJECT', extra={'title': project.name}) - download_dir = os.path.join(STORAGE_DIR, f'{project.id}_{project.name}') + sly.logger.info("DOWNLOAD_PROJECT", extra={"title": project.name}) + download_dir = os.path.join(STORAGE_DIR, f"{project.id}_{project.name}") sly.fs.mkdir(download_dir) meta_json = api.project.get_meta(project.id) - sly.io.json.dump_json_file(meta_json, os.path.join(download_dir, 'meta.json')) + sly.io.json.dump_json_file(meta_json, os.path.join(download_dir, "meta.json")) total_images = 0 dataset_info = ( [api.dataset.get_info_by_id(ds_id) for ds_id in dataset_ids] - if (dataset_ids is not None) else api.dataset.get_list(project.id)) + if (dataset_ids is not None) + else api.dataset.get_list(project.id) + ) for dataset in dataset_info: - ann_dir = os.path.join(download_dir, dataset.name, 'ann') + ann_dir = os.path.join(download_dir, dataset.name, "ann") sly.fs.mkdir(ann_dir) images = api.image.get_list(dataset.id) ds_progress = sly.Progress( - 'Downloading annotations for: {!r}/{!r}'.format(project.name, dataset.name), total_cnt=len(images)) + "Downloading annotations for: {!r}/{!r}".format(project.name, dataset.name), + total_cnt=len(images), + ) for batch in sly.batched(images, batch_size=10): image_ids = [image_info.id for image_info in batch] image_names = [image_info.name for image_info in batch] @@ -88,12 +98,14 @@ def download_only_json(api, project, dataset_ids): ann_infos = api.annotation.download_batch(dataset.id, image_ids) for image_name, ann_info in zip(image_names, ann_infos): - sly.io.json.dump_json_file(ann_info.annotation, os.path.join(ann_dir, image_name + '.json')) + sly.io.json.dump_json_file( + ann_info.annotation, os.path.join(ann_dir, image_name + ".json") + ) ds_progress.iters_done_report(len(batch)) total_images += len(batch) - sly.logger.info('Project {!r} has been successfully downloaded'.format(project.name)) - sly.logger.info('Total number of images: {!r}'.format(total_images)) + sly.logger.info("Project {!r} has been successfully downloaded".format(project.name)) + sly.logger.info("Total number of images: {!r}".format(total_images)) class MyExport(sly.app.Export): @@ -103,20 +115,24 @@ def process(self, context: sly.app.Export.Context): project = api.project.get_info_by_id(id=context.project_id) datasets = api.dataset.get_list(project.id) - dataset_ids = [dataset.id for dataset in datasets] - if mode == 'all': + if context.dataset_id is not None: + dataset_ids = [context.dataset_id] + else: + [dataset.id for dataset in datasets] + + if mode == "all": download_json_plus_images(api, project, dataset_ids) else: download_only_json(api, project, dataset_ids) - download_dir = os.path.join(STORAGE_DIR, f'{project.id}_{project.name}') - full_archive_name = str(project.id) + '_' + project.name + '.tar' + download_dir = os.path.join(STORAGE_DIR, f"{project.id}_{project.name}") + full_archive_name = str(project.id) + "_" + project.name + ".tar" result_archive = os.path.join(STORAGE_DIR, full_archive_name) sly.fs.archive_directory(download_dir, result_archive) sly.logger.info("Result directory is archived") return result_archive - + app = MyExport() app.run() From 31a1e789325b6aed15d3eaa9c059a773d3da2d44 Mon Sep 17 00:00:00 2001 From: peixed351 Date: Tue, 31 Jan 2023 16:41:34 +0300 Subject: [PATCH 3/3] check 'dataset_ids' referenced before assignment --- src/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.py b/src/main.py index 7ca5bb7..b22de70 100644 --- a/src/main.py +++ b/src/main.py @@ -118,7 +118,7 @@ def process(self, context: sly.app.Export.Context): if context.dataset_id is not None: dataset_ids = [context.dataset_id] else: - [dataset.id for dataset in datasets] + dataset_ids = [dataset.id for dataset in datasets] if mode == "all": download_json_plus_images(api, project, dataset_ids)