diff --git a/.vscode/launch.json b/.vscode/launch.json index f0f358c..9b269de 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,6 +1,17 @@ { "version": "0.2.0", "configurations": [ + { + "name": "Python: Current File", + "type": "debugpy", + "request": "launch", + "program": "${file}", + "console": "integratedTerminal", + "justMyCode": true, + "env": { + "PYTHONPATH": "${workspaceFolder}:${PYTHONPATH}" + } + }, { "name": "Launch app", "type": "python", diff --git a/local.env b/local.env index 480bac8..45f3db4 100644 --- a/local.env +++ b/local.env @@ -4,11 +4,15 @@ LOG_LEVEL="debug" TASK_ID=22046 -CONTEXT_TEAMID=8 -CONTEXT_WORKSPACEID=349 +#CONTEXT_TEAMID=8 +#CONTEXT_WORKSPACEID=349 +# +#CONTEXT_PROJECTID=14278 +#CONTEXT_DATASETID=51995 -CONTEXT_PROJECTID=14278 -CONTEXT_DATASETID=51995 +TEAM_ID = 431 +WORKSPACE_ID = 1019 +PROJECT_ID = 40721 DEBUG_APP_DIR="src/debug/app_debug_data" DEBUG_CACHE_DIR="src/debug/app_debug_cache" diff --git a/src/main.py b/src/main.py index 98b94b1..5b71a15 100644 --- a/src/main.py +++ b/src/main.py @@ -12,6 +12,9 @@ from src.globals import DATASET_ID, PROJECT_DIR, PROJECT_ID, api import src.workflow as w +from tinytimer import Timer +import asyncio + @sly.handle_exceptions(has_ui=False) def main(): project = api.project.get_info_by_id(id=PROJECT_ID) @@ -25,6 +28,12 @@ def main(): datasets = api.dataset.get_list(project_id=PROJECT_ID) w.workflow_input(api, PROJECT_ID, type="project") progress_ds = sly.Progress(message="Exporting datasets", total_cnt=len(datasets)) + + if api.server_address.startswith("https://"): + semaphore = asyncio.Semaphore(100) + else: + semaphore = None + for dataset in datasets: dataset_dir = os.path.join(PROJECT_DIR, dataset.name) images_dir = os.path.join(dataset_dir, "images") @@ -42,10 +51,20 @@ def main(): for img_info in images_infos ] + with Timer() as t: + coro = api.image.download_paths_async(images_ids, images_paths, semaphore) + loop = sly.utils.get_or_create_event_loop() + if loop.is_running(): + future = asyncio.run_coroutine_threadsafe(coro, loop) + future.result() + else: + loop.run_until_complete(coro) + sly.logger.info( + f"Downloading time: {t.elapsed:.4f} seconds per {len(images_ids)} images ({t.elapsed/len(images_ids):.4f} seconds per image)" + ) + progress_img = sly.Progress(message="Processing images", total_cnt=len(images_ids)) - for batch_img_ids, batch_img_paths, batch_anns, batch_anns_paths in zip( - sly.batched(images_ids), - sly.batched(images_paths), + for batch_anns, batch_anns_paths in zip( sly.batched(anns), sly.batched(anns_paths), ): @@ -54,10 +73,7 @@ def main(): anns=batch_anns, project_meta=project_meta, ) - api.image.download_paths( - dataset_id=dataset.id, ids=batch_img_ids, paths=batch_img_paths - ) - progress_img.iters_done_report(len(batch_img_ids)) + progress_img.iters_done_report(len(batch_anns_paths)) progress_ds.iter_done_report() file_info = upload_project_to_tf(api, project)