From f768b2c8707deaff5e393af09844ccc0ce3d8d64 Mon Sep 17 00:00:00 2001 From: Esteve Fernandez <33620+esteve@users.noreply.github.com> Date: Tue, 19 Sep 2023 10:37:11 +0200 Subject: [PATCH] feat: use ansible to download ONNX files (#3375) * feat: use ansible to download ONNX files Signed-off-by: Esteve Fernandez * feat: add remaining ONNX files Signed-off-by: Esteve Fernandez * feat: add data_dir option to choose the location to store models Signed-off-by: Esteve Fernandez * feat: update artifacts per package Signed-off-by: Esteve Fernandez * fix: fix paths for downloaded artifacts Signed-off-by: Esteve Fernandez * style(pre-commit): autofix * fix: create directories before downloading models Signed-off-by: Esteve Fernandez --------- Signed-off-by: Esteve Fernandez Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- ansible/playbooks/universe.yaml | 9 + ansible/roles/artifacts/defaults/main.yaml | 0 ansible/roles/artifacts/meta/main.yaml | 0 ansible/roles/artifacts/tasks/main.yaml | 342 +++++++++++++++++++++ setup-dev-env.sh | 9 + 5 files changed, 360 insertions(+) create mode 100644 ansible/roles/artifacts/defaults/main.yaml create mode 100644 ansible/roles/artifacts/meta/main.yaml create mode 100644 ansible/roles/artifacts/tasks/main.yaml diff --git a/ansible/playbooks/universe.yaml b/ansible/playbooks/universe.yaml index 43333357743..9ffb57bed3c 100644 --- a/ansible/playbooks/universe.yaml +++ b/ansible/playbooks/universe.yaml @@ -7,6 +7,11 @@ [Warning] Some Autoware components depend on the CUDA, cuDNN and TensorRT NVIDIA libraries which have end-user license agreements that should be reviewed before installation. Install NVIDIA libraries? [y/N] private: false + - name: prompt_download_artifacts + prompt: |- + [Warning] Should the ONNX model files and other artifacts be downloaded alongside setting up the development environment. + Download artifacts? [y/N] + private: false pre_tasks: - name: Verify OS ansible.builtin.fail: @@ -49,3 +54,7 @@ - role: autoware.dev_env.tensorrt when: prompt_install_nvidia == 'y' - role: autoware.dev_env.git_lfs + + # ONNX files and other artifacts + - role: autoware.dev_env.artifacts + when: prompt_download_artifacts == 'y' diff --git a/ansible/roles/artifacts/defaults/main.yaml b/ansible/roles/artifacts/defaults/main.yaml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ansible/roles/artifacts/meta/main.yaml b/ansible/roles/artifacts/meta/main.yaml new file mode 100644 index 00000000000..e69de29bb2d diff --git a/ansible/roles/artifacts/tasks/main.yaml b/ansible/roles/artifacts/tasks/main.yaml new file mode 100644 index 00000000000..56999fa03be --- /dev/null +++ b/ansible/roles/artifacts/tasks/main.yaml @@ -0,0 +1,342 @@ +# yabloc_pose_initializer +- name: Create yabloc_pose_initializer directory inside {{ data_dir }} + ansible.builtin.file: + path: "{{ data_dir }}/yabloc_pose_initializer" + mode: "755" + state: directory + +- name: Download yabloc_pose_initializer/resources.tar.gz + become: true + ansible.builtin.get_url: + url: https://s3.ap-northeast-2.wasabisys.com/pinto-model-zoo/136_road-segmentation-adas-0001/resources.tar.gz + dest: "{{ data_dir }}/yabloc_pose_initializer/resources.tar.gz" + mode: "644" + checksum: sha256:1f660e15f95074bade32b1f80dbf618e9cee1f0b9f76d3f4671cb9be7f56eb3a + +# image_projection_based_fusion +- name: Create image_projection_based_fusion directory inside {{ data_dir }} + ansible.builtin.file: + path: "{{ data_dir }}/image_projection_based_fusion" + mode: "755" + state: directory + +- name: Download image_projection_based_fusion/pts_voxel_encoder_pointpainting.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/pointpainting/v4/pts_voxel_encoder_pointpainting.onnx + dest: "{{ data_dir }}/image_projection_based_fusion/pts_voxel_encoder_pointpainting.onnx" + mode: "644" + checksum: md5:25c70f76a45a64944ccd19f604c99410 + +- name: Download image_projection_based_fusion/pts_backbone_neck_head_pointpainting.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/pointpainting/v4/pts_backbone_neck_head_pointpainting.onnx + dest: "{{ data_dir }}/image_projection_based_fusion/pts_backbone_neck_head_pointpainting.onnx" + mode: "644" + checksum: md5:2c7108245240fbd7bf0104dd68225868 + +# lidar_apollo_instance_segmentation +- name: Create lidar_apollo_instance_segmentation directory inside {{ data_dir }} + ansible.builtin.file: + path: "{{ data_dir }}/lidar_apollo_instance_segmentation" + mode: "755" + state: directory + +- name: Download lidar_apollo_instance_segmentation/vlp-16.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/lidar_apollo_instance_segmentation/vlp-16.onnx + dest: "{{ data_dir }}/lidar_apollo_instance_segmentation/vlp-16.onnx" + mode: "644" + checksum: md5:63a5a1bb73f7dbb64cf70d04eca45fb4 + +- name: Download lidar_apollo_instance_segmentation/hdl-64.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/lidar_apollo_instance_segmentation/hdl-64.onnx + dest: "{{ data_dir }}/lidar_apollo_instance_segmentation/hdl-64.onnx" + mode: "644" + checksum: md5:009745e33b1df44b68296431cc384cd2 + +- name: Download lidar_apollo_instance_segmentation/vls-128.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/lidar_apollo_instance_segmentation/vls-128.onnx + dest: "{{ data_dir }}/lidar_apollo_instance_segmentation/vls-128.onnx" + mode: "644" + checksum: md5:b2d709f56f73ae2518c9bf4d0214468f + +# lidar_centerpoint +- name: Create lidar_centerpoint directory inside {{ data_dir }} + ansible.builtin.file: + path: "{{ data_dir }}/lidar_centerpoint" + mode: "755" + state: directory + +- name: Download lidar_centerpoint/pts_voxel_encoder_centerpoint.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/centerpoint/v2/pts_voxel_encoder_centerpoint.onnx + dest: "{{ data_dir }}/lidar_centerpoint/pts_voxel_encoder_centerpoint.onnx" + mode: "644" + checksum: sha256:dc1a876580d86ee7a341d543f8ade2ede7f43bd032dc5b44155b1f0175405764 + +- name: Download lidar_centerpoint/pts_backbone_neck_head_centerpoint.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/centerpoint/v2/pts_backbone_neck_head_centerpoint.onnx + dest: "{{ data_dir }}/lidar_centerpoint/pts_backbone_neck_head_centerpoint.onnx" + mode: "644" + checksum: sha256:3fe7e128955646740c41a25be0c8f141d5a94594fe79d7405fe2a859e391542e + +- name: Download lidar_centerpoint/pts_voxel_encoder_centerpoint_tiny.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/centerpoint/v2/pts_voxel_encoder_centerpoint_tiny.onnx + dest: "{{ data_dir }}/lidar_centerpoint/pts_voxel_encoder_centerpoint_tiny.onnx" + mode: "644" + checksum: sha256:2c53465715c1fd2e9dc5727ef3fca74f4cdf0538f74286b0946e219d0ca5693b + +- name: Download lidar_centerpoint/pts_backbone_neck_head_centerpoint_tiny.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/centerpoint/v2/pts_backbone_neck_head_centerpoint_tiny.onnx + dest: "{{ data_dir }}/lidar_centerpoint/pts_backbone_neck_head_centerpoint_tiny.onnx" + mode: "644" + checksum: md5:e4658325b70222f7c3637fe00e586b82 + +# tensorrt_yolo +- name: Create tensorrt_yolo directory inside {{ data_dir }} + ansible.builtin.file: + path: "{{ data_dir }}/tensorrt_yolo" + mode: "755" + state: directory + +- name: Download tensorrt_yolo/yolov3.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/yolov3.onnx + dest: "{{ data_dir }}/tensorrt_yolo/yolov3.onnx" + mode: "644" + checksum: sha256:61e922f76918dd3d8e0abdc5fb7406f390609e08bd8ab9e5d3b97afb00f30f8c + +- name: Download tensorrt_yolo/yolov4.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/yolov4.onnx + dest: "{{ data_dir }}/tensorrt_yolo/yolov4.onnx" + mode: "644" + checksum: sha256:7c7343156c1bd4b397fd1e44b27334691a6219db3ce2e29a03b72af65ddb8f39 + +- name: Download tensorrt_yolo/yolov4-tiny.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/yolov4-tiny.onnx + dest: "{{ data_dir }}/tensorrt_yolo/yolov4-tiny.onnx" + mode: "644" + checksum: sha256:0e877c716fbf8a2b431ee3e57f6c7411a6741319b52c32c6dafc53c7e1b17027 + +- name: Download tensorrt_yolo/yolov5s.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/yolov5s.onnx + dest: "{{ data_dir }}/tensorrt_yolo/yolov5s.onnx" + mode: "644" + checksum: sha256:be335ff7746957debf1a6903a61fa3f568b780b4afe4958edf2d4bc98e9e0825 + +- name: Download tensorrt_yolo/yolov5m.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/yolov5m.onnx + dest: "{{ data_dir }}/tensorrt_yolo/yolov5m.onnx" + mode: "644" + checksum: sha256:ee6f67f7c00a34cc4cef2fdd9db30dd714df1a4fb2d7e9fc1731cfe85b673133 + +- name: Download tensorrt_yolo/yolov5l.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/yolov5l.onnx + dest: "{{ data_dir }}/tensorrt_yolo/yolov5l.onnx" + mode: "644" + checksum: sha256:a627e5f70180a8746482b572194090466db62c8d1256602c1cd20374dd960e34 + +- name: Download tensorrt_yolo/yolov5x.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/yolov5x.onnx + dest: "{{ data_dir }}/tensorrt_yolo/yolov5x.onnx" + mode: "644" + checksum: sha256:d7cb4cd7078f87bda22a37828d72867accecedf9f74d0d87b5cc1f6f1180a019 + +- name: Download tensorrt_yolo/coco.names + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/coco.names + dest: "{{ data_dir }}/tensorrt_yolo/coco.names" + mode: "644" + checksum: sha256:634a1132eb33f8091d60f2c346ababe8b905ae08387037aed883953b7329af84 + +# tensorrt_yolox +- name: Create tensorrt_yolox directory inside {{ data_dir }} + ansible.builtin.file: + path: "{{ data_dir }}/tensorrt_yolox" + mode: "755" + state: directory + +- name: Download tensorrt_yolox/yolox-tiny.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/yolox-tiny.onnx + dest: "{{ data_dir }}/tensorrt_yolox/yolox-tiny.onnx" + mode: "644" + checksum: sha256:471a665f4243e654dff62578394e508db22ee29fe65d9e389dfc3b0f2dee1255 + +- name: Download tensorrt_yolox/yolox-sPlus-opt.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/yolox-sPlus-opt.onnx + dest: "{{ data_dir }}/tensorrt_yolox/yolox-sPlus-opt.onnx" + mode: "644" + checksum: md5:bf3b0155351f90fcdca2626acbfd3bcf + +- name: Download tensorrt_yolox/yolox-sPlus-opt.EntropyV2-calibration.table + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/yolox-sPlus-opt.EntropyV2-calibration.table + dest: "{{ data_dir }}/tensorrt_yolox/yolox-sPlus-opt.EntropyV2-calibration.table" + mode: "644" + checksum: md5:c6e6f1999d5724a017516a956096701f + +- name: Download tensorrt_yolox/label.txt + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/label.txt + dest: "{{ data_dir }}/tensorrt_yolox/label.txt" + mode: "644" + checksum: sha256:3540a365bfd6d8afb1b5d8df4ec47f82cb984760d3270c9b41dbbb3422d09a0c + +# traffic_light_classifier +- name: Create traffic_light_classifier directory inside {{ data_dir }} + ansible.builtin.file: + path: "{{ data_dir }}/traffic_light_classifier" + mode: "755" + state: directory + +- name: Download traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_1.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_mobilenetv2_batch_1.onnx + dest: "{{ data_dir }}/traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_1.onnx" + mode: "644" + checksum: md5:caa51f2080aa2df943e4f884c41898eb + +- name: Download traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_4.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_mobilenetv2_batch_4.onnx + dest: "{{ data_dir }}/traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_4.onnx" + mode: "644" + checksum: md5:c2beaf60210f471debfe72b86d076ca0 + +- name: Download traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_6.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_mobilenetv2_batch_6.onnx + dest: "{{ data_dir }}/traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_6.onnx" + mode: "644" + checksum: md5:28b408710bcb24f4cdd4d746301d4e78 + +- name: Download traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_1.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_efficientNet_b1_batch_1.onnx + dest: "{{ data_dir }}/traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_1.onnx" + mode: "644" + checksum: md5:82baba4fcf1abe0c040cd55005e34510 + +- name: Download traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_4.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_efficientNet_b1_batch_4.onnx + dest: "{{ data_dir }}/traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_4.onnx" + mode: "644" + checksum: md5:21b549c2fe4fbb20d32cc019e6d70cd7 + +- name: Download traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_6.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_efficientNet_b1_batch_6.onnx + dest: "{{ data_dir }}/traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_6.onnx" + mode: "644" + checksum: md5:378526d9aa9fc6705cf399f7b35b3053 + +- name: Download traffic_light_classifier/lamp_labels.txt + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/lamp_labels.txt + dest: "{{ data_dir }}/traffic_light_classifier/lamp_labels.txt" + mode: "644" + checksum: sha256:1a5a49eeec5593963eab8d70f48b8a01bfb07e753e9688eb1510ad26e803579d + +# traffic_light_fine_detector +- name: Create traffic_light_fine_detector directory inside {{ data_dir }} + ansible.builtin.file: + path: "{{ data_dir }}/traffic_light_fine_detector" + mode: "755" + state: directory + +- name: Download traffic_light_fine_detector/tlr_yolox_s_batch_1.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/tlr_yolox_s/v2/tlr_yolox_s_batch_1.onnx + dest: "{{ data_dir }}/traffic_light_fine_detector/tlr_yolox_s_batch_1.onnx" + mode: "644" + checksum: md5:2b72d085022b8ee6aacff06bd722cfda + +- name: Download traffic_light_fine_detector/tlr_yolox_s_batch_4.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/tlr_yolox_s/v2/tlr_yolox_s_batch_4.onnx + dest: "{{ data_dir }}/traffic_light_fine_detector/tlr_yolox_s_batch_4.onnx" + mode: "644" + checksum: md5:4044daa86e7776ce241e94d98a09cc0e + +- name: Download traffic_light_fine_detector/tlr_yolox_s_batch_6.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/tlr_yolox_s/v2/tlr_yolox_s_batch_6.onnx + dest: "{{ data_dir }}/traffic_light_fine_detector/tlr_yolox_s_batch_6.onnx" + mode: "644" + checksum: md5:47255a11bde479320d703f1f45db1242 + +- name: Download traffic_light_fine_detector/tlr_labels.txt + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/tlr_yolox_s/v2/tlr_labels.txt + dest: "{{ data_dir }}/traffic_light_fine_detector/tlr_labels.txt" + mode: "644" + checksum: md5:e9f45efb02f2a9aa8ac27b3d5c164905 + +# traffic_light_ssd_fine_detector +- name: Create traffic_light_ssd_fine_detector directory inside {{ data_dir }} + ansible.builtin.file: + path: "{{ data_dir }}/traffic_light_ssd_fine_detector" + mode: "755" + state: directory + +- name: Download traffic_light_ssd_fine_detector/mb2-ssd-lite-tlr.onnx + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/mb2-ssd-lite-tlr.onnx + dest: "{{ data_dir }}/traffic_light_ssd_fine_detector/mb2-ssd-lite-tlr.onnx" + mode: "644" + checksum: sha256:e29e6ee68751a270fb285fd037713939ca7f61a897b4c3a7ab22b0d6a9a21ddf + +- name: Download traffic_light_ssd_fine_detector/voc_labels_tl.txt + become: true + ansible.builtin.get_url: + url: https://awf.ml.dev.web.auto/perception/models/voc_labels_tl.txt + dest: "{{ data_dir }}/traffic_light_ssd_fine_detector/voc_labels_tl.txt" + mode: "644" + checksum: sha256:a41e6e3324e32c30b3b2fe38908eaf3471e2bfdaeb9e14ca0c1c3bc0275119c6 diff --git a/setup-dev-env.sh b/setup-dev-env.sh index da3589512a0..42c714b7b9b 100755 --- a/setup-dev-env.sh +++ b/setup-dev-env.sh @@ -9,6 +9,8 @@ SCRIPT_DIR=$(readlink -f "$(dirname "$0")") # Parse arguments args=() +option_data_dir="$HOME/autoware_data" + while [ "$1" != "" ]; do case "$1" in -y) @@ -31,6 +33,11 @@ while [ "$1" != "" ]; do # Disable installation dev package of role 'cuda' and 'tensorrt'. option_runtime=true ;; + --data-dir) + # Set data directory + option_data_dir="$2" + shift + ;; *) args+=("$1") ;; @@ -90,6 +97,8 @@ else ansible_args+=("--extra-vars" "install_devel=true") fi +ansible_args+=("--extra-vars" "data_dir=$option_data_dir") + # Load env source "$SCRIPT_DIR/amd64.env" if [ "$(uname -m)" = "aarch64" ]; then