From a79ef50498a6a301446f66e55e834fbc58e78362 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=2E=20Fatih=20C=C4=B1r=C4=B1t?= Date: Thu, 2 May 2024 12:11:32 +0300 Subject: [PATCH] docs: add ansible installation, update manual artifact download process (#4675) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: M. Fatih Cırıt --- ansible/README.md | 30 +++++ ansible/playbooks/download_artifacts.yaml | 4 + ansible/roles/artifacts/README.md | 146 +++------------------- ansible/roles/artifacts/SHA256SUMS | 25 ---- 4 files changed, 51 insertions(+), 154 deletions(-) create mode 100644 ansible/playbooks/download_artifacts.yaml delete mode 100644 ansible/roles/artifacts/SHA256SUMS diff --git a/ansible/README.md b/ansible/README.md index 3a252f1a23a..6fd630f75fa 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -1 +1,31 @@ # Ansible Collection - autoware.dev_env + +This collection contains the playbooks to set up the development environment for Autoware. + +## Set up a development environment + +### Ansible installation + +```bash +# Remove apt installed ansible (In Ubuntu 22.04, ansible the version is old) +sudo apt-get purge ansible + +# Install pipx +sudo apt-get -y update +sudo apt-get -y install pipx + +# Add pipx to the system PATH +python3 -m pipx ensurepath + +# Install ansible +pipx install --include-deps --force "ansible==6.*" +``` + +### Install ansible collections + +This step should be repeated when a new playbook is added. + +```bash +cd ~/autoware # The root directory of the cloned repository +ansible-galaxy collection install -f -r "ansible-galaxy-requirements.yaml" +``` diff --git a/ansible/playbooks/download_artifacts.yaml b/ansible/playbooks/download_artifacts.yaml new file mode 100644 index 00000000000..17da12494d5 --- /dev/null +++ b/ansible/playbooks/download_artifacts.yaml @@ -0,0 +1,4 @@ +- name: Download Autoware artifacts + hosts: localhost + roles: + - autoware.dev_env.artifacts diff --git a/ansible/roles/artifacts/README.md b/ansible/roles/artifacts/README.md index 7b8d6f67d77..7d848a98eca 100644 --- a/ansible/roles/artifacts/README.md +++ b/ansible/roles/artifacts/README.md @@ -1,144 +1,32 @@ # Autoware artifacts -The Autoware perception stack uses models for inference. These models are automatically downloaded if using `ansible`, but they can also be downloaded manually. +The Autoware perception stack uses models for inference. These models are automatically downloaded as part of the `setup-dev-env.sh` script. -## Download instructions - -The artifacts files are stored in a common location, hosted by Web.Auto - -Any tool that can download files from the web (e.g. `wget` or `curl`) is the only requirement for downloading these files: - -```console -# yabloc_pose_initializer - -$ mkdir -p ~/autoware_data/yabloc_pose_initializer/ -$ wget -P ~/autoware_data/yabloc_pose_initializer/ \ - https://s3.ap-northeast-2.wasabisys.com/pinto-model-zoo/136_road-segmentation-adas-0001/resources.tar.gz - - -# image_projection_based_fusion - -$ mkdir -p ~/autoware_data/image_projection_based_fusion/ -$ wget -P ~/autoware_data/image_projection_based_fusion/ \ - https://awf.ml.dev.web.auto/perception/models/pointpainting/v4/pts_voxel_encoder_pointpainting.onnx \ - https://awf.ml.dev.web.auto/perception/models/pointpainting/v4/pts_backbone_neck_head_pointpainting.onnx - - -# lidar_apollo_instance_segmentation - -$ mkdir -p ~/autoware_data/lidar_apollo_instance_segmentation/ -$ wget -P ~/autoware_data/lidar_apollo_instance_segmentation/ \ - https://awf.ml.dev.web.auto/perception/models/lidar_apollo_instance_segmentation/vlp-16.onnx \ - https://awf.ml.dev.web.auto/perception/models/lidar_apollo_instance_segmentation/hdl-64.onnx \ - https://awf.ml.dev.web.auto/perception/models/lidar_apollo_instance_segmentation/vls-128.onnx - - -# lidar_centerpoint - -$ mkdir -p ~/autoware_data/lidar_centerpoint/ -$ wget -P ~/autoware_data/lidar_centerpoint/ \ - https://awf.ml.dev.web.auto/perception/models/centerpoint/v2/pts_voxel_encoder_centerpoint.onnx \ - https://awf.ml.dev.web.auto/perception/models/centerpoint/v2/pts_backbone_neck_head_centerpoint.onnx \ - https://awf.ml.dev.web.auto/perception/models/centerpoint/v2/pts_voxel_encoder_centerpoint_tiny.onnx \ - https://awf.ml.dev.web.auto/perception/models/centerpoint/v2/pts_backbone_neck_head_centerpoint_tiny.onnx - - -# tensorrt_yolox - -$ mkdir -p ~/autoware_data/tensorrt_yolox/ -$ wget -P ~/autoware_data/tensorrt_yolox/ \ - https://awf.ml.dev.web.auto/perception/models/yolox-tiny.onnx \ - https://awf.ml.dev.web.auto/perception/models/yolox-sPlus-opt.onnx \ - https://awf.ml.dev.web.auto/perception/models/yolox-sPlus-opt.EntropyV2-calibration.table \ - https://awf.ml.dev.web.auto/perception/models/object_detection_yolox_s/v1/yolox-sPlus-T4-960x960-pseudo-finetune.onnx \ - https://awf.ml.dev.web.auto/perception/models/object_detection_yolox_s/v1/yolox-sPlus-T4-960x960-pseudo-finetune.EntropyV2-calibration.table \ - https://awf.ml.dev.web.auto/perception/models/label.txt - - -# traffic_light_classifier - -$ mkdir -p ~/autoware_data/traffic_light_classifier/ -$ wget -P ~/autoware_data/traffic_light_classifier/ \ - https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_mobilenetv2_batch_1.onnx \ - https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_mobilenetv2_batch_4.onnx \ - https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_mobilenetv2_batch_6.onnx \ - https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_efficientNet_b1_batch_1.onnx \ - https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_efficientNet_b1_batch_4.onnx \ - https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/traffic_light_classifier_efficientNet_b1_batch_6.onnx \ - https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v2/lamp_labels.txt \ - https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v3/ped_traffic_light_classifier_mobilenetv2_batch_1.onnx \ - https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v3/ped_traffic_light_classifier_mobilenetv2_batch_4.onnx \ - https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v3/ped_traffic_light_classifier_mobilenetv2_batch_6.onnx \ - https://awf.ml.dev.web.auto/perception/models/traffic_light_classifier/v3/lamp_labels_ped.txt +The models are hosted by Web.Auto. +Default `data_dir` location is `~/autoware_data`. +## Download instructions -# traffic_light_fine_detector - -$ mkdir -p ~/autoware_data/traffic_light_fine_detector/ -$ wget -P ~/autoware_data/traffic_light_fine_detector/ \ - https://awf.ml.dev.web.auto/perception/models/tlr_yolox_s/v3/tlr_car_ped_yolox_s_batch_1.onnx \ - https://awf.ml.dev.web.auto/perception/models/tlr_yolox_s/v3/tlr_car_ped_yolox_s_batch_4.onnx \ - https://awf.ml.dev.web.auto/perception/models/tlr_yolox_s/v3/tlr_car_ped_yolox_s_batch_6.onnx \ - https://awf.ml.dev.web.auto/perception/models/tlr_yolox_s/v3/tlr_labels.txt - - -# tvm_utility - -$ mkdir -p ~/autoware_data/tvm_utility/models/yolo_v2_tiny -$ wget -P ~/autoware_data/tvm_utility/ \ - https://autoware-modelzoo.s3.us-east-2.amazonaws.com/models/3.0.0-20221221/yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz - - -# lidar_centerpoint_tvm - -$ mkdir -p ~/autoware_data/lidar_centerpoint_tvm/models/centerpoint_encoder -$ mkdir -p ~/autoware_data/lidar_centerpoint_tvm/models/centerpoint_backbone -$ wget -P ~/autoware_data/lidar_centerpoint_tvm/ \ - https://autoware-modelzoo.s3.us-east-2.amazonaws.com/models/3.0.0-20221221/centerpoint_encoder-x86_64-llvm-3.0.0-20221221.tar.gz \ - https://autoware-modelzoo.s3.us-east-2.amazonaws.com/models/3.0.0-20221221/centerpoint_backbone-x86_64-llvm-3.0.0-20221221.tar.gz - +### Requirements -# lidar_apollo_segmentation_tvm +Install ansible following the instructions in the [ansible installation guide](../../README.md#ansible-installation). -$ mkdir -p ~/autoware_data/lidar_apollo_segmentation_tvm/models/baidu_cnn -$ wget -P ~/autoware_data/lidar_apollo_segmentation_tvm/ \ - https://autoware-modelzoo.s3.us-east-2.amazonaws.com/models/3.0.0-20221221/baidu_cnn-x86_64-llvm-3.0.0-20221221.tar.gz -``` +### Download artifacts -After downloading you can check integrity of the files with `sha256sum`: +#### Install ansible collections -```console -# -$ cd ~/autoware_data/ -$ wget -q -O - https://raw.githubusercontent.com/autowarefoundation/autoware/main/ansible/roles/artifacts/SHA256SUMS | sha256sum -c +```bash +cd ~/autoware # The root directory of the cloned repository +ansible-galaxy collection install -f -r "ansible-galaxy-requirements.yaml" ``` -Extracting files: +This step should be repeated when a new playbook is added. -```console -# yabloc_pose_initializer +#### Run the playbook -$ tar -xf ~/autoware_data/yabloc_pose_initializer/resources.tar.gz \ - -C ~/autoware_data/yabloc_pose_initializer/ - - -# tvm_utility - -$ tar -xf ~/autoware_data/tvm_utility/yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz \ - -C ~/autoware_data/tvm_utility/models/yolo_v2_tiny/ - - -# lidar_centerpoint_tvm - -$ tar -xf ~/autoware_data/lidar_centerpoint_tvm/centerpoint_encoder-x86_64-llvm-3.0.0-20221221.tar.gz \ - -C ~/autoware_data/lidar_centerpoint_tvm/models/centerpoint_encoder -$ tar -xf ~/autoware_data/lidar_centerpoint_tvm/centerpoint_backbone-x86_64-llvm-3.0.0-20221221.tar.gz \ - -C ~/autoware_data/lidar_centerpoint_tvm/models/centerpoint_backbone - - -# lidar_apollo_segmentation_tvm - -$ tar -xf ~/autoware_data/lidar_apollo_segmentation_tvm/baidu_cnn-x86_64-llvm-3.0.0-20221221.tar.gz \ - -C ~/autoware_data/lidar_apollo_segmentation_tvm/models/baidu_cnn +```bash +ansible-playbook autoware.dev_env.download_artifacts -e "data_dir=$HOME/autoware_data" --ask-become-pass ``` + +This will download and extract the artifacts to the specified directory and validate the checksums. diff --git a/ansible/roles/artifacts/SHA256SUMS b/ansible/roles/artifacts/SHA256SUMS deleted file mode 100644 index 0abb9037e64..00000000000 --- a/ansible/roles/artifacts/SHA256SUMS +++ /dev/null @@ -1,25 +0,0 @@ -7fe62fcebe0e0f62a000d06aa94d779feb444d933671a4a3189fe01be8c19a00 ./image_projection_based_fusion/pts_backbone_neck_head_pointpainting.onnx -3ca452ea5ca9467bf782955f75704ba8466841e275e8b8acd991b9911d53249e ./image_projection_based_fusion/pts_voxel_encoder_pointpainting.onnx -86348d8c4bced750f54288b01cc471c0d4f1ec9c693466169ef19413731e6ecc ./lidar_apollo_instance_segmentation/hdl-64.onnx -eec521ebad7553d0ea2c90472a293aecb7499ab592632f0e100481c8196eb421 ./lidar_apollo_instance_segmentation/vlp-16.onnx -95ef950bb694bd6de91b7e47f5d191d557e92a7f5e2a6bdf655a8b5eed4075cc ./lidar_apollo_instance_segmentation/vls-128.onnx -4293e6196ec937d2cd5ec658e5ce70933647d2d38633a1805febb36cafd684e3 ./lidar_apollo_segmentation_tvm/baidu_cnn-x86_64-llvm-3.0.0-20221221.tar.gz -3fe7e128955646740c41a25be0c8f141d5a94594fe79d7405fe2a859e391542e ./lidar_centerpoint/pts_backbone_neck_head_centerpoint.onnx -9bb0b634f3664bd098ce7d6a3d8a9fb7cc8d9b8252b27f302c71e43316bab551 ./lidar_centerpoint/pts_backbone_neck_head_centerpoint_tiny.onnx -dc1a876580d86ee7a341d543f8ade2ede7f43bd032dc5b44155b1f0175405764 ./lidar_centerpoint/pts_voxel_encoder_centerpoint.onnx -2c53465715c1fd2e9dc5727ef3fca74f4cdf0538f74286b0946e219d0ca5693b ./lidar_centerpoint/pts_voxel_encoder_centerpoint_tiny.onnx -3840b6b3590984e8115d66b12061aea3a2cfaed70b4e8d59457f04b0d6f6a1fc ./lidar_centerpoint_tvm/centerpoint_backbone-x86_64-llvm-3.0.0-20221221.tar.gz -41e19de43f30549c325968aee06b4ad0d9701220be819e79d37efdfa86b918d0 ./lidar_centerpoint_tvm/centerpoint_encoder-x86_64-llvm-3.0.0-20221221.tar.gz -1a5a49eeec5593963eab8d70f48b8a01bfb07e753e9688eb1510ad26e803579d ./traffic_light_classifier/lamp_labels.txt -55ebb0d117a5e8943f8d1c6769f1d856b533079d4d871d8e923255cc992ad48a ./traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_1.onnx -684e29843e3128eadb774018730644b3ab9b0a06dc4cdaeed579c2f3fa5d5265 ./traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_4.onnx -44d94540fa8b89dfb39cd9a8523cf010ddfb10ea2f1f9b53bf3618ce7f4912ad ./traffic_light_classifier/traffic_light_classifier_efficientNet_b1_batch_6.onnx -455b71b3b20d3a96aa0e49f32714ba50421f668a2f9b9907c30b1346ac8a3703 ./traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_1.onnx -41bb79a23a4ac57956adb8e9cb3904420db1b0cd032e97b670cc4f8b174ae3fe ./traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_4.onnx -e4792eed6a46fdbd02be2f3a4f1ce91f36fa77698493caf3102e445178c0f058 ./traffic_light_classifier/traffic_light_classifier_mobilenetv2_batch_6.onnx -a2a91f5fe9c2e68e3e3647a272bb9bb25cd07631a1990a3fb15efddce7691131 ./traffic_light_fine_detector/tlr_labels.txt -1ad633066a1195006f4709f8fa07800dd65a74a814b3efb4c99bcc5a1a7962f6 ./traffic_light_fine_detector/tlr_car_ped_yolox_s_batch_1.onnx -cf93eb1e1a97aefc6edd0c0c4d77c7f5fc2aa1e81e3c5c9cd49d976173d03a04 ./traffic_light_fine_detector/tlr_car_ped_yolox_s_batch_4.onnx -0b05a89fb30f1f92c6ec687d48e8ceda4da6f81cbd82d8a102d168753a6cedb6 ./traffic_light_fine_detector/tlr_car_ped_yolox_s_batch_6.onnx -66b3ca668e577393b657fbe1ed626538d89ca3adccd5862de6c7fa190238dbca ./tvm_utility/yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz -1f660e15f95074bade32b1f80dbf618e9cee1f0b9f76d3f4671cb9be7f56eb3a ./yabloc_pose_initializer/resources.tar.gz