feat: upstream to main (#128)

* chore(docker): remove `/autoware/log` after `colcon build` (autowarefoundation#5329) * chore(.github): always run `Show disk space` (autowarefoundation#5354) always show disk space Signed-off-by: Yutaka Kondo <[email protected]> * build(autoware.repos): remove ament_cmake fork repository (autowarefoundation#5360) Signed-off-by: Chengyong Lin <[email protected]> * fix: remove `ndt_omp` (autowarefoundation#5390) Removed ndt_omp Signed-off-by: Shintaro Sakoda <[email protected]> * fix: change the organization of `awsim_sensor_kit_launch` from RobotecAI to tier4 (autowarefoundation#5403) Fixed organization of `awsim_sensor_kit_launch` from RobotecAI to tier4 Signed-off-by: Shintaro Sakoda <[email protected]> * chore(.devcontainer): rename `.devcontainer` directories (autowarefoundation#5407) rename .devcontainer dirs Signed-off-by: Yutaka Kondo <[email protected]> * feat(docker): integrate `cuda`/`no-cuda` jobs into single job (autowarefoundation#5363) * chore(.github): rename `bake-target` to `target-image` and add descriptions to args (autowarefoundation#5413) * rename target-image and add descriptions Signed-off-by: Yutaka Kondo <[email protected]> * fix feedback Signed-off-by: Yutaka Kondo <[email protected]> --------- Signed-off-by: Yutaka Kondo <[email protected]> * feat(docker): install CUDA development/runtime libraries only 1 time (autowarefoundation#5419) * add base-cuda Signed-off-by: Yutaka Kondo <[email protected]> * update svg Signed-off-by: Yutaka Kondo <[email protected]> * not cache cuda packages Signed-off-by: Yutaka Kondo <[email protected]> * push base-cuda image Signed-off-by: Yutaka Kondo <[email protected]> * Update docker-build-and-push.yaml * Update docker-build-and-push-arm64.yaml * separate jobs Signed-off-by: Yutaka Kondo <[email protected]> --------- Signed-off-by: Yutaka Kondo <[email protected]> * refactor(docker): replace the multiple `rosdep` commands with `resolve_rosdep_keys.sh`. (autowarefoundation#5424) * run resolve_rosdep_keys.sh Signed-off-by: Yutaka Kondo <[email protected]> * style(pre-commit): autofix * update .dockerignore Signed-off-by: Yutaka Kondo <[email protected]> * chmod +x Signed-off-by: Yutaka Kondo <[email protected]> * fix location Signed-off-by: Yutaka Kondo <[email protected]> * remove rosdep update Signed-off-by: Yutaka Kondo <[email protected]> --------- Signed-off-by: Yutaka Kondo <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * refactor(docker): replace the multiple `colcon` commands with `build_and_clean.sh`. (autowarefoundation#5425) * run resolve_rosdep_keys.sh Signed-off-by: Yutaka Kondo <[email protected]> * style(pre-commit): autofix * update .dockerignore Signed-off-by: Yutaka Kondo <[email protected]> * chmod +x Signed-off-by: Yutaka Kondo <[email protected]> * fix location Signed-off-by: Yutaka Kondo <[email protected]> * remove rosdep update Signed-off-by: Yutaka Kondo <[email protected]> * run build_and_clean.sh Signed-off-by: Yutaka Kondo <[email protected]> * style(pre-commit): autofix --------- Signed-off-by: Yutaka Kondo <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * refactor(docker): replace the multiple `rm -rf` commands with `cleanup_system.sh`. (autowarefoundation#5426) * fix(.github): fix `target-image` (autowarefoundation#5428) fix target-image Signed-off-by: Yutaka Kondo <[email protected]> * chore: remove tvm_utility artifacts from Ansible tasks (autowarefoundation#5427) Signed-off-by: Esteve Fernandez <[email protected]> --------- Signed-off-by: Yutaka Kondo <[email protected]> Signed-off-by: Chengyong Lin <[email protected]> Signed-off-by: Shintaro Sakoda <[email protected]> Signed-off-by: Esteve Fernandez <[email protected]> Co-authored-by: chgyg <[email protected]> Co-authored-by: SakodaShintaro <[email protected]> Co-authored-by: SakodaShintaro <[email protected]> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Esteve Fernandez <[email protected]>
youtalk · Nov 14, 2024 · 3bf9a82 · 3bf9a82
1 parent a151e80
commit 3bf9a82
Show file tree

Hide file tree

Showing 13 changed files with 647 additions and 597 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -9,6 +9,7 @@
 # Ignore Docker files
 docker
 !docker/etc
+!docker/scripts
 
 # Ignore a part of files under src
 src/**/.*

diff --git a/.github/actions/docker-build-and-push-cuda/action.yaml b/.github/actions/docker-build-and-push-cuda/action.yaml
@@ -1,20 +1,33 @@
 name: docker-build-and-push-cuda
-description: Workflow do build and push CUDA images to registry.
+description: Composite action to build and push CUDA images to registry.
 
 inputs:
   platform:
-    description: ""
+    description: Target platform.
     required: true
-  bake-target:
-    description: ""
+  target-image:
+    description: Target docker image name in the registry.
     required: true
   build-args:
-    description: ""
+    description: Additional build args.
     required: false
 
 runs:
   using: composite
   steps:
+    - name: Install jq and vcstool
+      run: |
+        sudo apt-get -y update
+        sudo apt-get -y install jq python3-pip
+        pip install --no-cache-dir vcstool
+      shell: bash
+
+    - name: Run vcs import
+      run: |
+        mkdir src
+        vcs import src < autoware.repos
+      shell: bash
+
     - name: Setup Docker Buildx
       uses: docker/setup-buildx-action@v3
 
@@ -51,11 +64,23 @@ runs:
       run: echo "date=$(date +'%Y%m%d')" >> $GITHUB_OUTPUT
       shell: bash
 
+    - name: Docker meta for autoware:base-cuda
+      id: meta-base-cuda
+      uses: docker/metadata-action@v5
+      with:
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
+        tags: |
+          type=raw,value=base-cuda-${{ inputs.platform }}
+          type=raw,value=base-cuda-${{ steps.date.outputs.date }}-${{ inputs.platform }}
+        bake-target: docker-metadata-action-base-cuda
+        flavor: |
+          latest=false
+
     - name: Docker meta for autoware:universe-sensing-perception-devel-cuda
       id: meta-universe-sensing-perception-devel-cuda
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-sensing-perception-devel-cuda-${{ inputs.platform }}
           type=raw,value=universe-sensing-perception-devel-cuda-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -67,7 +92,7 @@ runs:
       id: meta-universe-sensing-perception-cuda
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-sensing-perception-cuda-${{ inputs.platform }}
           type=raw,value=universe-sensing-perception-cuda-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -79,7 +104,7 @@ runs:
       id: meta-universe-devel-cuda
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-devel-cuda-${{ inputs.platform }}
           type=raw,value=universe-devel-cuda-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -91,7 +116,7 @@ runs:
       id: meta-universe-cuda
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-cuda-${{ inputs.platform }}
           type=raw,value=universe-cuda-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -112,6 +137,7 @@ runs:
         push: true
         files: |
           docker/docker-bake-cuda.hcl
+          ${{ steps.meta-base-cuda.outputs.bake-file }}
           ${{ steps.meta-universe-sensing-perception-devel-cuda.outputs.bake-file }}
           ${{ steps.meta-universe-sensing-perception-cuda.outputs.bake-file }}
           ${{ steps.meta-universe-devel-cuda.outputs.bake-file }}

diff --git a/.github/actions/docker-build-and-push/action.yaml b/.github/actions/docker-build-and-push/action.yaml
@@ -1,15 +1,15 @@
 name: docker-build-and-push
-description: ""
+description: Composite action to build and push non CUDA images to registry.
 
 inputs:
   platform:
-    description: ""
+    description: Target platform.
     required: true
-  bake-target:
-    description: ""
+  target-image:
+    description: Target docker image name in the registry.
     required: true
   build-args:
-    description: ""
+    description: Additional build args.
     required: false
 
 runs:
@@ -68,7 +68,7 @@ runs:
       id: meta-base
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=base-${{ inputs.platform }}
           type=raw,value=base-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -80,7 +80,7 @@ runs:
       id: meta-core-devel
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=core-devel-${{ inputs.platform }}
           type=raw,value=core-devel-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -92,7 +92,7 @@ runs:
       id: meta-universe-sensing-perception-devel
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-sensing-perception-devel-${{ inputs.platform }}
           type=raw,value=universe-sensing-perception-devel-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -104,7 +104,7 @@ runs:
       id: meta-universe-sensing-perception
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-sensing-perception-${{ inputs.platform }}
           type=raw,value=universe-sensing-perception-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -116,7 +116,7 @@ runs:
       id: meta-universe-localization-mapping-devel
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-localization-mapping-devel-${{ inputs.platform }}
           type=raw,value=universe-localization-mapping-devel-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -128,7 +128,7 @@ runs:
       id: meta-universe-localization-mapping
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-localization-mapping-${{ inputs.platform }}
           type=raw,value=universe-localization-mapping-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -140,7 +140,7 @@ runs:
       id: meta-universe-planning-control-devel
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-planning-control-devel-${{ inputs.platform }}
           type=raw,value=universe-planning-control-devel-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -152,7 +152,7 @@ runs:
       id: meta-universe-planning-control
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-planning-control-${{ inputs.platform }}
           type=raw,value=universe-planning-control-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -164,7 +164,7 @@ runs:
       id: meta-universe-vehicle-system-devel
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-vehicle-system-devel-${{ inputs.platform }}
           type=raw,value=universe-vehicle-system-devel-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -176,7 +176,7 @@ runs:
       id: meta-universe-vehicle-system
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-vehicle-system-${{ inputs.platform }}
           type=raw,value=universe-vehicle-system-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -188,7 +188,7 @@ runs:
       id: meta-universe-devel
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-devel-${{ inputs.platform }}
           type=raw,value=universe-devel-${{ steps.date.outputs.date }}-${{ inputs.platform }}
@@ -200,7 +200,7 @@ runs:
       id: meta-universe
       uses: docker/metadata-action@v5
       with:
-        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.bake-target }}
+        images: ghcr.io/${{ github.repository_owner }}/${{ inputs.target-image }}
         tags: |
           type=raw,value=universe-${{ inputs.platform }}
           type=raw,value=universe-${{ steps.date.outputs.date }}-${{ inputs.platform }}

diff --git a/.github/actions/docker-build/action.yaml b/.github/actions/docker-build/action.yaml
@@ -1,16 +1,16 @@
 name: docker-build
-description: ""
+description: Composite action to build images only.
 
 inputs:
   platform:
-    description: ""
+    description: Target platform.
     required: true
   cache-tag-suffix:
-    description: ""
+    description: Suffix of the target cache tag.
     required: true
   build-args:
-    description: ""
-    required: true
+    description: Additional build args.
+    required: false
 
 runs:
   using: composite

diff --git a/.github/workflows/docker-build-and-push-arm64.yaml b/.github/workflows/docker-build-and-push-arm64.yaml
@@ -57,7 +57,7 @@ jobs:
         uses: ./.github/actions/docker-build-and-push
         with:
           platform: arm64
-          bake-target: autoware
+          target-image: autoware
           build-args: |
             *.platform=linux/arm64
             *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }}
@@ -67,14 +67,44 @@ jobs:
             *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:arm64-main
             *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:arm64-${{ github.ref_name }},mode=max
 
+      - name: Show disk space
+        if: always()
+        run: |
+          df -h
+
+  docker-build-and-push-cuda:
+    needs: [load-env, docker-build-and-push]
+    runs-on: buildjet-16vcpu-ubuntu-2204-arm
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set git config
+        uses: autowarefoundation/autoware-github-actions/set-git-config@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v45
+        with:
+          files: |
+            *.env
+            *.repos
+            .github/actions/docker-build-and-push/action.yaml
+            .github/workflows/docker-build-and-push*.yaml
+            ansible-galaxy-requirements.yaml
+            ansible/**
+            docker/**
+
       - name: Build 'Autoware' with CUDA
         if: ${{ steps.changed-files.outputs.any_changed == 'true' ||
           github.event_name == 'workflow_dispatch' ||
           (github.event_name == 'push' && github.ref_type == 'tag') }}
         uses: ./.github/actions/docker-build-and-push-cuda
         with:
           platform: arm64
-          bake-target: autoware
+          target-image: autoware
           build-args: |
             *.platform=linux/arm64
             *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }}

diff --git a/.github/workflows/docker-build-and-push.yaml b/.github/workflows/docker-build-and-push.yaml
@@ -57,7 +57,7 @@ jobs:
         uses: ./.github/actions/docker-build-and-push
         with:
           platform: amd64
-          bake-target: autoware
+          target-image: autoware
           build-args: |
             *.platform=linux/amd64
             *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }}
@@ -67,14 +67,44 @@ jobs:
             *.cache-from=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:amd64-main
             *.cache-to=type=registry,ref=ghcr.io/${{ github.repository }}-buildcache:amd64-${{ github.ref_name }},mode=max
 
+      - name: Show disk space
+        if: always()
+        run: |
+          df -h
+
+  docker-build-and-push-cuda:
+    needs: [load-env, docker-build-and-push]
+    runs-on: buildjet-16vcpu-ubuntu-2204
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set git config
+        uses: autowarefoundation/autoware-github-actions/set-git-config@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/changed-files@v45
+        with:
+          files: |
+            *.env
+            *.repos
+            .github/actions/docker-build-and-push/action.yaml
+            .github/workflows/docker-build-and-push*.yaml
+            ansible-galaxy-requirements.yaml
+            ansible/**
+            docker/**
+
       - name: Build 'Autoware' with CUDA
         if: ${{ steps.changed-files.outputs.any_changed == 'true' ||
           github.event_name == 'workflow_dispatch' ||
           (github.event_name == 'push' && github.ref_type == 'tag') }}
         uses: ./.github/actions/docker-build-and-push-cuda
         with:
           platform: amd64
-          bake-target: autoware
+          target-image: autoware
           build-args: |
             *.platform=linux/amd64
             *.args.ROS_DISTRO=${{ needs.load-env.outputs.rosdistro }}

diff --git a/ansible/roles/artifacts/tasks/main.yaml b/ansible/roles/artifacts/tasks/main.yaml
@@ -402,29 +402,3 @@
     dest: "{{ data_dir }}/traffic_light_fine_detector/tlr_labels.txt"
     mode: "644"
     checksum: sha256:a2a91f5fe9c2e68e3e3647a272bb9bb25cd07631a1990a3fb15efddce7691131
-
-# tvm_utility
-- name: Create tvm_utility/models directory inside {{ data_dir }}
-  ansible.builtin.file:
-    path: "{{ data_dir }}/tvm_utility/models"
-    mode: "755"
-    state: directory
-
-- name: Download yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz
-  become: true
-  ansible.builtin.get_url:
-    url: https://autoware-modelzoo.s3.us-east-2.amazonaws.com/models/3.0.0-20221221/yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz
-    dest: "{{ data_dir }}/tvm_utility/yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz"
-    mode: "644"
-    checksum: sha256:66b3ca668e577393b657fbe1ed626538d89ca3adccd5862de6c7fa190238dbca
-
-- name: Create yolo_v2_tiny folder in tvm_utility/models of {{ data_dir }}
-  ansible.builtin.file:
-    path: "{{ data_dir }}/tvm_utility/models/yolo_v2_tiny"
-    mode: "755"
-    state: directory
-
-- name: Extract yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz
-  ansible.builtin.unarchive:
-    src: "{{ data_dir }}/tvm_utility/yolo_v2_tiny-x86_64-llvm-3.0.0-20221221.tar.gz"
-    dest: "{{ data_dir }}/tvm_utility/models/yolo_v2_tiny"