From e2b4d34804a26146bbfb7552e8f345574239804f Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Wed, 5 Jun 2024 16:02:41 +0900 Subject: [PATCH 01/44] ccache -s Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3f0a53c0a84..d0ec61c9837 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -88,6 +88,7 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src +RUN ccache -s RUN --mount=type=cache,target=${CCACHE_DIR} \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release --cmake-args \ @@ -98,6 +99,7 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \ && find /autoware/install -type d -exec chmod 777 {} \; \ && chmod -R 777 /var/tmp/ccache \ && rm -rf /autoware/build /autoware/src +RUN ccache -s CMD ["/bin/bash"] From f610c191d68509c8de87fd9a6e313fd193f3a787 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 05:49:01 +0900 Subject: [PATCH 02/44] empty Signed-off-by: Yutaka Kondo From a4818ba9815aee52cd6f8ef563f3a92d65d4557a Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 06:42:47 +0900 Subject: [PATCH 03/44] update ccache Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index d0ec61c9837..eb138535f55 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -66,8 +66,6 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG SETUP_ARGS ENV CCACHE_DIR="/var/tmp/ccache" -ENV CC="/usr/lib/ccache/gcc" -ENV CXX="/usr/lib/ccache/g++" # cspell: ignore libcu libnv # Set up development environment @@ -92,6 +90,8 @@ RUN ccache -s RUN --mount=type=cache,target=${CCACHE_DIR} \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release --cmake-args \ + " -DCMAKE_C_COMPILER_LAUNCHER=ccache", + " -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" " -Wno-dev" \ " --no-warn-unused-cli" \ -DCMAKE_BUILD_TYPE=Release \ From 5494ef6ee049fe70a39c561b3ea406fe2dde8da8 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 06:50:59 +0900 Subject: [PATCH 04/44] fix flags Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index eb138535f55..172c91d8803 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -90,10 +90,10 @@ RUN ccache -s RUN --mount=type=cache,target=${CCACHE_DIR} \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release --cmake-args \ - " -DCMAKE_C_COMPILER_LAUNCHER=ccache", - " -DCMAKE_CXX_COMPILER_LAUNCHER=ccache" " -Wno-dev" \ " --no-warn-unused-cli" \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ && find /autoware/install -type d -exec chmod 777 {} \; \ From 160f822e08e93e7b73e2e70ca4d22fb98df2c213 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 09:20:01 +0900 Subject: [PATCH 05/44] change dir Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 172c91d8803..32a2704b5b2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,7 +65,7 @@ FROM base as prebuilt SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG SETUP_ARGS -ENV CCACHE_DIR="/var/tmp/ccache" +ENV CCACHE_DIR="/autoware/ccache" # cspell: ignore libcu libnv # Set up development environment From 063769c087f6e2298b292c1af3ed93720ca1295c Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 09:24:21 +0900 Subject: [PATCH 06/44] du Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 32a2704b5b2..e6c668f5013 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -86,7 +86,7 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src -RUN ccache -s +RUN ccache -s && du -sh ${CCACHE_DIR} RUN --mount=type=cache,target=${CCACHE_DIR} \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release --cmake-args \ @@ -99,7 +99,7 @@ RUN --mount=type=cache,target=${CCACHE_DIR} \ && find /autoware/install -type d -exec chmod 777 {} \; \ && chmod -R 777 /var/tmp/ccache \ && rm -rf /autoware/build /autoware/src -RUN ccache -s +RUN ccache -s && du -sh ${CCACHE_DIR} CMD ["/bin/bash"] From 7b0ba79290b0b9a25ee29dc3fa966dc528717f7e Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 10:06:43 +0900 Subject: [PATCH 07/44] mkdir Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e6c668f5013..b5338fc7236 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,7 +65,7 @@ FROM base as prebuilt SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG SETUP_ARGS -ENV CCACHE_DIR="/autoware/ccache" +ENV CCACHE_DIR="/.cache/ccache" # cspell: ignore libcu libnv # Set up development environment @@ -86,7 +86,7 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src -RUN ccache -s && du -sh ${CCACHE_DIR} +RUN mkdir -p ${CCACHE_DIR} && ccache -s && du -sh ${CCACHE_DIR} RUN --mount=type=cache,target=${CCACHE_DIR} \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release --cmake-args \ From d8eb1b4528fe5b71dbf9ad6b6c72a1435ced751b Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 10:53:29 +0900 Subject: [PATCH 08/44] remove args Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b5338fc7236..6362e6b564c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -90,8 +90,6 @@ RUN mkdir -p ${CCACHE_DIR} && ccache -s && du -sh ${CCACHE_DIR} RUN --mount=type=cache,target=${CCACHE_DIR} \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release --cmake-args \ - " -Wno-dev" \ - " --no-warn-unused-cli" \ -DCMAKE_C_COMPILER_LAUNCHER=ccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ -DCMAKE_BUILD_TYPE=Release \ From 199ab7464ac22d2257e6c260a54eed67282e9528 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 11:12:53 +0900 Subject: [PATCH 09/44] remove duplicate release flag Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 6362e6b564c..3f21f4d7943 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -89,11 +89,11 @@ COPY --from=src-imported /autoware/src /autoware/src RUN mkdir -p ${CCACHE_DIR} && ccache -s && du -sh ${CCACHE_DIR} RUN --mount=type=cache,target=${CCACHE_DIR} \ source /opt/ros/"$ROS_DISTRO"/setup.bash \ - && colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release --cmake-args \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + && colcon build --cmake-args \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ && find /autoware/install -type d -exec chmod 777 {} \; \ && chmod -R 777 /var/tmp/ccache \ && rm -rf /autoware/build /autoware/src From 5f6d931a4dd8f3e95f743df831121da88909cbbf Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 11:55:20 +0900 Subject: [PATCH 10/44] remove --mount=type=cache --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 3f21f4d7943..055bc18e4bc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -87,8 +87,7 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src RUN mkdir -p ${CCACHE_DIR} && ccache -s && du -sh ${CCACHE_DIR} -RUN --mount=type=cache,target=${CCACHE_DIR} \ - source /opt/ros/"$ROS_DISTRO"/setup.bash \ +RUN source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ From ada50851ec2ed728f926091ddc9937a8ee2dd11d Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 12:10:37 +0900 Subject: [PATCH 11/44] use mixin instead Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 055bc18e4bc..7d2d3870105 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,7 +65,6 @@ FROM base as prebuilt SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG SETUP_ARGS -ENV CCACHE_DIR="/.cache/ccache" # cspell: ignore libcu libnv # Set up development environment @@ -86,17 +85,17 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src -RUN mkdir -p ${CCACHE_DIR} && ccache -s && du -sh ${CCACHE_DIR} -RUN source /opt/ros/"$ROS_DISTRO"/setup.bash \ +RUN ccache -s +RUN --mount=type=cache,target=/root/.ccache \ + source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + --mixin ccache \ && find /autoware/install -type d -exec chmod 777 {} \; \ && chmod -R 777 /var/tmp/ccache \ && rm -rf /autoware/build /autoware/src -RUN ccache -s && du -sh ${CCACHE_DIR} +RUN ccache -s CMD ["/bin/bash"] From 1f3bcc068d00e70add671ca49d3414b0323e0e62 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 13:20:21 +0900 Subject: [PATCH 12/44] ccache inline Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 7d2d3870105..d3a3594142f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -85,17 +85,17 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src -RUN ccache -s RUN --mount=type=cache,target=/root/.ccache \ - source /opt/ros/"$ROS_DISTRO"/setup.bash \ + ccache -s \ + && source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ --mixin ccache \ && find /autoware/install -type d -exec chmod 777 {} \; \ && chmod -R 777 /var/tmp/ccache \ - && rm -rf /autoware/build /autoware/src -RUN ccache -s + && rm -rf /autoware/build /autoware/src \ + && ccache -s CMD ["/bin/bash"] From 711c4f8ff364f8ed43bfa235d88b14b6fa324a13 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 15:06:19 +0900 Subject: [PATCH 13/44] cache .ccache Signed-off-by: Yutaka Kondo --- .github/workflows/docker-build-and-push-main.yaml | 8 ++++++++ docker/Dockerfile | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index ef009a751bb..6fc172e8472 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -58,6 +58,14 @@ jobs: - name: Check out repository uses: actions/checkout@v4 + - name: Cache ccache + uses: actions/cache@v4 + with: + path: .ccache + key: ccache-${{ matrix.lib_dir }}-${{ matrix.name }} + restore-keys: | + ccache-${{ matrix.lib_dir }} + - name: Install vcstool run: | sudo apt-get -y update diff --git a/docker/Dockerfile b/docker/Dockerfile index d3a3594142f..ff979be0176 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -85,7 +85,7 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src -RUN --mount=type=cache,target=/root/.ccache \ +RUN --mount=type=bind,source=.ccache,target=/root/.ccache \ ccache -s \ && source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args \ From 684a698cf608fd78013ad63a2dd550bb628f37c5 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 15:12:52 +0900 Subject: [PATCH 14/44] cache .ccache Signed-off-by: Yutaka Kondo --- .github/workflows/build-main-self-hosted.yaml | 8 ++++++++ .github/workflows/build-main.yaml | 8 ++++++++ .../workflows/docker-build-and-push-main-self-hosted.yaml | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/.github/workflows/build-main-self-hosted.yaml b/.github/workflows/build-main-self-hosted.yaml index c0fbe3f21b6..c43df3afd0e 100644 --- a/.github/workflows/build-main-self-hosted.yaml +++ b/.github/workflows/build-main-self-hosted.yaml @@ -44,6 +44,14 @@ jobs: - name: Check out repository uses: actions/checkout@v4 + - name: Cache ccache + uses: actions/cache@v4 + with: + path: .ccache + key: ccache-${{ matrix.lib_dir }}-${{ matrix.name }} + restore-keys: | + ccache-${{ matrix.lib_dir }} + - name: Install vcstool run: | sudo apt-get -y update diff --git a/.github/workflows/build-main.yaml b/.github/workflows/build-main.yaml index 5beada55314..ba47b00f139 100644 --- a/.github/workflows/build-main.yaml +++ b/.github/workflows/build-main.yaml @@ -39,6 +39,14 @@ jobs: - name: Check out repository uses: actions/checkout@v4 + - name: Cache ccache + uses: actions/cache@v4 + with: + path: .ccache + key: ccache-${{ matrix.lib_dir }}-${{ matrix.name }} + restore-keys: | + ccache-${{ matrix.lib_dir }} + - name: Install vcstool run: | sudo apt-get -y update diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 772ba389e82..9e5dc936f88 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -56,6 +56,14 @@ jobs: - name: Check out repository uses: actions/checkout@v4 + - name: Cache ccache + uses: actions/cache@v4 + with: + path: .ccache + key: ccache-${{ matrix.lib_dir }}-${{ matrix.name }} + restore-keys: | + ccache-${{ matrix.lib_dir }} + - name: Install vcstool run: | sudo apt-get -y update From 7a327980bd0b450a5da7ef643d737134e991ed0e Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 15:14:17 +0900 Subject: [PATCH 15/44] remove chmod Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index ff979be0176..be2bb36a8b2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -93,7 +93,6 @@ RUN --mount=type=bind,source=.ccache,target=/root/.ccache \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ --mixin ccache \ && find /autoware/install -type d -exec chmod 777 {} \; \ - && chmod -R 777 /var/tmp/ccache \ && rm -rf /autoware/build /autoware/src \ && ccache -s From 074c51bb28807779f70becb3899be3451baac824 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 15:18:17 +0900 Subject: [PATCH 16/44] add .ccache Signed-off-by: Yutaka Kondo --- .ccache/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .ccache/.gitkeep diff --git a/.ccache/.gitkeep b/.ccache/.gitkeep new file mode 100644 index 00000000000..e69de29bb2d From 14c751186cfc1d66eed79ec40d680ffe3dadd714 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 15:24:59 +0900 Subject: [PATCH 17/44] revert options Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index be2bb36a8b2..bf074991a81 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -91,6 +91,8 @@ RUN --mount=type=bind,source=.ccache,target=/root/.ccache \ && colcon build --cmake-args \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + " -Wno-dev" \ + " --no-warn-unused-cli" \ --mixin ccache \ && find /autoware/install -type d -exec chmod 777 {} \; \ && rm -rf /autoware/build /autoware/src \ From 9dfc62db83a32ac106581dca383a51654fdfafbb Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 15:25:55 +0900 Subject: [PATCH 18/44] add rw Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index bf074991a81..dcc184d3b64 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -85,7 +85,7 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src -RUN --mount=type=bind,source=.ccache,target=/root/.ccache \ +RUN --mount=type=bind,source=.ccache,target=/root/.ccache,rw \ ccache -s \ && source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args \ From 9b0a77abf8bb75dbdf8a22cd719e2da79aa8a6e1 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 16:05:33 +0900 Subject: [PATCH 19/44] check ccache Signed-off-by: Yutaka Kondo --- .../docker-build-and-push-main-self-hosted.yaml | 10 ++++++++++ .github/workflows/docker-build-and-push-main.yaml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 9e5dc936f88..4dcebbcfff1 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -64,6 +64,11 @@ jobs: restore-keys: | ccache-${{ matrix.lib_dir }} + - name: Check ccache + run: | + du -sh .ccache + ls -R .ccache + - name: Install vcstool run: | sudo apt-get -y update @@ -99,6 +104,11 @@ jobs: tag-prefix: ${{ needs.load-env.outputs.rosdistro }} allow-push: true + - name: Check ccache + run: | + du -sh .ccache + ls -R .ccache + - name: Show disk space run: | df -h diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 6fc172e8472..cbd4aea2bc0 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -66,6 +66,11 @@ jobs: restore-keys: | ccache-${{ matrix.lib_dir }} + - name: Check ccache + run: | + du -sh .ccache + ls -R .ccache + - name: Install vcstool run: | sudo apt-get -y update @@ -99,6 +104,11 @@ jobs: tag-prefix: ${{ needs.load-env.outputs.rosdistro }} allow-push: true + - name: Check ccache + run: | + du -sh .ccache + ls -R .ccache + - name: Show disk space run: | df -h From 8b2141740bd9d1a6b5271f02b0f2004d4e4da709 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 16:43:40 +0900 Subject: [PATCH 20/44] mkdir .ccache instead Signed-off-by: Yutaka Kondo --- .ccache/.gitkeep | 0 .github/workflows/docker-build-and-push-main-self-hosted.yaml | 1 + .github/workflows/docker-build-and-push-main.yaml | 1 + 3 files changed, 2 insertions(+) delete mode 100644 .ccache/.gitkeep diff --git a/.ccache/.gitkeep b/.ccache/.gitkeep deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 4dcebbcfff1..9907e46356b 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -66,6 +66,7 @@ jobs: - name: Check ccache run: | + mkdir -p .ccache du -sh .ccache ls -R .ccache diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index cbd4aea2bc0..19612410ba9 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -68,6 +68,7 @@ jobs: - name: Check ccache run: | + mkdir -p .ccache du -sh .ccache ls -R .ccache From d99a5db5fc5df3b19d34bf24743e6aa67f828839 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 16:49:15 +0900 Subject: [PATCH 21/44] ignore .ccache Signed-off-by: Yutaka Kondo --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 024b426a3b1..0f3240ce591 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ install/ log/ /src/ +# C++ +.ccache + # Python *.pyc From c8fae8175a0f2d6db7083c8bf81dacb8de543bf1 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 17:28:21 +0900 Subject: [PATCH 22/44] chmod Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index dcc184d3b64..05ff04dfff4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -95,6 +95,7 @@ RUN --mount=type=bind,source=.ccache,target=/root/.ccache,rw \ " --no-warn-unused-cli" \ --mixin ccache \ && find /autoware/install -type d -exec chmod 777 {} \; \ + && chmod -R 777 /root/.ccache \ && rm -rf /autoware/build /autoware/src \ && ccache -s From 3e69bc034dbb51f25e0d4066fc2184821145844a Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 19:06:55 +0900 Subject: [PATCH 23/44] set CCACHE_DIR Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 05ff04dfff4..9b3bbe80220 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,6 +65,7 @@ FROM base as prebuilt SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG SETUP_ARGS +ENV CCACHE_DIR="/var/tmp/ccache" # cspell: ignore libcu libnv # Set up development environment @@ -85,8 +86,8 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src -RUN --mount=type=bind,source=.ccache,target=/root/.ccache,rw \ - ccache -s \ +RUN --mount=type=bind,source=.ccache,target=${CCACHE_DIR},rw \ + ccache -s && du -sh ${CCACHE_DIR} \ && source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args \ -DCMAKE_BUILD_TYPE=Release \ @@ -95,9 +96,9 @@ RUN --mount=type=bind,source=.ccache,target=/root/.ccache,rw \ " --no-warn-unused-cli" \ --mixin ccache \ && find /autoware/install -type d -exec chmod 777 {} \; \ - && chmod -R 777 /root/.ccache \ + && chmod -R 777 ${CCACHE_DIR} \ && rm -rf /autoware/build /autoware/src \ - && ccache -s + && ccache -s && du -sh ${CCACHE_DIR} CMD ["/bin/bash"] From 58d8522a48a0107962dade7e3e4e8354fdb652b5 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 20:06:05 +0900 Subject: [PATCH 24/44] test Signed-off-by: Yutaka Kondo --- .github/workflows/docker-build-and-push-main-self-hosted.yaml | 1 + .github/workflows/docker-build-and-push-main.yaml | 1 + docker/Dockerfile | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 9907e46356b..a5544180aaf 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -67,6 +67,7 @@ jobs: - name: Check ccache run: | mkdir -p .ccache + echo hello > .cache/hello.txt du -sh .ccache ls -R .ccache diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 19612410ba9..55e95064bb2 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -69,6 +69,7 @@ jobs: - name: Check ccache run: | mkdir -p .ccache + echo hello > .cache/hello.txt du -sh .ccache ls -R .ccache diff --git a/docker/Dockerfile b/docker/Dockerfile index 9b3bbe80220..b682f62e271 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -87,7 +87,7 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src RUN --mount=type=bind,source=.ccache,target=${CCACHE_DIR},rw \ - ccache -s && du -sh ${CCACHE_DIR} \ + ccache -s && du -sh ${CCACHE_DIR} && ls -R ${CCACHE_DIR} \ && source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args \ -DCMAKE_BUILD_TYPE=Release \ @@ -98,7 +98,7 @@ RUN --mount=type=bind,source=.ccache,target=${CCACHE_DIR},rw \ && find /autoware/install -type d -exec chmod 777 {} \; \ && chmod -R 777 ${CCACHE_DIR} \ && rm -rf /autoware/build /autoware/src \ - && ccache -s && du -sh ${CCACHE_DIR} + && ccache -s && du -sh ${CCACHE_DIR} && ls -R ${CCACHE_DIR} CMD ["/bin/bash"] From f1d8645162dc1d49cafc9f8f9b704d4107a746ff Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 20:10:05 +0900 Subject: [PATCH 25/44] typo Signed-off-by: Yutaka Kondo --- .github/workflows/docker-build-and-push-main-self-hosted.yaml | 2 +- .github/workflows/docker-build-and-push-main.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index a5544180aaf..6a84e33cb37 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -67,7 +67,7 @@ jobs: - name: Check ccache run: | mkdir -p .ccache - echo hello > .cache/hello.txt + echo hello > .ccache/hello.txt du -sh .ccache ls -R .ccache diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 55e95064bb2..a93d60ae791 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -69,7 +69,7 @@ jobs: - name: Check ccache run: | mkdir -p .ccache - echo hello > .cache/hello.txt + echo hello > .ccache/hello.txt du -sh .ccache ls -R .ccache From 68d5598217c08663285ac5b5432d52f2dc97892a Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 21:04:55 +0900 Subject: [PATCH 26/44] refine cache key Signed-off-by: Yutaka Kondo --- .github/workflows/build-main-self-hosted.yaml | 6 ++++-- .github/workflows/build-main.yaml | 6 ++++-- .../workflows/docker-build-and-push-main-self-hosted.yaml | 6 ++++-- .github/workflows/docker-build-and-push-main.yaml | 6 ++++-- 4 files changed, 16 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build-main-self-hosted.yaml b/.github/workflows/build-main-self-hosted.yaml index c43df3afd0e..84e5e88f6d0 100644 --- a/.github/workflows/build-main-self-hosted.yaml +++ b/.github/workflows/build-main-self-hosted.yaml @@ -48,9 +48,11 @@ jobs: uses: actions/cache@v4 with: path: .ccache - key: ccache-${{ matrix.lib_dir }}-${{ matrix.name }} + key: cache-arm64-${{ matrix.name }}-${{ github.ref_name }} restore-keys: | - ccache-${{ matrix.lib_dir }} + cache-arm64-${{ matrix.name }}-main + cache-arm64-${{ matrix.name }}- + cache-arm64- - name: Install vcstool run: | diff --git a/.github/workflows/build-main.yaml b/.github/workflows/build-main.yaml index ba47b00f139..067fe9b9756 100644 --- a/.github/workflows/build-main.yaml +++ b/.github/workflows/build-main.yaml @@ -43,9 +43,11 @@ jobs: uses: actions/cache@v4 with: path: .ccache - key: ccache-${{ matrix.lib_dir }}-${{ matrix.name }} + key: cache-amd64-${{ matrix.name }}-${{ github.ref_name }} restore-keys: | - ccache-${{ matrix.lib_dir }} + cache-amd64-${{ matrix.name }}-main + cache-amd64-${{ matrix.name }}- + cache-amd64- - name: Install vcstool run: | diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 6a84e33cb37..06085954409 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -60,9 +60,11 @@ jobs: uses: actions/cache@v4 with: path: .ccache - key: ccache-${{ matrix.lib_dir }}-${{ matrix.name }} + key: cache-arm64-${{ matrix.name }}-${{ github.ref_name }} restore-keys: | - ccache-${{ matrix.lib_dir }} + cache-arm64-${{ matrix.name }}-main + cache-arm64-${{ matrix.name }}- + cache-arm64- - name: Check ccache run: | diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index a93d60ae791..9db8cf6fcc0 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -62,9 +62,11 @@ jobs: uses: actions/cache@v4 with: path: .ccache - key: ccache-${{ matrix.lib_dir }}-${{ matrix.name }} + key: cache-amd64-${{ matrix.name }}-${{ github.ref_name }} restore-keys: | - ccache-${{ matrix.lib_dir }} + cache-amd64-${{ matrix.name }}-main + cache-amd64-${{ matrix.name }}- + cache-amd64- - name: Check ccache run: | From 4212cfa6cae917c088224626bf8f1e70ed3c3d10 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 21:26:57 +0900 Subject: [PATCH 27/44] change dir Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b682f62e271..e7d7fc98f2a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,7 +65,7 @@ FROM base as prebuilt SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG SETUP_ARGS -ENV CCACHE_DIR="/var/tmp/ccache" +ENV CCACHE_DIR="/root/.ccache" # cspell: ignore libcu libnv # Set up development environment @@ -96,7 +96,6 @@ RUN --mount=type=bind,source=.ccache,target=${CCACHE_DIR},rw \ " --no-warn-unused-cli" \ --mixin ccache \ && find /autoware/install -type d -exec chmod 777 {} \; \ - && chmod -R 777 ${CCACHE_DIR} \ && rm -rf /autoware/build /autoware/src \ && ccache -s && du -sh ${CCACHE_DIR} && ls -R ${CCACHE_DIR} From 6b26991a0ce8123f971469fe70be9253a1f4d548 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 21:34:32 +0900 Subject: [PATCH 28/44] change source dir Signed-off-by: Yutaka Kondo --- .../docker-build-and-push-main-self-hosted.yaml | 13 ++++++------- .github/workflows/docker-build-and-push-main.yaml | 13 ++++++------- docker/Dockerfile | 2 +- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 06085954409..1be0d1869c1 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -59,7 +59,7 @@ jobs: - name: Cache ccache uses: actions/cache@v4 with: - path: .ccache + path: /var/tmp/ccache key: cache-arm64-${{ matrix.name }}-${{ github.ref_name }} restore-keys: | cache-arm64-${{ matrix.name }}-main @@ -68,10 +68,9 @@ jobs: - name: Check ccache run: | - mkdir -p .ccache - echo hello > .ccache/hello.txt - du -sh .ccache - ls -R .ccache + mkdir -p /var/tmp/ccache + du -sh /var/tmp/ccache + ls -R /var/tmp/ccache - name: Install vcstool run: | @@ -110,8 +109,8 @@ jobs: - name: Check ccache run: | - du -sh .ccache - ls -R .ccache + du -sh /var/tmp/ccache + ls -R /var/tmp/ccache - name: Show disk space run: | diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 9db8cf6fcc0..9b18249a299 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -61,7 +61,7 @@ jobs: - name: Cache ccache uses: actions/cache@v4 with: - path: .ccache + path: /var/tmp/ccache key: cache-amd64-${{ matrix.name }}-${{ github.ref_name }} restore-keys: | cache-amd64-${{ matrix.name }}-main @@ -70,10 +70,9 @@ jobs: - name: Check ccache run: | - mkdir -p .ccache - echo hello > .ccache/hello.txt - du -sh .ccache - ls -R .ccache + mkdir -p /var/tmp/ccache + du -sh /var/tmp/ccache + ls -R /var/tmp/ccache - name: Install vcstool run: | @@ -110,8 +109,8 @@ jobs: - name: Check ccache run: | - du -sh .ccache - ls -R .ccache + du -sh /var/tmp/ccache + ls -R /var/tmp/ccache - name: Show disk space run: | diff --git a/docker/Dockerfile b/docker/Dockerfile index e7d7fc98f2a..190d1587170 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -86,7 +86,7 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src -RUN --mount=type=bind,source=.ccache,target=${CCACHE_DIR},rw \ +RUN --mount=type=bind,source=${CCACHE_DIR},target=${CCACHE_DIR},rw \ ccache -s && du -sh ${CCACHE_DIR} && ls -R ${CCACHE_DIR} \ && source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args \ From 57f68c9cbc5038fd1fcc53ae42ddea7b9bb12a15 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 21:39:51 +0900 Subject: [PATCH 29/44] fix dir Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 190d1587170..8c967d4366b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,7 +65,7 @@ FROM base as prebuilt SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG SETUP_ARGS -ENV CCACHE_DIR="/root/.ccache" +ENV CCACHE_DIR="/var/tmp/ccache" # cspell: ignore libcu libnv # Set up development environment From faaa6fc950d67e0221eccee0c2ec01553a1d61f8 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Thu, 6 Jun 2024 21:50:53 +0900 Subject: [PATCH 30/44] rename dir Signed-off-by: Yutaka Kondo --- .../docker-build-and-push-main-self-hosted.yaml | 12 ++++++------ .github/workflows/docker-build-and-push-main.yaml | 12 ++++++------ docker/Dockerfile | 4 ++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index 1be0d1869c1..bcf0437ffb3 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -59,7 +59,7 @@ jobs: - name: Cache ccache uses: actions/cache@v4 with: - path: /var/tmp/ccache + path: ./ccache key: cache-arm64-${{ matrix.name }}-${{ github.ref_name }} restore-keys: | cache-arm64-${{ matrix.name }}-main @@ -68,9 +68,9 @@ jobs: - name: Check ccache run: | - mkdir -p /var/tmp/ccache - du -sh /var/tmp/ccache - ls -R /var/tmp/ccache + mkdir -p ./ccache + du -sh ./ccache + ls -R ./ccache - name: Install vcstool run: | @@ -109,8 +109,8 @@ jobs: - name: Check ccache run: | - du -sh /var/tmp/ccache - ls -R /var/tmp/ccache + du -sh ./ccache + ls -R ./ccache - name: Show disk space run: | diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 9b18249a299..9f34d967d01 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -61,7 +61,7 @@ jobs: - name: Cache ccache uses: actions/cache@v4 with: - path: /var/tmp/ccache + path: ./ccache key: cache-amd64-${{ matrix.name }}-${{ github.ref_name }} restore-keys: | cache-amd64-${{ matrix.name }}-main @@ -70,9 +70,9 @@ jobs: - name: Check ccache run: | - mkdir -p /var/tmp/ccache - du -sh /var/tmp/ccache - ls -R /var/tmp/ccache + mkdir -p ./ccache + du -sh ./ccache + ls -R ./ccache - name: Install vcstool run: | @@ -109,8 +109,8 @@ jobs: - name: Check ccache run: | - du -sh /var/tmp/ccache - ls -R /var/tmp/ccache + du -sh ./ccache + ls -R ./ccache - name: Show disk space run: | diff --git a/docker/Dockerfile b/docker/Dockerfile index 8c967d4366b..23e5c9726f4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,7 +65,7 @@ FROM base as prebuilt SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG SETUP_ARGS -ENV CCACHE_DIR="/var/tmp/ccache" +ENV CCACHE_DIR="/root/.ccache" # cspell: ignore libcu libnv # Set up development environment @@ -86,7 +86,7 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src -RUN --mount=type=bind,source=${CCACHE_DIR},target=${CCACHE_DIR},rw \ +RUN --mount=type=bind,source=ccache,target=${CCACHE_DIR},rw \ ccache -s && du -sh ${CCACHE_DIR} && ls -R ${CCACHE_DIR} \ && source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args \ From 5d558f628641e5b230290f8a74f1a40dc6f62ba8 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 06:04:40 +0900 Subject: [PATCH 31/44] change dir Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 23e5c9726f4..18bc35ac393 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,7 +65,7 @@ FROM base as prebuilt SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG SETUP_ARGS -ENV CCACHE_DIR="/root/.ccache" +ENV CCACHE_DIR="/root/ccache" # cspell: ignore libcu libnv # Set up development environment @@ -87,7 +87,7 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src RUN --mount=type=bind,source=ccache,target=${CCACHE_DIR},rw \ - ccache -s && du -sh ${CCACHE_DIR} && ls -R ${CCACHE_DIR} \ + ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} \ && source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args \ -DCMAKE_BUILD_TYPE=Release \ @@ -95,9 +95,7 @@ RUN --mount=type=bind,source=ccache,target=${CCACHE_DIR},rw \ " -Wno-dev" \ " --no-warn-unused-cli" \ --mixin ccache \ - && find /autoware/install -type d -exec chmod 777 {} \; \ - && rm -rf /autoware/build /autoware/src \ - && ccache -s && du -sh ${CCACHE_DIR} && ls -R ${CCACHE_DIR} + && ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} CMD ["/bin/bash"] From 9040d19d41a3aeb414748e6a3b14923c34bc2f5a Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 06:41:31 +0900 Subject: [PATCH 32/44] comment out Signed-off-by: Yutaka Kondo --- .../workflows/docker-build-and-push-main.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index 9f34d967d01..f7de667d98d 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -58,15 +58,15 @@ jobs: - name: Check out repository uses: actions/checkout@v4 - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ./ccache - key: cache-amd64-${{ matrix.name }}-${{ github.ref_name }} - restore-keys: | - cache-amd64-${{ matrix.name }}-main - cache-amd64-${{ matrix.name }}- - cache-amd64- + # - name: Cache ccache + # uses: actions/cache@v4 + # with: + # path: ./ccache + # key: cache-amd64-${{ matrix.name }}-${{ github.ref_name }} + # restore-keys: | + # cache-amd64-${{ matrix.name }}-main + # cache-amd64-${{ matrix.name }}- + # cache-amd64- - name: Check ccache run: | From d6575f09d019a58f8d8c9112627fdfd4bb45f889 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 09:33:50 +0900 Subject: [PATCH 33/44] revert to use cache mount Signed-off-by: Yutaka Kondo --- .github/workflows/build-main-self-hosted.yaml | 10 --------- .github/workflows/build-main.yaml | 10 --------- ...ocker-build-and-push-main-self-hosted.yaml | 21 ------------------- .../workflows/docker-build-and-push-main.yaml | 21 ------------------- .gitignore | 3 --- docker/Dockerfile | 4 ++-- 6 files changed, 2 insertions(+), 67 deletions(-) diff --git a/.github/workflows/build-main-self-hosted.yaml b/.github/workflows/build-main-self-hosted.yaml index 84e5e88f6d0..c0fbe3f21b6 100644 --- a/.github/workflows/build-main-self-hosted.yaml +++ b/.github/workflows/build-main-self-hosted.yaml @@ -44,16 +44,6 @@ jobs: - name: Check out repository uses: actions/checkout@v4 - - name: Cache ccache - uses: actions/cache@v4 - with: - path: .ccache - key: cache-arm64-${{ matrix.name }}-${{ github.ref_name }} - restore-keys: | - cache-arm64-${{ matrix.name }}-main - cache-arm64-${{ matrix.name }}- - cache-arm64- - - name: Install vcstool run: | sudo apt-get -y update diff --git a/.github/workflows/build-main.yaml b/.github/workflows/build-main.yaml index 067fe9b9756..5beada55314 100644 --- a/.github/workflows/build-main.yaml +++ b/.github/workflows/build-main.yaml @@ -39,16 +39,6 @@ jobs: - name: Check out repository uses: actions/checkout@v4 - - name: Cache ccache - uses: actions/cache@v4 - with: - path: .ccache - key: cache-amd64-${{ matrix.name }}-${{ github.ref_name }} - restore-keys: | - cache-amd64-${{ matrix.name }}-main - cache-amd64-${{ matrix.name }}- - cache-amd64- - - name: Install vcstool run: | sudo apt-get -y update diff --git a/.github/workflows/docker-build-and-push-main-self-hosted.yaml b/.github/workflows/docker-build-and-push-main-self-hosted.yaml index bcf0437ffb3..772ba389e82 100644 --- a/.github/workflows/docker-build-and-push-main-self-hosted.yaml +++ b/.github/workflows/docker-build-and-push-main-self-hosted.yaml @@ -56,22 +56,6 @@ jobs: - name: Check out repository uses: actions/checkout@v4 - - name: Cache ccache - uses: actions/cache@v4 - with: - path: ./ccache - key: cache-arm64-${{ matrix.name }}-${{ github.ref_name }} - restore-keys: | - cache-arm64-${{ matrix.name }}-main - cache-arm64-${{ matrix.name }}- - cache-arm64- - - - name: Check ccache - run: | - mkdir -p ./ccache - du -sh ./ccache - ls -R ./ccache - - name: Install vcstool run: | sudo apt-get -y update @@ -107,11 +91,6 @@ jobs: tag-prefix: ${{ needs.load-env.outputs.rosdistro }} allow-push: true - - name: Check ccache - run: | - du -sh ./ccache - ls -R ./ccache - - name: Show disk space run: | df -h diff --git a/.github/workflows/docker-build-and-push-main.yaml b/.github/workflows/docker-build-and-push-main.yaml index f7de667d98d..ef009a751bb 100644 --- a/.github/workflows/docker-build-and-push-main.yaml +++ b/.github/workflows/docker-build-and-push-main.yaml @@ -58,22 +58,6 @@ jobs: - name: Check out repository uses: actions/checkout@v4 - # - name: Cache ccache - # uses: actions/cache@v4 - # with: - # path: ./ccache - # key: cache-amd64-${{ matrix.name }}-${{ github.ref_name }} - # restore-keys: | - # cache-amd64-${{ matrix.name }}-main - # cache-amd64-${{ matrix.name }}- - # cache-amd64- - - - name: Check ccache - run: | - mkdir -p ./ccache - du -sh ./ccache - ls -R ./ccache - - name: Install vcstool run: | sudo apt-get -y update @@ -107,11 +91,6 @@ jobs: tag-prefix: ${{ needs.load-env.outputs.rosdistro }} allow-push: true - - name: Check ccache - run: | - du -sh ./ccache - ls -R ./ccache - - name: Show disk space run: | df -h diff --git a/.gitignore b/.gitignore index 0f3240ce591..024b426a3b1 100644 --- a/.gitignore +++ b/.gitignore @@ -12,9 +12,6 @@ install/ log/ /src/ -# C++ -.ccache - # Python *.pyc diff --git a/docker/Dockerfile b/docker/Dockerfile index 18bc35ac393..4a9261830d2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,7 +65,7 @@ FROM base as prebuilt SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG SETUP_ARGS -ENV CCACHE_DIR="/root/ccache" +ENV CCACHE_DIR="/var/tmp/ccache" # cspell: ignore libcu libnv # Set up development environment @@ -86,7 +86,7 @@ RUN --mount=type=ssh \ # Build Autoware COPY --from=src-imported /autoware/src /autoware/src -RUN --mount=type=bind,source=ccache,target=${CCACHE_DIR},rw \ +RUN --mount=type=cache,target=${CCACHE_DIR} \ ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} \ && source /opt/ros/"$ROS_DISTRO"/setup.bash \ && colcon build --cmake-args \ From e2f37c44c685e979eabcd3a0c5ee6056ce25941b Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 11:22:05 +0900 Subject: [PATCH 34/44] change dir Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4a9261830d2..5fa7ed623aa 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -65,7 +65,7 @@ FROM base as prebuilt SHELL ["/bin/bash", "-o", "pipefail", "-c"] ARG ROS_DISTRO ARG SETUP_ARGS -ENV CCACHE_DIR="/var/tmp/ccache" +ENV CCACHE_DIR="/root/.ccache" # cspell: ignore libcu libnv # Set up development environment From 7426cb47d444559f136d87aa55d65eea62513c16 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 11:23:13 +0900 Subject: [PATCH 35/44] remove original .ccache Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5fa7ed623aa..287c59d550d 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -85,6 +85,7 @@ RUN --mount=type=ssh \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache # Build Autoware +RUN rm -rf /root/.ccache COPY --from=src-imported /autoware/src /autoware/src RUN --mount=type=cache,target=${CCACHE_DIR} \ ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} \ From 03457d2b19ab3014bd32cf755a705c9ee7eb2328 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 12:41:22 +0900 Subject: [PATCH 36/44] add syntax Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 287c59d550d..e7f702a91bc 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 # hadolint global ignore=DL3006,DL3008,DL3013 ARG BASE_IMAGE From 1e040692d2830da25801fb8a62135cc8a3bb7ef7 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 12:47:28 +0900 Subject: [PATCH 37/44] mkdir Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e7f702a91bc..1889a30b754 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -86,7 +86,7 @@ RUN --mount=type=ssh \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache # Build Autoware -RUN rm -rf /root/.ccache +RUN mkdir -p ${CCACHE_DIR} COPY --from=src-imported /autoware/src /autoware/src RUN --mount=type=cache,target=${CCACHE_DIR} \ ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} \ From 6756223eb62256583a0b3acd3203f3949377cd09 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 12:57:46 +0900 Subject: [PATCH 38/44] disable ansible install Signed-off-by: Yutaka Kondo --- ansible/playbooks/openadkit.yaml | 4 ++-- docker/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/playbooks/openadkit.yaml b/ansible/playbooks/openadkit.yaml index f343d9cef55..12320023779 100644 --- a/ansible/playbooks/openadkit.yaml +++ b/ansible/playbooks/openadkit.yaml @@ -24,8 +24,8 @@ when: module == 'base' - role: autoware.dev_env.kisak_mesa when: module == 'base' - - role: autoware.dev_env.build_tools - when: module == 'all' and install_devel=='y' + # - role: autoware.dev_env.build_tools + # when: module == 'all' and install_devel=='y' # Module specific dependencies - role: autoware.dev_env.geographiclib diff --git a/docker/Dockerfile b/docker/Dockerfile index 1889a30b754..e7fc31504af 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -73,6 +73,7 @@ ENV CCACHE_DIR="/root/.ccache" RUN --mount=type=ssh \ ./setup-dev-env.sh -y --module all ${SETUP_ARGS} --no-cuda-drivers openadkit \ && pip uninstall -y ansible ansible-core \ + && apt-get update && apt-get install -y --no-install-recommends ccache \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache \ && find / -name 'libcu*.a' -delete \ && find / -name 'libnv*.a' -delete @@ -86,7 +87,6 @@ RUN --mount=type=ssh \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache # Build Autoware -RUN mkdir -p ${CCACHE_DIR} COPY --from=src-imported /autoware/src /autoware/src RUN --mount=type=cache,target=${CCACHE_DIR} \ ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} \ From 1c6e4e6d40d0ce3e63dd2a48d70aff21452f05b5 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 17:11:04 +0900 Subject: [PATCH 39/44] copy from host Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e7fc31504af..c8f577d9a06 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -87,7 +87,7 @@ RUN --mount=type=ssh \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache # Build Autoware -COPY --from=src-imported /autoware/src /autoware/src +COPY src /autoware/src RUN --mount=type=cache,target=${CCACHE_DIR} \ ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} \ && source /opt/ros/"$ROS_DISTRO"/setup.bash \ From 3776dcc1d12c81e9a921615c229bb03f0c74b357 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 19:13:08 +0900 Subject: [PATCH 40/44] just test cache mount --- docker/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index c8f577d9a06..adcc5578a29 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -86,6 +86,9 @@ RUN --mount=type=ssh \ && cat /tmp/rosdep-all-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache +RUN --mount=type=cache,target=/root/.test \ + mkdir -p /root/.test && ls /root/.test && echo "Test" > /root/.test/test.txt + # Build Autoware COPY src /autoware/src RUN --mount=type=cache,target=${CCACHE_DIR} \ From 4fbd9ba0696a844d548199d04f8c2341594a06dd Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 19:13:47 +0900 Subject: [PATCH 41/44] comment out Signed-off-by: Yutaka Kondo --- docker/Dockerfile | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index adcc5578a29..35d05721506 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -89,18 +89,18 @@ RUN --mount=type=ssh \ RUN --mount=type=cache,target=/root/.test \ mkdir -p /root/.test && ls /root/.test && echo "Test" > /root/.test/test.txt -# Build Autoware -COPY src /autoware/src -RUN --mount=type=cache,target=${CCACHE_DIR} \ - ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} \ - && source /opt/ros/"$ROS_DISTRO"/setup.bash \ - && colcon build --cmake-args \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ - " -Wno-dev" \ - " --no-warn-unused-cli" \ - --mixin ccache \ - && ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} +# # Build Autoware +# COPY src /autoware/src +# RUN --mount=type=cache,target=${CCACHE_DIR} \ +# ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} \ +# && source /opt/ros/"$ROS_DISTRO"/setup.bash \ +# && colcon build --cmake-args \ +# -DCMAKE_BUILD_TYPE=Release \ +# -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ +# " -Wno-dev" \ +# " --no-warn-unused-cli" \ +# --mixin ccache \ +# && ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} CMD ["/bin/bash"] From 08daf379f0edd738ed117dd1409e93dc81140f71 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 19:21:01 +0900 Subject: [PATCH 42/44] empty Signed-off-by: Yutaka Kondo From a220f0f1676a25df0ce44e036de44ae84a19dd72 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 19:29:27 +0900 Subject: [PATCH 43/44] Revert "comment out" This reverts commit 4fbd9ba0696a844d548199d04f8c2341594a06dd. --- docker/Dockerfile | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 35d05721506..adcc5578a29 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -89,18 +89,18 @@ RUN --mount=type=ssh \ RUN --mount=type=cache,target=/root/.test \ mkdir -p /root/.test && ls /root/.test && echo "Test" > /root/.test/test.txt -# # Build Autoware -# COPY src /autoware/src -# RUN --mount=type=cache,target=${CCACHE_DIR} \ -# ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} \ -# && source /opt/ros/"$ROS_DISTRO"/setup.bash \ -# && colcon build --cmake-args \ -# -DCMAKE_BUILD_TYPE=Release \ -# -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ -# " -Wno-dev" \ -# " --no-warn-unused-cli" \ -# --mixin ccache \ -# && ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} +# Build Autoware +COPY src /autoware/src +RUN --mount=type=cache,target=${CCACHE_DIR} \ + ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} \ + && source /opt/ros/"$ROS_DISTRO"/setup.bash \ + && colcon build --cmake-args \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + " -Wno-dev" \ + " --no-warn-unused-cli" \ + --mixin ccache \ + && ccache -s && du -sh ${CCACHE_DIR} && ls ${CCACHE_DIR} CMD ["/bin/bash"] From ba9e860332f1c92555369f7634b01205a2828fd0 Mon Sep 17 00:00:00 2001 From: Yutaka Kondo Date: Fri, 7 Jun 2024 19:29:35 +0900 Subject: [PATCH 44/44] Revert "just test cache mount" This reverts commit 3776dcc1d12c81e9a921615c229bb03f0c74b357. --- docker/Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index adcc5578a29..c8f577d9a06 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -86,9 +86,6 @@ RUN --mount=type=ssh \ && cat /tmp/rosdep-all-depend-packages.txt | xargs apt-get install -y --no-install-recommends \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* "$HOME"/.cache -RUN --mount=type=cache,target=/root/.test \ - mkdir -p /root/.test && ls /root/.test && echo "Test" > /root/.test/test.txt - # Build Autoware COPY src /autoware/src RUN --mount=type=cache,target=${CCACHE_DIR} \