diff --git a/.circleci/config.yml b/.circleci/config.yml index 99a7a13929..b77df85505 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -536,38 +536,38 @@ _parameters: workflows: version: 2 - build_and_test: - jobs: - - release_build: - <<: *release_parameters - - system_build: - requires: - - release_build - - release_test: - requires: - - system_build - cache_test: true - nightly: - jobs: - - release_build: - <<: *release_parameters - - system_build: - requires: - - release_build - - release_test: - requires: - - system_build - <<: *release_parameters - matrix: - alias: release_test - parameters: - rmw: - - rmw_cyclonedds_cpp - - rmw_fastrtps_cpp - triggers: - - schedule: - cron: "0 13 * * *" - filters: - branches: - only: - - main + # build_and_test: + # jobs: + # - release_build: + # <<: *release_parameters + # - system_build: + # requires: + # - release_build + # - release_test: + # requires: + # - system_build + # cache_test: true + # nightly: + # jobs: + # - release_build: + # <<: *release_parameters + # - system_build: + # requires: + # - release_build + # - release_test: + # requires: + # - system_build + # <<: *release_parameters + # matrix: + # alias: release_test + # parameters: + # rmw: + # - rmw_cyclonedds_cpp + # - rmw_fastrtps_cpp + # triggers: + # - schedule: + # cron: "0 13 * * *" + # filters: + # branches: + # only: + # - main diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 0000000000..b16924accd --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,149 @@ +# Dev Containers + +This folder contains the necessary files to build and run development containers for the Navigation2 project. The containers are based on the ROS 2 Rolling distribution and include all necessary dependencies to build and run the Navigation2 stack. + +## Quick Start + +To get started, follow the instructions below. + +### Prerequisites + +First, ensure your using a recent enough version of Docker Engine that supports [BuildKit](https://docs.docker.com/build/buildkit/). If you plan on running robot simulations locally, Hardware Acceleration for sensor raytracing and 3D rendering is also recommended. While other compatible devcontainer tools may be used, Visual Studio Code is recommended for simplicity. + +#### System Software +- [Docker Engine](https://docs.docker.com/engine/install/) + - https://get.docker.com - simple universal install script + - [Linux post-installation](https://docs.docker.com/engine/install/linux-postinstall/) - manage Docker as a non-root user +- [Git LFS](https://git-lfs.github.com/) - optional for managing large assets + - Use for version controlling media such as figures +- [NVIDIA Container Toolkit](https://github.com/NVIDIA/nvidia-container-toolkit) - optional for enabling Hardware Acceleration + - [Installing the Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) - only necessary host running Docker Engine + +#### Development Tools +- [Visual Studio Code](https://code.visualstudio.com/) - alternative to Dev Containers CLI + - [Remote Development](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.vscode-remote-extensionpack) - via SSH, Tunnels, Containers, WSL + - [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) - specific to just Containers + - [Docker extension](https://marketplace.visualstudio.com/items?itemName=ms-azuretools.vscode-docker) - for introspecting Docker daemon + - [Using SSH keys](https://code.visualstudio.com/remote/advancedcontainers/sharing-git-credentials#_using-ssh-keys) - sharing Git credentials with container +- [Dev Container CLI](https://github.com/devcontainers/cli) - alternative to VSCode + - [Installation via NPM](https://github.com/devcontainers/cli?tab=readme-ov-file#npm-install) - for custom install setup + - [Installation via VSCode](https://code.visualstudio.com/docs/devcontainers/devcontainer-cli) - for simple install setup + - Note: CLI installed via VSCode is warped but bugged, install via NPM is recommended for now + - https://github.com/devcontainers/cli/issues/799 +- [GitHub CLI](https://cli.github.com/) - optional for interacting with CI Workflows + - [Installation](https://github.com/cli/cli#installation) - specifically [for Linux](https://github.com/cli/cli/blob/trunk/docs/install_linux.md) + - [Configuration](https://cli.github.com/manual/) - login authentication and setup + +### Environment Setup + +Once you've setup your ssh keys for GitHub and account credentials for the CLI, you can add your private SSH key to the SSH agent if you'd like to use your local credentials for Git operations inside the dev container: + +```shell +# Add your SSH key to the SSH agent +ssh-add ~/.ssh/ +``` + +You may also configure Docker to use those credentials when interacting with the GitHub Container Registry (GHCR). This setup is optional, but can be useful to avoid rate limiting issues when pulling images from a popular public IP address. This can be done by using docker login with the GitHub CLI: + +```shell +# Login to GitHub Container Registry +gh auth token | docker login ghcr.io --username --password-stdin +``` + +### Submodule Setup + +Next, recursively clone this repository and included submodules. + +```shell +# Clone the repository and submodules +git clone --recurse-submodules -j8 \ + git@github.com:ros-navigation/navigation2.git + +# Change into the repository directory +cd navigation2 + +# Configure the local git include path +git config --local include.path ../.gitconfig +``` + +### Images Setup + +To create the base image for the container, one can either build the image locally, or pull CI image layers cached remotely. If you need to modify the base image during the development process, such as adding or removing dependencies, you can build images locally before opening any PRs. If you want to evaluate modifications to the base image during the review process, you could then also pull pre-built CI images from an opened PR. + +#### Building Locally + +Building locally leverages various caching strategies to speedup subsequent docker rebuilds, such as caching apt package downloads, incremental build artifacts, and multi-stage optimizations for image layer reuse. You can either let the dev container's initialization lifecycle script build the image for you, or you can pre-build the image manually: + +```shell +# Bake the dever image tag as a test +docker buildx bake dever + +# Run container from image as a test +docker run -it --rm nav2:dever bash +``` + +#### Pulling Remotely + +Alternatively, you can pull the CI image layers from the GitHub Container Registry (GHCR) to bootstrap the dev container. While this method may be faster or slower depending on your local network connection, it remains particularly useful for downloading pre-built colcon workspaces built by CI and baked into the debugger stage, avoiding the need to build the image layers or re-compile a PR locally. Simply use the `DEV_FROM_STAGE` environment variable to shortcut the build process to use a given reference image, as commented from the initialization script: + +```shell +REFERENCE_IMAGE=ghcr.io/ros-navigation/navigation2:main-debugger +docker pull $REFERENCE_IMAGE +export DEV_FROM_STAGE=$REFERENCE_IMAGE +``` + +> Note: you may want to clean or create new named volumes as defined in the dev container config, given cached volumes only initialize from their first attached container. I.e. to switch between different PRs, you'll want to avoid using a named volume that was seeded from a different/older docker image. + +### Launching Development Containers + +Note: using Dev Containers from a remote host is also possible: + +- [Open a folder on a remote SSH host in a container](https://code.visualstudio.com/docs/devcontainers/containers#_open-a-folder-on-a-remote-ssh-host-in-a-container) +- [Open a folder on a remote Tunnel host in a container](https://code.visualstudio.com/docs/devcontainers/containers#_open-a-folder-on-a-remote-tunnel-host-in-a-container) + +#### Visual Studio Code +Finally, open VSCode and use the Remote Containers extension: + +```shell +code . +# Press Ctrl+Shift+P to open the Command Palette +# Type and select `Dev Containers: Reopen in Container` +``` + +#### Dev Containers CLI +Alternatively, use the CLI to bring up and exec into the Dev Container: + +```shell +devcontainer up --workspace-folder . +devcontainer exec --workspace-folder . bash +``` + +### Verifying Development Containers + +To verify the dev container is setup correctly, i.e. hardware acceleration and display forwarding is working as expected, you can run simulation examples to check: + +```shell +# Included alias to source overlay workspace +sows +# Launch simulation example with GUIs enabled +ros2 launch nav2_bringup tb4_simulation_launch.py headless:=False +``` + +## Further Reading and Concepts + +Afterwards, you may want to further familiarize yourself more with the following topics: + +- Git Submodules + - https://git-scm.com/book/en/Git-Tools-Submodules + - https://git-scm.com/docs/git-submodule +- Docker + - Multi-stage + - https://docs.docker.com/build/building/multi-stage/ + - BuildKit + - https://docs.docker.com/build/buildkit/ + - Bake + - https://docs.docker.com/build/bake/ +- Development Containers + - https://navigation.ros.org/development_guides/devcontainer_docs/index.html + - https://containers.dev/ + - https://code.visualstudio.com/docs/devcontainers/containers diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 77a326648d..adf6d9ce3f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,44 +1,100 @@ { "name": "Nav2", - "build": { - "dockerfile": "../Dockerfile", - "context": "..", - "target": "dever", - "cacheFrom": "ghcr.io/ros-navigation/navigation2:main" - }, + "initializeCommand": ".devcontainer/initialize-command.sh dever", // Bakes to tag nav2:devcontainer + "image": "nav2:devcontainer", "runArgs": [ - "--name=nav2" // "--cap-add=SYS_PTRACE", // enable debugging, e.g. gdb // "--ipc=host", // shared memory transport with host, e.g. rviz GUIs // "--network=host", // network access to host interfaces, e.g. eth0 // "--pid=host", // DDS discovery with host, without --network=host // "--privileged", // device access to host peripherals, e.g. USB // "--security-opt=seccomp=unconfined", // enable debugging, e.g. gdb + // "--device=/dev/dri", // enable Intel integrated graphics + // "--ulimit", "nofile=1024:4096", // increase file descriptor limit for valgrind + // + "--runtime=nvidia", // enable NVIDIA Container Toolkit + "--env=NVIDIA_VISIBLE_DEVICES=all", // enable GPUs with env as --gpus doesn't parse nicely + "--env=NVIDIA_DRIVER_CAPABILITIES=all", // enable all capabilities, including `graphics` ], - "workspaceFolder": "/opt/overlay_ws/src/navigation2", + "workspaceFolder": "/opt/nav2_ws/src/navigation2", "workspaceMount": "source=${localWorkspaceFolder},target=${containerWorkspaceFolder},type=bind", "onCreateCommand": ".devcontainer/on-create-command.sh", "updateContentCommand": ".devcontainer/update-content-command.sh", "postCreateCommand": ".devcontainer/post-create-command.sh", "remoteEnv": { + "CCACHE_DIR": "/opt/nav2_ws/.ccache", + // Explicitly set DISPLAY for NVIDIA Container Toolkit + "DISPLAY": "${localEnv:DISPLAY}", "OVERLAY_MIXINS": "release ccache lld", - "CCACHE_DIR": "/tmp/.ccache" + // Explicitly set SSH_AUTH_SOCK for devcontainer CLI + "SSH_AUTH_SOCK": "${localEnv:SSH_AUTH_SOCK}", }, + "remoteUser": "ubuntu", "mounts": [ + // ################################################################################ + // # MARK: Cache mounts - for development + // ################################################################################ + { + // Cache apt downloads + "source": "apt-cache", + "target": "/var/cache/apt", + "type": "volume" + }, + { + // Cache ccache caches + "source": "ccache", + "target": "/opt/nav2_ws/.ccache", + "type": "volume" + }, { - "source": "ccache-${devcontainerId}", - "target": "/tmp/.ccache", + // Cache colcon workspace + "source": "nav2-ws-${devcontainerId}", + "target": "/opt/nav2_ws", "type": "volume" }, + // ################################################################################ + // # MARK: Personal mounts - for convenience + // ################################################################################ { - "source": "overlay-${devcontainerId}", - "target": "/opt/overlay_ws", + // Mount home dotfiles + "source": "nav2-home-${localEnv:USER}", + "target": "/home/ubuntu", "type": "volume" + }, + // { + // // Mount home nav2 + // "source": "${localEnv:HOME}/nav2/", + // "target": "/home/ubuntu/nav2", + // "type": "bind" + // }, + // ################################################################################ + // # MARK: Socket mounts - for tooling + // ################################################################################ + { + // Mount docker socket + "source": "/var/run/docker.sock", + "target": "/var/run/docker-host.sock", + "type": "bind" + }, + { + // Explicitly mount X11 socket for NVIDIA Container Toolkit + // as setting NVIDIA_DRIVER_CAPABILITIES to include `graphics` + // interferes with VSCode's default X11 forwarding behavior + "source": "/tmp/.X11-unix", + "target": "/tmp/.X11-unix", + "type": "bind" + }, + { + // Explicitly mount SSH socket for devcontainer CLI + "source": "${localEnv:SSH_AUTH_SOCK}", + "target": "${localEnv:SSH_AUTH_SOCK}", + "type": "bind" } ], "features": { // "ghcr.io/devcontainers/features/desktop-lite:1": {}, - "ghcr.io/devcontainers/features/github-cli:1": {} + "ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}, + "ghcr.io/devcontainers/features/github-cli:1": {}, }, "customizations": { "codespaces": { @@ -53,9 +109,11 @@ "eamodio.gitlens", "esbenp.prettier-vscode", "GitHub.copilot", + "hashicorp.hcl", + "ms-azuretools.vscode-docker", "ms-iot.vscode-ros", "streetsidesoftware.code-spell-checker", - "twxs.cmake" + "vitaliymaz.vscode-svg-previewer", ] } } diff --git a/.devcontainer/initialize-command.sh b/.devcontainer/initialize-command.sh new file mode 100755 index 0000000000..79546fe52b --- /dev/null +++ b/.devcontainer/initialize-command.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# Immediately catch all errors +set -eo pipefail + +# Uncomment for debugging +# set -x +# env + +# Use first argument as target name +target=$1 + +################################################################################ +# MARK: Pull image - download image from CI and GHCR for local dev container +# REFERENCE_IMAGE=ghcr.io/ros-navigation/navigation2:main-debugger +# docker pull $REFERENCE_IMAGE +# export DEV_FROM_STAGE=$REFERENCE_IMAGE +################################################################################ + +# Bake the target and export locally to static tag +docker buildx bake --load \ + --file docker-bake.hcl \ + --set $target.tags=nav2:devcontainer \ + $target + +# mkdir -p $HOME/nav2 diff --git a/.devcontainer/on-create-command.sh b/.devcontainer/on-create-command.sh index f86d22bc8d..883a1cb862 100755 --- a/.devcontainer/on-create-command.sh +++ b/.devcontainer/on-create-command.sh @@ -7,6 +7,26 @@ set -eo pipefail # set -x # env +# Set git config for submodules +git config --local include.path ../.gitconfig + +# Set git config for colcon cache git config --global --add safe.directory "*" -.devcontainer/update-content-command.sh +# NOTE: This is slow if not using a mounted volumes, +# i.e. using workspace from the docker image directly, +# presumably due to docker overlayfs driver overhead. +# If needing to use workspace pre-baked into base image, +# consider using a new volume to be auto populated with +# the workspace pre-baked in image via devcontainer tools. +# Either by deleting old volume from the docker engine +# Or simpler changing volume name in devcontainer.json +sudo chown -R :ubuntu $OVERLAY_WS +# Recursively update group permissions for workspace +# to allow write access via dev users current group +sudo chmod -R g+rwx $OVERLAY_WS + +# Recursively update group permissions for ros home +# to allow write access such as ros node logs +sudo chown -R :ubuntu /opt/.ros +sudo chmod -R g+rwx /opt/.ros diff --git a/.devcontainer/post-create-command.sh b/.devcontainer/post-create-command.sh index 9f64a2c6ab..02cc2c89fb 100755 --- a/.devcontainer/post-create-command.sh +++ b/.devcontainer/post-create-command.sh @@ -10,9 +10,9 @@ set -eo pipefail # Enable autocomplete for user cp /etc/skel/.bashrc ~/ -# Check if srv folder exists -if [ -d "$ROOT_SRV" ]; then - # Setup Nav2 web app - for dir in $OVERLAY_WS/src/navigation2/.devcontainer/caddy/srv/*; \ - do if [ -d "$dir" ]; then ln -s "$dir" $ROOT_SRV; fi done -fi +# Enable autocomplete using colcon +echo "source /usr/share/colcon_argcomplete/hook/colcon-argcomplete.bash" >> ~/.bashrc + +# Add aliases to .bashrc +echo "alias sows='source $OVERLAY_WS/install/setup.bash'" >> ~/.bashrc +echo "alias suws='source $ROS_DISTRO/setup.bash'" >> ~/.bashrc diff --git a/.devcontainer/update-content-command.sh b/.devcontainer/update-content-command.sh index 2bdbf5cc1a..7597c63285 100755 --- a/.devcontainer/update-content-command.sh +++ b/.devcontainer/update-content-command.sh @@ -42,19 +42,21 @@ then colcon list \ --names-only \ --packages-above \ - $BUILD_UNFINISHED \ - $BUILD_FAILED \ - $BUILD_INVALID \ + $BUILD_UNFINISHED \ + $BUILD_FAILED \ + $BUILD_INVALID \ | xargs) fi echo BUILD_PACKAGES: $BUILD_PACKAGES +# DEBUG: Uncoment for more sterile but slower builds # colcon clean packages --yes \ # --packages-select ${BUILD_PACKAGES} \ -# --base-select install +# --base-select build install -. $UNDERLAY_WS/install/setup.sh +# OPTOINAL: Uncomment to build packages upon update +. /opt/ros/$ROS_DISTRO/setup.sh colcon build \ - --symlink-install \ --mixin $OVERLAY_MIXINS \ - --packages-select ${BUILD_PACKAGES} + --packages-select ${BUILD_PACKAGES} \ + || true diff --git a/.docker/.ccache/ccache.conf b/.docker/.ccache/ccache.conf new file mode 100644 index 0000000000..7bb7470470 --- /dev/null +++ b/.docker/.ccache/ccache.conf @@ -0,0 +1,32 @@ +# Set the maximum size of the cache to 5G +# max_size = 5G + +# Set the base directory for the cache +# cache_dir = + +# Set the compression level +# compression = true + +# Set the compression level +# compression_level = 0 + +# Set the log file +# log_file = + +# Set the temporary directory +# temporary_dir = + +# Set the umask +# umask = + +# Set the compiler check +# compiler_check = mtime + +# Set the direct mode +# direct_mode = true + +# Set the read-only mode +# read_only = false + +# Set the hard link mode +# hard_link = false \ No newline at end of file diff --git a/.docker/.colcon/defaults.yaml b/.docker/.colcon/defaults.yaml new file mode 100644 index 0000000000..70abdba213 --- /dev/null +++ b/.docker/.colcon/defaults.yaml @@ -0,0 +1,18 @@ +_common: &common + # Record test results to separate directory + # for simpler parsing and exporting with CI + "test-result-base": "test_results" + +"clean.packages": + <<: *common +"build": + <<: *common + "executor": "parallel" + # "parallel-workers": 4 + "symlink-install": false +"test": + <<: *common + "executor": "parallel" + # "parallel-workers": 1 +"test-result": + <<: *common diff --git a/.docker/Dockerfile b/.docker/Dockerfile new file mode 100644 index 0000000000..8613efebff --- /dev/null +++ b/.docker/Dockerfile @@ -0,0 +1,291 @@ +# syntax=docker/dockerfile:1.7 +ARG DEV_FROM_STAGE=tooler +ARG EXPORT_FROM_STAGE=builder +ARG PREP_FROM_STAGE=runner +ARG SHIP_FROM_STAGE=runner +ARG WS_CACHE_ID=nav2 + +ARG FROM_IMAGE=base +# Stage from full image tag name for dependabot detection +FROM ros:rolling AS base + +################################################################################ +# MARK: baser - setup base image using snapshots +################################################################################ +FROM $FROM_IMAGE AS baser +ENV FROM_IMAGE=${FROM_IMAGE} + +# Configure ubuntu snapshot +RUN UBUNTU_DEB_SNAPSHOT=$(date -r /var/lib/dpkg/info +%Y%m%dT%H%M%SZ) && \ + sed -i "s|http://archive.ubuntu.com/ubuntu/|http://snapshot.ubuntu.com/ubuntu/${UBUNTU_DEB_SNAPSHOT}|g" /etc/apt/sources.list && \ + sed -i "s|http://security.ubuntu.com/ubuntu/|http://snapshot.ubuntu.com/ubuntu/${UBUNTU_DEB_SNAPSHOT}|g" /etc/apt/sources.list + +# Edit apt config for caching +RUN mv /etc/apt/apt.conf.d/docker-clean /etc/apt/ && \ + echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' \ + > /etc/apt/apt.conf.d/keep-cache && \ + # Given fixed snapshots, just cache apt update once + apt-get update && echo v1 + +# Configure overlay workspace +ENV OVERLAY_WS=/opt/nav2_ws +WORKDIR $OVERLAY_WS + +# install bootstrap tools +RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ + apt-get install -y --no-install-recommends \ + gettext-base \ + wget \ + zstd + +# init and update rosdep +ENV ROS_HOME=/opt/.ros +RUN rosdep update \ + --rosdistro $ROS_DISTRO + +################################################################################ +# MARK: cacher - cache source and dependency instructions +################################################################################ +FROM baser AS cacher + +# copy overlay source +COPY ./ ./src/navigation2 + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +RUN dep_types=(\ + "exec:--dependency-types=exec" \ + "test:--dependency-types=test" \ + "build:"\ + ) && \ + for dep_type in "${dep_types[@]}"; do \ + IFS=":"; set -- $dep_type; \ + rosdep install -y \ + --from-paths src \ + --ignore-src \ + --skip-keys " \ + # From Nav2 + slam_toolbox \ + turtlebot3_gazebo \ + "\ + --reinstall \ + --simulate \ + ${2} \ + | sed -E "s/'//g; s/ \(alternative .*\)//g" \ + | tail -n +2 \ + | awk '{print $NF}' \ + | sort > /tmp/rosdep_${1}_debs.txt; \ + done + +################################################################################ +# MARK: runner - setup runtime dependencies for deployment +################################################################################ +FROM baser AS runner + +# install packages for field work +COPY .docker/runner_apt_debs.txt /tmp/runner_apt_debs.txt +RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ + cut -d# -f1 < /tmp/runner_apt_debs.txt | envsubst \ + | xargs apt-get install -y --no-install-recommends + +COPY --from=cacher /tmp/rosdep_exec_debs.txt /tmp/rosdep_exec_debs.txt +RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ + < /tmp/rosdep_exec_debs.txt xargs apt-get install -y --no-install-recommends + +################################################################################ +# MARK: prepper - bootstrap general dependencies for development +################################################################################ +FROM $PREP_FROM_STAGE AS prepper + +# install bootstrap tools +RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ + apt-get install -y --no-install-recommends \ + git \ + python3-colcon-clean \ + python3-pip \ + && pip3 install --break-system-packages \ + git+https://github.com/ruffsl/colcon-cache.git@6076815bbb574da028d270cf6eb93bdd5b29c7f4 +ENV COLCON_EXTENSION_BLOCKLIST="$COLCON_EXTENSION_BLOCKLIST:colcon_core.package_augmentation.cache_git" + +# setup colcon mixin and metadata +ENV COLCON_HOME=/opt/.colcon +RUN colcon mixin add default \ + https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml && \ + colcon mixin update && \ + colcon metadata add default \ + https://raw.githubusercontent.com/colcon/colcon-metadata-repository/master/index.yaml && \ + colcon metadata update + +################################################################################ +# MARK: validator - setup test dependencies for validation +################################################################################ +FROM prepper AS validator + +COPY --from=cacher /tmp/rosdep_test_debs.txt /tmp/rosdep_test_debs.txt +RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ + < /tmp/rosdep_test_debs.txt xargs apt-get install -y --no-install-recommends + +################################################################################ +# MARK: tooler - setup build dependencies for compilation +################################################################################ +FROM validator AS tooler + +COPY --from=cacher /tmp/rosdep_build_debs.txt /tmp/ +RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ + < /tmp/rosdep_build_debs.txt xargs apt-get install -y --no-install-recommends + +# install packages for build work +COPY .docker/tooler_apt_debs.txt /tmp/tooler_apt_debs.txt +RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ + cut -d# -f1 < /tmp/tooler_apt_debs.txt | envsubst \ + | xargs apt-get install -y --no-install-recommends + +# setup default ccache configuration +COPY .docker/.ccache/ccache.conf /opt/.ccache/ +ENV CCACHE_CONFIGPATH="/opt/.ccache/ccache.conf" +ENV CCACHE_DIR="$OVERLAY_WS/.ccache" + +# setup default colcon configuration +COPY .docker/.colcon/defaults.yaml $COLCON_HOME/ +ARG OVERLAY_MIXINS="release ccache lld" +ENV OVERLAY_MIXINS=${OVERLAY_MIXINS} + +# capture environment to modify layer digest +RUN env > /tmp/env.txt + +################################################################################ +# MARK: seeder - seed workspace artifacts for caching +################################################################################ +FROM baser AS seeder +ARG WS_CACHE_ID + +ARG CLEAR_WS_CACHE +RUN --mount=type=cache,id=$WS_CACHE_ID,sharing=private,target=$OVERLAY_WS \ + if [ -n "$CLEAR_WS_CACHE" ]; then \ + echo "Clearing cache!" && \ + rm -rf $OVERLAY_WS/* && \ + echo "Cache cleared!"; \ + fi + +ARG SEED_WS_CACHE +RUN --mount=type=bind,source=./,target=/seeder/\ + --mount=type=cache,id=$WS_CACHE_ID,sharing=private,target=$OVERLAY_WS \ + if [ -n "$SEED_WS_CACHE" ]; then \ + echo "Seeding cache!" && \ + cp -rT /seeder/cache/$OVERLAY_WS $OVERLAY_WS && \ + echo "Cache seeded!"; \ + fi + +RUN --mount=type=cache,from=cacher,target=/cacher \ + --mount=type=cache,id=$WS_CACHE_ID,sharing=private,target=$OVERLAY_WS \ + rm -rf ./src && \ + cp -r /cacher/$OVERLAY_WS/src ./ && \ + echo $(date) > /tmp/seeder_stamp.txt + +################################################################################ +# MARK: builder - build workspace artifacts for deployment +################################################################################ +FROM tooler AS builder +ARG WS_CACHE_ID + +# build overlay source +ARG BUST_BUILD_CACHE +RUN --mount=type=cache,from=seeder,target=/seeder \ + --mount=type=cache,id=$WS_CACHE_ID,sharing=private,target=$OVERLAY_WS \ + . /opt/ros/$ROS_DISTRO/setup.sh && \ + colcon cache lock \ + --dirhash-reset && \ + colcon clean packages -y \ + --packages-select-cache-invalid \ + --packages-select-cache-key build \ + --base-select install && \ + colcon build \ + --packages-skip-cache-valid \ + --mixin $OVERLAY_MIXINS \ + || true && \ + echo $(date) > /tmp/builder_stamp.txt + +ARG FAIL_ON_BUILD_FAILURE=1 +RUN --mount=type=cache,id=$WS_CACHE_ID,sharing=private,target=$OVERLAY_WS \ + . /opt/ros/$ROS_DISTRO/setup.sh && \ + BUILD_FAILED=$( \ + colcon list \ + --packages-select-build-failed) && \ + if [ -n "$BUILD_FAILED" ]; then \ + echo "BUILD_FAILED: \n$BUILD_FAILED" && \ + ([ -z "$FAIL_ON_BUILD_FAILURE" ] || exit 1); \ + fi + +################################################################################ +# MARK: dever - setup user account for development +################################################################################ +FROM $DEV_FROM_STAGE AS dever + +# install packages for development work +COPY .docker/dever_apt_debs.txt /tmp/dever_apt_debs.txt +RUN --mount=type=cache,sharing=locked,target=/var/cache/apt \ + cut -d# -f1 < /tmp/dever_apt_debs.txt | envsubst \ + | xargs apt-get install -y --no-install-recommends + +# add default user for devcontainer +ENV DEV_USER=ubuntu +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +################################################################################ +# MARK: tester - test workspace artifacts for development +################################################################################ +FROM validator AS tester +ARG WS_CACHE_ID + +# test overlay build +ARG BUST_TEST_CACHE +RUN --mount=type=cache,from=builder,target=/builder \ + --mount=type=cache,id=$WS_CACHE_ID,sharing=private,target=$OVERLAY_WS \ + . install/setup.sh && \ + colcon test \ + --packages-skip-cache-valid && \ + echo $(date) > /tmp/tester_stamp.txt + +ARG FAIL_ON_TEST_FAILURE=1 +RUN --mount=type=cache,id=$WS_CACHE_ID,sharing=private,target=$OVERLAY_WS \ + . install/setup.sh && \ + colcon test-result \ + --verbose \ + || ([ -z "$FAIL_ON_TEST_FAILURE" ] || exit 1) + +################################################################################ +# MARK: dancer - multi-stage for caches dancing +################################################################################ +FROM $EXPORT_FROM_STAGE AS dancer +ARG WS_CACHE_ID + +RUN --mount=type=cache,id=$WS_CACHE_ID,sharing=private,target=$OVERLAY_WS,readonly \ + echo "Exporting cache!" && \ + mkdir -p /dancer/$OVERLAY_WS && \ + cp -rT $OVERLAY_WS /dancer/$OVERLAY_WS && \ + echo "Cache exported!" + +################################################################################ +# MARK: exporter - multi-stage for exporting caches +################################################################################ +FROM $EXPORT_FROM_STAGE AS exporter + +COPY --link --from=dancer /dancer/$OVERLAY_WS $OVERLAY_WS + +################################################################################ +# MARK: shipper - setup production images using shared instructions +################################################################################ +FROM $SHIP_FROM_STAGE AS shipper + +################################################################################ +# MARK: debugger - stage target for debuggin in production +################################################################################ +FROM shipper AS debugger + +COPY --link --from=dancer /dancer/$OVERLAY_WS $OVERLAY_WS + +################################################################################ +# MARK: releaser - stage target for releasing in production +################################################################################ +FROM shipper AS releaser + +COPY --link --from=dancer /dancer/$OVERLAY_WS/install $OVERLAY_WS/install \ No newline at end of file diff --git a/.docker/dever_apt_debs.txt b/.docker/dever_apt_debs.txt new file mode 100644 index 0000000000..62103367bf --- /dev/null +++ b/.docker/dever_apt_debs.txt @@ -0,0 +1,4 @@ +# ROS tools +ros-${ROS_DISTRO}-rqt-common-plugins +ros-${ROS_DISTRO}-rviz-common +ros-${ROS_DISTRO}-rviz2 diff --git a/.docker/runner_apt_debs.txt b/.docker/runner_apt_debs.txt new file mode 100644 index 0000000000..7e8f34b54c --- /dev/null +++ b/.docker/runner_apt_debs.txt @@ -0,0 +1,16 @@ +# ROS +ros-${ROS_DISTRO}-rmw-cyclonedds-cpp +ros-${ROS_DISTRO}-ros2cli-common-extensions + +# System +bash-completion + +# Tooling +bashtop +byobu +fish +htop +iftop +rsync +tmux +tmuxinator diff --git a/.docker/tooler_apt_debs.txt b/.docker/tooler_apt_debs.txt new file mode 100644 index 0000000000..5d746b71d8 --- /dev/null +++ b/.docker/tooler_apt_debs.txt @@ -0,0 +1,14 @@ +# System tools +build-essential +ca-certificates +ccache +distcc +gdb +git +git-lfs +lld +mesa-utils +nano +unzip +valgrind +yadm diff --git a/.dockerignore b/.dockerignore index 19c4cc591c..5f8a31589d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,12 +1,11 @@ -################################################################################ -# Repo +# Ignore everything by default +* -.circleci/ -.devcontainer/ -.dockerignore -.git/ -.github/ -.gitignore -**.Dockerfile -**Dockerfile -doc/ +# First-order allow exception for select directories +!/.docker +!/cache +!/nav* +!/tools + +# Second-order deny exception for Dockerfile +/.docker/Dockerfile diff --git a/.gitconfig b/.gitconfig new file mode 100644 index 0000000000..fcb2876001 --- /dev/null +++ b/.gitconfig @@ -0,0 +1,11 @@ +[diff] + submodule = diff +[submodule] + recurse = true + fetchJobs = 8 +[status] + submodulesummary = yes +[advice] + addIgnoredFile = false +[alias] + rinse = !git clean -xfd && git submodule foreach --recursive git clean -xfd && git reset --hard && git submodule foreach --recursive git reset --hard && git submodule update --init --recursive diff --git a/.github/actions/bake-base-stages/action.yml b/.github/actions/bake-base-stages/action.yml new file mode 100644 index 0000000000..76e0573a09 --- /dev/null +++ b/.github/actions/bake-base-stages/action.yml @@ -0,0 +1,158 @@ +name: "Bake base stages" +description: "GitHub Action to bake base stages and push images for CI" +author: 'ros-navigation' +branding: + icon: 'package' + color: 'blue' + +inputs: + integration_repo: + required: true + type: string + description: The repository to checkout for running CI + token: + required: true + type: string + description: The token to use for image registry login + +outputs: + overlay_cache_sha: + description: 'Cache sha to restore overlay workspace cache' + value: ${{ steps.job_workflow_info.outputs.overlay_cache_sha }} + metadata: + description: 'Layer result metadata' + value: ${{ steps.get_layer_metadata.outputs.metadata }} + tooler_image: + description: 'Docker image for building overlay workspace' + value: ${{ steps.job_workflow_info.outputs.tooler_image }} + validator_image: + description: 'Docker image for testing overlay workspace' + value: ${{ steps.job_workflow_info.outputs.validator_image }} + +runs: + using: "composite" + steps: + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + # TODO: Check if default cleanup clears cache on host runner + # and if so, postpone cleanup when baking base stages + # later when baking prod stages if using the same runner + with: + cleanup: false + buildkitd-config-inline: | + [gc] + enabled = false + [worker.oci] + gc = false + [worker.containerd] + gc = false + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.token }} + - name: GHCR Info + shell: bash + run: | + echo "GHCR_REGISTRY=ghcr.io" >> $GITHUB_ENV + echo "INTEGRATION_REPO=${{ inputs.integration_repo }}" >> $GITHUB_ENV + echo "GHCR_REPO=${{ github.repository }}" >> $GITHUB_ENV + + - name: Docker validator meta + id: docker_meta_validator + uses: docker/metadata-action@v5 + with: + images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_REPO }} + flavor: suffix=-validator + bake-target: validator + tags: type=raw,value=${{ github.head_ref || github.ref_name }} + - name: Docker tooler meta + id: docker_meta_tooler + uses: docker/metadata-action@v5 + with: + images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_REPO }} + flavor: suffix=-tooler + bake-target: tooler + tags: type=raw,value=${{ github.head_ref || github.ref_name }} + + - name: Set Extra Bake Args + id: docker_bake_extra + shell: bash + run: | + { + echo 'EXTRA_BAKE_SET<> "$GITHUB_ENV" + - name: Bake base stages + id: bake_base_stages + uses: docker/bake-action@v5 + with: + pull: true + push: true + load: false + provenance: false + no-cache: false + targets: | + tooler + validator + set: | + *.cache-from=type=registry,ref=${{ env.GHCR_REGISTRY }}/${{ env.INTEGRATION_REPO }}:main-tooler.cache + *.cache-to=type=gha,mode=max + ${{ env.EXTRA_BAKE_SET }} + files: | + ./docker-bake.hcl + ${{ steps.docker_meta_tooler.outputs.bake-file }} + ${{ steps.docker_meta_validator.outputs.bake-file }} + + - name: Get Layer Metadata + id: get_layer_metadata + uses: ros-navigation/navigation2/.github/actions/get-layer-metadata@buildkit + with: + metadata: ${{ steps.bake_base_stages.outputs.metadata }} + load: false + - name: Job Workflow Info + id: job_workflow_info + shell: bash + run: | + overlay_cache_sha="${{ fromJSON(steps.get_layer_metadata.outputs.metadata)['tooler']['layer.digest'] }}" + overlay_cache_sha="${overlay_cache_sha#sha256:}" + overlay_cache_sha="$(echo $overlay_cache_sha | cut -c1-12)" + echo "overlay_cache_sha=${overlay_cache_sha}" >> $GITHUB_OUTPUT + + tooler_image_name="${{ fromJSON(steps.get_layer_metadata.outputs.metadata)['tooler']['image.name'] }}" + tooler_image_digest="${{ fromJSON(steps.get_layer_metadata.outputs.metadata)['tooler']['containerimage.digest'] }}" + tooler_image="${tooler_image_name}@${tooler_image_digest}" + echo "tooler_image=${tooler_image}" >> $GITHUB_OUTPUT + + validator_image_name="${{ fromJSON(steps.get_layer_metadata.outputs.metadata)['validator']['image.name'] }}" + validator_image_digest="${{ fromJSON(steps.get_layer_metadata.outputs.metadata)['validator']['containerimage.digest'] }}" + validator_image="${validator_image_name}@${validator_image_digest}" + echo "validator_image=${validator_image}" >> $GITHUB_OUTPUT + + cat << EOF > $GITHUB_STEP_SUMMARY + ## Base Images + | Image tag | Digest | + |-|-| + | \`${tooler_image_name##*:}\` | \`${tooler_image_digest}\` | + | \`${validator_image_name##*:}\` | \`${validator_image_digest}\` | + + ### tooler + \`\`\` + ${tooler_image} + \`\`\` + + ### validator + \`\`\` + ${validator_image} + \`\`\` + EOF diff --git a/.github/actions/bake-prod-stages/action.yml b/.github/actions/bake-prod-stages/action.yml new file mode 100644 index 0000000000..c922302252 --- /dev/null +++ b/.github/actions/bake-prod-stages/action.yml @@ -0,0 +1,151 @@ +name: "Bake prod stages" +description: "GitHub Action to bake prod stages and push images for CD" +author: 'ros-navigation' +branding: + icon: 'package' + color: 'blue' + +inputs: + cache_image: + required: true + type: string + description: The image to cache from via registry + integration_repo: + required: true + type: string + description: The repository to checkout for running CI + token: + required: true + type: string + description: The token to use for image registry login + +outputs: + metadata: + description: 'Layer result metadata' + value: ${{ steps.bake_prod_stages.outputs.metadata }} + releaser_image: + description: 'Docker image for releasing in production' + value: ${{ steps.job_workflow_info.outputs.releaser_image }} + debugger_image: + description: 'Docker image for debugging in production' + value: ${{ steps.job_workflow_info.outputs.debugger_image }} + +runs: + using: "composite" + steps: + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + cleanup: false + buildkitd-config-inline: | + [gc] + enabled = false + [worker.oci] + gc = false + [worker.containerd] + gc = false + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.token }} + - name: GHCR Info + shell: bash + run: | + echo "GHCR_REGISTRY=ghcr.io" >> $GITHUB_ENV + echo "INTEGRATION_REPO=${{ inputs.integration_repo }}" >> $GITHUB_ENV + echo "GHCR_REPO=${{ github.repository }}" >> $GITHUB_ENV + + - name: Docker debugger meta + id: docker_meta_debugger + uses: docker/metadata-action@v5 + with: + images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_REPO }} + flavor: suffix=-debugger + bake-target: debugger + tags: type=raw,value=${{ github.head_ref || github.ref_name }} + - name: Docker releaser meta + id: docker_meta_releaser + uses: docker/metadata-action@v5 + with: + images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_REPO }} + flavor: suffix=-releaser + bake-target: releaser + tags: type=raw,value=${{ github.head_ref || github.ref_name }} + + - name: Set Extra Bake Args + id: docker_bake_extra + shell: bash + run: | + { + echo 'EXTRA_BAKE_SET<> "$GITHUB_ENV" + - name: Bake prod stages + id: bake_prod_stages + uses: docker/bake-action@v5 + with: + pull: true + push: true + load: false + provenance: false + no-cache: false + targets: | + debugger + releaser + set: | + *.args.CLEAR_WS_CACHE=${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} + *.args.SEED_WS_CACHE=${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }} + *.args.EXPORT_FROM_STAGE=seeder + debugger.cache-from=type=registry,ref=${{ env.GHCR_REGISTRY }}/${{ env.INTEGRATION_REPO }}:main-debugger.cache + releaser.cache-from=type=registry,ref=${{ env.GHCR_REGISTRY }}/${{ env.INTEGRATION_REPO }}:main-releaser.cache + ${{ env.EXTRA_BAKE_SET }} + files: | + ./docker-bake.hcl + ${{ steps.docker_meta_debugger.outputs.bake-file }} + ${{ steps.docker_meta_releaser.outputs.bake-file }} + + - name: Job Workflow Info + id: job_workflow_info + shell: bash + run: | + debugger_image_name="${{ fromJSON(steps.bake_prod_stages.outputs.metadata)['debugger']['image.name'] }}" + debugger_image_digest="${{ fromJSON(steps.bake_prod_stages.outputs.metadata)['debugger']['containerimage.digest'] }}" + debugger_image="${debugger_image_name}@${debugger_image_digest}" + echo "debugger_image=${debugger_image}" >> $GITHUB_OUTPUT + + releaser_image_name="${{ fromJSON(steps.bake_prod_stages.outputs.metadata)['releaser']['image.name'] }}" + releaser_image_digest="${{ fromJSON(steps.bake_prod_stages.outputs.metadata)['releaser']['containerimage.digest'] }}" + releaser_image="${releaser_image_name}@${releaser_image_digest}" + echo "releaser_image=${releaser_image}" >> $GITHUB_OUTPUT + + cat << EOF > $GITHUB_STEP_SUMMARY + ## Prod Images + | Image tag | Digest | + |-|-| + | \`${debugger_image_name##*:}\` | \`${debugger_image_digest}\` | + | \`${releaser_image_name##*:}\` | \`${releaser_image_digest}\` | + + ### debugger + \`\`\` + ${debugger_image} + \`\`\` + + ### releaser + \`\`\` + ${releaser_image} + \`\`\` + EOF diff --git a/.github/actions/cache-source-checkout/action.yml b/.github/actions/cache-source-checkout/action.yml new file mode 100644 index 0000000000..02f62d6ce2 --- /dev/null +++ b/.github/actions/cache-source-checkout/action.yml @@ -0,0 +1,57 @@ +name: "Cache Source Checkout" +description: "GitHub Action to cache checkout of source repos" +author: 'ros-navigation' +branding: + icon: 'play-circle' + color: 'blue' + +inputs: + integration_repo: + required: true + type: string + description: The repository to checkout for running CI + overlay_ws: + required: true + type: string + description: The absolute path to the overlay workspace + +outputs: + checkout_cache_key: + description: 'Cache key to restore source checkout cache' + value: ${{ steps.cache_key_info.outputs.checkout_cache_key }} + +runs: + using: "composite" + steps: + - name: Cache key info + id: cache_key_info + shell: bash + run: | + checkout_cache_key="checkout-v1"\ + "-${{ github.ref }}"\ + "-${{ github.run_id }}"\ + "-${{ github.run_number }}"\ + "-${{ github.run_attempt }}" + echo "checkout_cache_key=${checkout_cache_key}" >> $GITHUB_OUTPUT + + - name: Checkout cache dance + id: cache_dance_info + shell: bash + run: | + overlay_ws_src="${{ inputs.overlay_ws }}/src" + echo "overlay_ws_src=${overlay_ws_src}" >> $GITHUB_OUTPUT + rm -rf $overlay_ws_src || true + + integration_repo_name=$(basename ${{ inputs.integration_repo }}) + integration_repo_path="$overlay_ws_src/$integration_repo_name" + mkdir -p $integration_repo_path + + shopt -s dotglob + mv ${{ github.workspace }}/* $integration_repo_path + + - name: Save checkout + id: save_checkout + uses: actions/cache/save@v4 + with: + path: ${{ steps.cache_dance_info.outputs.overlay_ws_src }}/ + key: ${{ steps.cache_key_info.outputs.checkout_cache_key }} diff --git a/.github/actions/get-layer-metadata/README.md b/.github/actions/get-layer-metadata/README.md new file mode 100644 index 0000000000..06fcfed24e --- /dev/null +++ b/.github/actions/get-layer-metadata/README.md @@ -0,0 +1,60 @@ +# get-layer-metadata + +GitHub Action to get layer metadata from Docker Buildx Bake output result. + +## Usage + +```yaml +jobs: + get-layer-metadata: + runs-on: ubuntu-latest + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Build and load + id: docker_bake_load + uses: docker/bake-action@v5 + with: + push: false + load: true + - name: Get Layer Metadata Locally + id: get_layer_metadata_local + uses: ros-navigation/navigation2/.github/actions/get-layer-metadata@buildkit + with: + metadata: ${{ steps.docker_bake_load.outputs.metadata }} + load: true + + - name: Build and push + id: docker_bake_push + uses: docker/bake-action@v5 + with: + push: true + load: false + - name: Get Layer Metadata Remotely + id: get_layer_metadata_remote + uses: ros-navigation/navigation2/.github/actions/get-layer-metadata@buildkit + with: + metadata: ${{ steps.docker_bake_push.outputs.metadata }} + load: false + + - name: Layer metadata + id: layer_metadata + run: | + local_layer_digest="${{ fromJSON(steps.get_layer_metadata_local.outputs.metadata)['']['layer.digest'] }}" + remote_layer_digest="${{ fromJSON(steps.get_layer_metadata_remote.outputs.metadata)['']['layer.digest'] }}" + +``` + +## Inputs + +| Name | Description | +| --- | --- | +| metadata | Build result metadata | +| load | Load is a shorthand to use local registry | + +## Outputs + +| Name | Description | +| --- | --- | +| metadata | Layer result metadata | \ No newline at end of file diff --git a/.github/actions/get-layer-metadata/action.yml b/.github/actions/get-layer-metadata/action.yml new file mode 100644 index 0000000000..cb6647a6c0 --- /dev/null +++ b/.github/actions/get-layer-metadata/action.yml @@ -0,0 +1,50 @@ +name: "Get Layer Metadata" +description: "GitHub Action to get layer metadata from Docker Buildx Bake output result" +author: 'ros-navigation' +branding: + icon: 'layers' + color: 'blue' + +inputs: + metadata: + description: 'Build result metadata' + required: true + load: + description: "Load is a shorthand to use local registry" + required: false + default: 'false' + +outputs: + metadata: + description: 'Layer result metadata' + value: ${{ steps.iterate_metadata.outputs.metadata }} + +runs: + using: "composite" + steps: + - name: Iterate Metadata + id: iterate_metadata + env: + METADATA_INPUT: ${{ inputs.metadata }} + LOAD: ${{ inputs.load }} + shell: bash + run: | + set -eo pipefail + metadata_output=$METADATA_INPUT + for target in $(jq -r 'keys[]' <<< $METADATA_INPUT); do + data=$(jq -r ".${target}" <<< $METADATA_INPUT) + if [[ $LOAD == 'true' ]]; then + image_digest=$(jq -r '."containerimage.config.digest"' <<< $data) + layer_digest=$(docker inspect $image_digest | jq -r '.[0].RootFS.Layers[-1]') + else + image_digest=$(jq -r '."containerimage.digest"' <<< $data) + image_name=$(jq -r '."image.name"' <<< $data) + layer_digest=$(docker buildx imagetools inspect --raw $image_name@$image_digest | jq -r '.layers[-1].digest') + fi + metadata_output=$(jq ".${target}.\"layer.digest\" = \"$layer_digest\"" <<< $metadata_output) + done + { + echo "metadata<> $GITHUB_OUTPUT \ No newline at end of file diff --git a/.github/workflows/build_prod_images.yaml b/.github/workflows/build_prod_images.yaml new file mode 100644 index 0000000000..5d264539b7 --- /dev/null +++ b/.github/workflows/build_prod_images.yaml @@ -0,0 +1,106 @@ +name: Build Prod Images + +on: + workflow_call: + inputs: + ccache_cache_key: + required: true + type: string + checkout_cache_key: + required: true + type: string + overlay_cache_key: + required: true + type: string + overlay_ws: + required: true + type: string + metadata: + required: true + type: string + runs_on: + required: true + type: string + cache_image: + required: true + type: string + outputs: + debugger_image: + value: ${{ jobs.build_prod_images.outputs.debugger_image }} + metadata: + value: ${{ jobs.build_prod_images.outputs.metadata }} + releaser_image: + value: ${{ jobs.build_prod_images.outputs.releaser_image }} + +jobs: + build_prod_images: + name: Build Images + outputs: + debugger_image: ${{ steps.bake_prod_stages.outputs.debugger_image }} + metadata: ${{ steps.bake_prod_stages.outputs.metadata }} + releaser_image: ${{ steps.bake_prod_stages.outputs.releaser_image }} + runs-on: ${{ inputs.runs_on }} + steps: + - name: Workspace cache prep + id: workspace_cache_prep + shell: bash + run: | + rm -rf ${{ inputs.overlay_ws }} || true + rm -rf /home/runner/${{ inputs.overlay_ws }} || true + + - name: Restore checkout + id: restore_checkout + uses: actions/cache/restore@v4 + with: + fail-on-cache-miss: true + path: ${{ inputs.overlay_ws }}/src/ + key: ${{ inputs.checkout_cache_key }} + - name: Restore ccache + id: restore_ccache + uses: actions/cache/restore@v4 + with: + fail-on-cache-miss: true + path: | + ${{ inputs.overlay_ws }}/.ccache + key: ${{ inputs.ccache_cache_key }} + - name: Restore overlay + id: restore_overlay + uses: actions/cache/restore@v4 + with: + fail-on-cache-miss: true + path: | + ${{ inputs.overlay_ws }}/build + ${{ inputs.overlay_ws }}/install + ${{ inputs.overlay_ws }}/test_results + key: ${{ inputs.overlay_cache_key }} + + - name: Workspace cache dance + id: workspace_cache_dance + shell: bash + run: | + overlay_ws_src="${{ inputs.overlay_ws }}/src" + integration_repo_name=$(basename ${{ github.repository }}) + integration_repo_path="$overlay_ws_src/$integration_repo_name" + + shopt -s dotglob + mv $integration_repo_path/* ${{ github.workspace }} + + overlay_ws_dir=$(dirname ${{ inputs.overlay_ws }}) + mkdir -p ${{ github.workspace }}/cache/$overlay_ws_dir + # FIXME: GitHub Cache Action restores caches relative to the working directory + # when the cache was intially saved, I.e. that of the orginating job container. + # Thus the restore_overlay step restores to ../../opt//... + # instead of simply /opt//... as the restore_checkout step does. + # In either case, paths outside of github.workspace could result in crosstalk + # if the same self-hosted runner VM is configured to run multiple concurrent jobs + mv /home/runner/${{ inputs.overlay_ws }} ${{ github.workspace }}/cache/$overlay_ws_dir/ + + tree -L 4 ${{ github.workspace }}/cache + + - name: Bake prod stages + id: bake_prod_stages + uses: ros-navigation/navigation2/.github/actions/bake-prod-stages@buildkit + with: + cache_image: ${{ inputs.cache_image }} + integration_repo: ${{ github.repository }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/build_test.yaml b/.github/workflows/build_test.yaml new file mode 100644 index 0000000000..1543f9fa90 --- /dev/null +++ b/.github/workflows/build_test.yaml @@ -0,0 +1,30 @@ +name: Call + +on: + push: + branches: + - main + paths: + - '.docker/**' + - '.github/workflows/build_prod_images.yaml' + - '.github/workflows/build_test_colcon.yaml' + - '.github/workflows/build_test_integration.yaml' + - '.github/workflows/build_test.yaml' + - 'docker-bake.hcl' + - 'nav*/**' + pull_request: + paths: + - '.docker/**' + - '.github/workflows/build_prod_images.yaml' + - '.github/workflows/build_test_colcon.yaml' + - '.github/workflows/build_test_integration.yaml' + - '.github/workflows/build_test.yaml' + - 'docker-bake.hcl' + - 'nav*/**' + workflow_dispatch: + +jobs: + integration: + name: Integration + uses: ros-navigation/navigation2/.github/workflows/build_test_integration.yaml@buildkit + secrets: inherit diff --git a/.github/workflows/build_test_colcon.yaml b/.github/workflows/build_test_colcon.yaml new file mode 100644 index 0000000000..600f1ef370 --- /dev/null +++ b/.github/workflows/build_test_colcon.yaml @@ -0,0 +1,245 @@ +name: Colcon Build and Test + +on: + workflow_call: + inputs: + checkout_cache_key: + required: true + type: string + overlay_cache_sha: + required: true + type: string + overlay_ws: + required: true + type: string + metadata: + required: true + type: string + runs_on: + required: true + type: string + tooler_image: + required: true + type: string + validator_image: + required: true + type: string + outputs: + build_result: + value: ${{ jobs.build_colcon_workspace.result }} + ccache_cache_key: + value: ${{ jobs.build_colcon_workspace.outputs.ccache_cache_key }} + overlay_cache_key: + value: ${{ jobs.test_colcon_workspace.outputs.overlay_cache_key }} + test_result: + value: ${{ jobs.test_colcon_workspace.result }} + +jobs: + build_colcon_workspace: + name: Build Workspace + outputs: + ccache_cache_key: ${{ steps.cache_ccache_info.outputs.ccache_cache_key }} + overlay_cache_key: ${{ steps.cache_overlay_info.outputs.overlay_cache_key }} + build_packages: ${{ steps.check_differential.outputs.build_packages }} + runs-on: ${{ inputs.runs_on }} + container: + image: ${{ inputs.tooler_image }} + defaults: + run: + shell: bash + working-directory: ${{ inputs.overlay_ws }}/ + steps: + - name: Restore checkout + id: restore_checkout + uses: actions/cache/restore@v4 + with: + fail-on-cache-miss: true + path: ${{ inputs.overlay_ws }}/src/ + key: ${{ inputs.checkout_cache_key }} + - name: Cache ccache info + id: cache_ccache_info + run: | + ccache_cache_key="ccache-v1"\ + "-${{ github.ref }}"\ + "-${{ github.run_id }}"\ + "-${{ github.run_number }}"\ + "-${{ github.run_attempt }}" + echo "ccache_cache_key=${ccache_cache_key}" >> $GITHUB_OUTPUT + - name: Restore ccache + id: restore_ccache + uses: actions/cache/restore@v4 + with: + path: | + ${{ inputs.overlay_ws }}/.ccache + key: ${{ steps.cache_ccache_info.outputs.ccache_cache_key }} + restore-keys: | + ccache-v1-${{ github.ref }} + ccache-v1-refs/heads/${{ github.head_ref }} + ccache-v1-refs/heads/${{ github.base_ref }} + - name: Cache overlay info + id: cache_overlay_info + run: | + overlay_cache_key="overlay-v1"\ + "-${{ github.ref }}"\ + "-${{ inputs.overlay_cache_sha }}"\ + "-${{ github.run_id }}"\ + "-${{ github.run_number }}"\ + "-${{ github.run_attempt }}" + echo "overlay_cache_key=${overlay_cache_key}" >> $GITHUB_OUTPUT + - name: Restore overlay + id: restore_overlay + uses: actions/cache/restore@v4 + with: + path: | + ${{ inputs.overlay_ws }}/build + ${{ inputs.overlay_ws }}/install + ${{ inputs.overlay_ws }}/test_results + key: ${{ steps.cache_overlay_info.outputs.overlay_cache_key }} + restore-keys: | + overlay-v1-${{ github.ref }}-${{ inputs.overlay_cache_sha }} + overlay-v1-refs/heads/${{ github.head_ref }}-${{ inputs.overlay_cache_sha }} + overlay-v1-refs/heads/${{ github.base_ref }}-${{ inputs.overlay_cache_sha }} + - name: Check Differential + id: check_differential + shell: bash + run: | + marked_base_paths=$(find ./src -name "COLCON_MARK" -exec dirname {} \; | xargs) + if [ -n "$marked_base_paths" ]; then + echo "Differential build detected!" + base_paths=$marked_base_paths + else + echo "Defaulting full build..." + base_paths='./src' + fi + BUILD_PACKAGES=$( + colcon list \ + --names-only \ + --base-paths $base_paths \ + | xargs) + echo BUILD_PACKAGES: $BUILD_PACKAGES + echo "build_packages=$BUILD_PACKAGES" >> $GITHUB_OUTPUT + - name: Ccache Info + run: | + ccache -z # zero stats + ccache -V # show version + ccache -p # show config + - name: Colcon Build + id: colcon_build + run: | + . /opt/ros/$ROS_DISTRO/setup.sh + colcon cache lock \ + --dirhash-reset + colcon clean packages -y \ + --packages-select-cache-invalid \ + --packages-select-cache-key build \ + --base-select install + colcon build \ + --packages-up-to \ + ${{ steps.check_differential.outputs.build_packages }} \ + --packages-skip-cache-valid \ + --mixin $OVERLAY_MIXINS + - name: Ccache Stats + if: always() + run: | + ccache -s # show stats + - name: Upload Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: build-logs + path: ${{ inputs.overlay_ws }}/log/ + - name: Save ccache + id: save_ccache + uses: actions/cache/save@v4 + if: always() + with: + path: | + ${{ inputs.overlay_ws }}/.ccache + key: ${{ steps.cache_ccache_info.outputs.ccache_cache_key }} + - name: Save overlay + id: save_overlay + uses: actions/cache/save@v4 + if: always() + with: + path: | + ${{ inputs.overlay_ws }}/build + ${{ inputs.overlay_ws }}/install + ${{ inputs.overlay_ws }}/test_results + key: ${{ steps.cache_overlay_info.outputs.overlay_cache_key }} + + test_colcon_workspace: + name: Test Workspace + outputs: + overlay_cache_key: ${{ needs.build_colcon_workspace.outputs.overlay_cache_key }}-${{ github.run_attempt }} + runs-on: ${{ inputs.runs_on }} + needs: + - build_colcon_workspace + container: + image: ${{ inputs.validator_image }} + defaults: + run: + shell: bash + working-directory: ${{ inputs.overlay_ws }}/ + steps: + - name: Restore checkout + id: restore_checkout + uses: actions/cache/restore@v4 + with: + fail-on-cache-miss: true + path: ${{ inputs.overlay_ws }}/src/ + key: ${{ inputs.checkout_cache_key }} + - name: Restore overlay + id: restore_overlay + uses: actions/cache/restore@v4 + with: + path: | + ${{ inputs.overlay_ws }}/build + ${{ inputs.overlay_ws }}/install + ${{ inputs.overlay_ws }}/test_results + key: ${{ needs.build_colcon_workspace.outputs.overlay_cache_key }}-${{ github.run_attempt }} + restore-keys: | + ${{ needs.build_colcon_workspace.outputs.overlay_cache_key }} + - name: Colcon Test + id: colcon_test + run: | + . install/setup.sh + colcon test \ + --packages-select \ + ${{ needs.build_colcon_workspace.outputs.build_packages }} \ + --packages-skip-cache-valid + TEST_FAILURES=$( + colcon list \ + --names-only \ + --packages-select-test-failures \ + | xargs) + echo TEST_FAILURES: $TEST_FAILURES + TEST_UNFINISHED=$( + colcon list \ + --names-only \ + --packages-skip-test-passed \ + | xargs) + echo TEST_UNFINISHED: $TEST_UNFINISHED + colcon test-result \ + --verbose + - name: Upload Logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-logs + path: ${{ inputs.overlay_ws }}/log/ + - name: Upload Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: test-results + path: ${{ inputs.overlay_ws }}/test_results/ + - name: Save overlay + id: save_overlay + uses: actions/cache/save@v4 + if: always() + with: + path: | + ${{ inputs.overlay_ws }}/build + ${{ inputs.overlay_ws }}/install + ${{ inputs.overlay_ws }}/test_results + key: ${{ needs.build_colcon_workspace.outputs.overlay_cache_key }}-${{ github.run_attempt }} diff --git a/.github/workflows/build_test_integration.yaml b/.github/workflows/build_test_integration.yaml new file mode 100644 index 0000000000..c3f3f0011e --- /dev/null +++ b/.github/workflows/build_test_integration.yaml @@ -0,0 +1,98 @@ +name: Integration Build and Test + +on: + workflow_call: + inputs: + integration_ref: + required: false + type: string + default: ${{ github.sha }} + integration_repo: + required: false + type: string + default: ${{ github.repository }} + runs_on: + required: false + type: string + default: ubuntu-latest + workflow_dispatch: + inputs: + integration_ref: + required: true + type: string + integration_repo: + required: false + type: string + default: 'ros-navigation/navigation2' + runs_on: + required: true + type: string + +env: + OVERLAY_WS: /opt/nav2_ws + +jobs: + build_base_images: + name: Build Base Images + outputs: + checkout_cache_key: ${{ steps.cache_source_checkout.outputs.checkout_cache_key }} + metadata: ${{ steps.bake_base_stages.outputs.metadata }} + overlay_cache_sha: ${{ steps.bake_base_stages.outputs.overlay_cache_sha }} + overlay_ws: ${{ env.OVERLAY_WS }} + tooler_image: ${{ steps.bake_base_stages.outputs.tooler_image }} + validator_image: ${{ steps.bake_base_stages.outputs.validator_image }} + runs-on: ${{ inputs.runs_on }} + steps: + - name: Checkout Integration Repo + id: checkout_integration_repo + uses: actions/checkout@v4 + with: + ref: ${{ inputs.integration_ref }} + repository: ${{ inputs.integration_repo }} + token: ${{ secrets.GITHUB_TOKEN }} + submodules: 'recursive' + + - name: Bake base stages + id: bake_base_stages + uses: ros-navigation/navigation2/.github/actions/bake-base-stages@buildkit + with: + integration_repo: ${{ inputs.integration_repo }} + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Cache source checkout + id: cache_source_checkout + uses: ros-navigation/navigation2/.github/actions/cache-source-checkout@buildkit + with: + integration_repo: ${{ inputs.integration_repo }} + overlay_ws: ${{ env.OVERLAY_WS }} + + colcon_build_test: + name: Colcon + uses: ros-navigation/navigation2/.github/workflows/build_test_colcon.yaml@buildkit + secrets: inherit + needs: build_base_images + with: + checkout_cache_key: ${{ needs.build_base_images.outputs.checkout_cache_key }} + metadata: ${{ needs.build_base_images.outputs.metadata }} + overlay_cache_sha: ${{ needs.build_base_images.outputs.overlay_cache_sha }} + overlay_ws: ${{ needs.build_base_images.outputs.overlay_ws }} + runs_on: ${{ inputs.runs_on }} + tooler_image: ${{ needs.build_base_images.outputs.tooler_image }} + validator_image: ${{ needs.build_base_images.outputs.validator_image }} + + build_prod_images: + name: Prod + uses: ros-navigation/navigation2/.github/workflows/build_prod_images.yaml@buildkit + if: ${{ always() && (needs.colcon_build_test.outputs.build_result == 'success') }} + secrets: inherit + needs: + - build_base_images + - colcon_build_test + with: + cache_image: ${{ needs.build_base_images.outputs.tooler_image }} + ccache_cache_key: ${{ needs.colcon_build_test.outputs.ccache_cache_key }} + checkout_cache_key: ${{ needs.build_base_images.outputs.checkout_cache_key }} + metadata: ${{ needs.build_base_images.outputs.metadata }} + overlay_cache_key: ${{ needs.colcon_build_test.outputs.overlay_cache_key }} + overlay_ws: ${{ needs.build_base_images.outputs.overlay_ws }} + runs_on: ${{ inputs.runs_on }} diff --git a/.github/workflows/ghac_eviction.yaml b/.github/workflows/ghac_eviction.yaml new file mode 100644 index 0000000000..8a74e2a57b --- /dev/null +++ b/.github/workflows/ghac_eviction.yaml @@ -0,0 +1,47 @@ +name: GH Action Cache Eviction Policy + +on: + pull_request: + types: + - closed + workflow_dispatch: + +jobs: + clean-ghac: + name: Delete old unused action caches + runs-on: ubuntu-latest + permissions: + actions: write + contents: read + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Cleanup + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + BRANCH=refs/pull/${{ github.event.pull_request.number }}/merge + + # loop until the list is empty, because it deletes only 30 per page + has_items=true + while [ "$has_items" = true ] + do + echo "Fetching list of cache key" + cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 ) + if [ -z "$cache_keys" ]; then + has_items=false + fi + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm + done + done + echo "Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ghcr_retention.yaml b/.github/workflows/ghcr_retention.yaml new file mode 100644 index 0000000000..db4c276213 --- /dev/null +++ b/.github/workflows/ghcr_retention.yaml @@ -0,0 +1,58 @@ +name: GHCR Container Retention Policy + +on: + pull_request: + types: + - closed + schedule: + - cron: "0 0 * * *" # every day at midnight + workflow_dispatch: + +jobs: + clean-old-ghcr: + name: Delete old container images + if: github.event_name == 'schedule' + runs-on: ubuntu-latest + steps: + - name: Delete untagged containers + uses: snok/container-retention-policy@v2.2.1 + with: + image-names: navigation2 + cut-off: A day ago UTC + account-type: org + org-name: ros-navigation + # keep-at-least: 0 + untagged-only: true + token: ${{ secrets.GHCR_RETENTION_PAT }} + dry-run: false + clean-unused-ghcr: + name: Delete unused container images + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: GHCR Info + shell: bash + run: | + echo "GHCR_REGISTRY=ghcr.io" >> $GITHUB_ENV + echo "GHCR_REPO=${{ github.repository }}" >> $GITHUB_ENV + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_REPO }} + flavor: suffix=- + tags: type=raw,value=${{ github.head_ref || github.ref_name }} + - name: Delete untagged containers + uses: snok/container-retention-policy@v2.2.1 + with: + image-names: navigation2 + cut-off: now UTC + account-type: org + org-name: ros-navigation + # keep-at-least: 0 + # TODO: come up with a better way to filter out the PRs + # as well as a better tag naming convention + filter-tags: "${{ fromJSON(steps.docker_meta.outputs.json).tags[0] }}*" + untagged-only: false + token: ${{ secrets.GHCR_RETENTION_PAT }} + dry-run: false diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c181fe25f4..9d2c8ac738 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,6 +1,6 @@ name: Lint on: - pull_request: + # pull_request: jobs: ament_lint_general: diff --git a/.github/workflows/update_ci_image.yaml b/.github/workflows/update_ci_image.yaml index 16d172aff8..e3af76aae2 100644 --- a/.github/workflows/update_ci_image.yaml +++ b/.github/workflows/update_ci_image.yaml @@ -6,13 +6,13 @@ on: # 7am UTC, 12am PDT - cron: '0 7 * * *' push: - branches: - - main - paths: - - '**/package.xml' - - '**/*.repos' - - 'Dockerfile' - - '.github/workflows/update_ci_image.yaml' + # branches: + # - main + # paths: + # - '**/package.xml' + # - '**/*.repos' + # - 'Dockerfile' + # - '.github/workflows/update_ci_image.yaml' jobs: check_ci_files: diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000..9ccb2025a0 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "nav2_minimal_turtlebot_simulation"] + path = tools/underlay/nav2_minimal_turtlebot_simulation + url = https://github.com/ros-navigation/nav2_minimal_turtlebot_simulation.git diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 1f959ff1f7..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,226 +0,0 @@ -# This dockerfile can be configured via --build-arg -# Build context must be the /navigation2 root folder for COPY. -# Example build command: -# export UNDERLAY_MIXINS="debug ccache lld" -# export OVERLAY_MIXINS="debug ccache coverage-gcc lld" -# docker build -t nav2:latest \ -# --build-arg UNDERLAY_MIXINS \ -# --build-arg OVERLAY_MIXINS ./ -ARG FROM_IMAGE=ros:rolling -ARG UNDERLAY_WS=/opt/underlay_ws -ARG OVERLAY_WS=/opt/overlay_ws - -# multi-stage for caching -FROM $FROM_IMAGE AS cacher - -# clone underlay source -ARG UNDERLAY_WS -WORKDIR $UNDERLAY_WS/src -COPY ./tools/underlay.repos ../ -RUN vcs import ./ < ../underlay.repos - -# copy overlay source -ARG OVERLAY_WS -WORKDIR $OVERLAY_WS/src -COPY ./ ./navigation2 - -# copy manifests for caching -WORKDIR /opt -RUN find . -name "src" -type d \ - -mindepth 1 -maxdepth 2 -printf '%P\n' \ - | xargs -I % mkdir -p /tmp/opt/% && \ - find . -name "package.xml" \ - | xargs cp --parents -t /tmp/opt && \ - find . -name "COLCON_IGNORE" \ - | xargs cp --parents -t /tmp/opt || true - -# multi-stage for building -FROM $FROM_IMAGE AS builder - -# config dependencies install -ARG DEBIAN_FRONTEND=noninteractive -RUN echo '\ -APT::Install-Recommends "0";\n\ -APT::Install-Suggests "0";\n\ -' > /etc/apt/apt.conf.d/01norecommend -ENV PYTHONUNBUFFERED 1 - -# install CI dependencies -ARG RTI_NC_LICENSE_ACCEPTED=yes -RUN apt-get update && \ - apt-get upgrade -y --with-new-pkgs && \ - apt-get install -y \ - ccache \ - lcov \ - lld \ - python3-pip \ - ros-$ROS_DISTRO-rmw-fastrtps-cpp \ - ros-$ROS_DISTRO-rmw-connextdds \ - ros-$ROS_DISTRO-rmw-cyclonedds-cpp \ - && pip3 install --break-system-packages \ - fastcov \ - git+https://github.com/ruffsl/colcon-cache.git@a937541bfc496c7a267db7ee9d6cceca61e470ca \ - git+https://github.com/ruffsl/colcon-clean.git@a7f1074d1ebc1a54a6508625b117974f2672f2a9 \ - && rosdep update \ - && colcon mixin update \ - && colcon metadata update \ - && rm -rf /var/lib/apt/lists/* - -# install underlay dependencies -ARG UNDERLAY_WS -ENV UNDERLAY_WS $UNDERLAY_WS -WORKDIR $UNDERLAY_WS -COPY --from=cacher /tmp/$UNDERLAY_WS ./ -RUN . /opt/ros/$ROS_DISTRO/setup.sh && \ - apt-get update && rosdep install -q -y \ - --from-paths src \ - --skip-keys " \ - slam_toolbox \ - " \ - --ignore-src \ - && rm -rf /var/lib/apt/lists/* - -# build underlay source -COPY --from=cacher $UNDERLAY_WS ./ -ARG UNDERLAY_MIXINS="release ccache lld" -ARG CCACHE_DIR="$UNDERLAY_WS/.ccache" -RUN . /opt/ros/$ROS_DISTRO/setup.sh && \ - colcon cache lock && \ - colcon build \ - --symlink-install \ - --mixin $UNDERLAY_MIXINS \ - --event-handlers console_direct+ - -# install overlay dependencies -ARG OVERLAY_WS -ENV OVERLAY_WS $OVERLAY_WS -WORKDIR $OVERLAY_WS -COPY --from=cacher /tmp/$OVERLAY_WS ./ - -RUN . $UNDERLAY_WS/install/setup.sh && \ - apt-get update && rosdep install -q -y \ - --from-paths src \ - --skip-keys " \ - slam_toolbox \ - "\ - --ignore-src \ - && rm -rf /var/lib/apt/lists/* - -# multi-stage for testing -FROM builder AS tester - -# build overlay source -COPY --from=cacher $OVERLAY_WS ./ -ARG OVERLAY_MIXINS="release ccache lld" -ARG CCACHE_DIR="$OVERLAY_WS/.ccache" -RUN . $UNDERLAY_WS/install/setup.sh && \ - colcon cache lock && \ - colcon build \ - --symlink-install \ - --mixin $OVERLAY_MIXINS - -# source overlay from entrypoint -RUN sed --in-place \ - 's|^source .*|source "$OVERLAY_WS/install/setup.bash"|' \ - /ros_entrypoint.sh - -# test overlay build -ARG RUN_TESTS -ARG FAIL_ON_TEST_FAILURE -RUN if [ -n "$RUN_TESTS" ]; then \ - . install/setup.sh && \ - colcon test && \ - colcon test-result \ - || ([ -z "$FAIL_ON_TEST_FAILURE" ] || exit 1) \ - fi - -# multi-stage for developing -FROM builder AS dever - -# edit apt for caching -RUN mv /etc/apt/apt.conf.d/docker-clean /etc/apt/ - -# install developer dependencies -RUN apt-get update && \ - apt-get install -y \ - bash-completion \ - gdb \ - wget && \ - pip3 install --break-system-packages \ - bottle \ - glances - -# source underlay for shell -RUN echo 'source "$UNDERLAY_WS/install/setup.bash"' >> /etc/bash.bashrc - -# multi-stage for caddy -FROM caddy:builder AS caddyer - -# build custom modules -RUN xcaddy build \ - --with github.com/caddyserver/replace-response - -# multi-stage for visualizing -FROM dever AS visualizer - -ENV ROOT_SRV /srv -RUN mkdir -p $ROOT_SRV - -# install demo dependencies -RUN apt-get update && apt-get install -y \ - ros-$ROS_DISTRO-rviz2 - -# install gzweb dependacies -RUN apt-get install -y --no-install-recommends \ - imagemagick \ - libboost-all-dev \ - libgazebo-dev \ - libgts-dev \ - libjansson-dev \ - libtinyxml-dev \ - nodejs \ - npm \ - psmisc \ - xvfb - -# clone gzweb -ENV GZWEB_WS /opt/gzweb -RUN git clone https://github.com/osrf/gzweb.git $GZWEB_WS - -# setup gzweb -RUN cd $GZWEB_WS && . /usr/share/gazebo/setup.sh && \ - GAZEBO_MODEL_PATH=$GAZEBO_MODEL_PATH:$(find /opt/ros/$ROS_DISTRO/share \ - -mindepth 1 -maxdepth 2 -type d -name "models" | paste -s -d: -) && \ - sed -i "s|var modelList =|var modelList = []; var oldModelList =|g" gz3d/src/gzgui.js && \ - xvfb-run -s "-screen 0 1280x1024x24" ./deploy.sh -m local && \ - ln -s $GZWEB_WS/http/client/assets http/client/assets/models && \ - ln -s $GZWEB_WS/http/client $ROOT_SRV/gzweb - -# patch gzsever -RUN GZSERVER=$(which gzserver) && \ - mv $GZSERVER $GZSERVER.orig && \ - echo '#!/bin/bash' > $GZSERVER && \ - echo 'exec xvfb-run -s "-screen 0 1280x1024x24" gzserver.orig "$@"' >> $GZSERVER && \ - chmod +x $GZSERVER - -# install foxglove dependacies -RUN apt-get install -y --no-install-recommends \ - ros-$ROS_DISTRO-foxglove-bridge - -# setup foxglove -# Use custom fork until PR is merged: -# https://github.com/foxglove/studio/pull/5987 -# COPY --from=ghcr.io/foxglove/studio /src $ROOT_SRV/foxglove -COPY --from=ghcr.io/ruffsl/foxglove_studio@sha256:8a2f2be0a95f24b76b0d7aa536f1c34f3e224022eed607cbf7a164928488332e /src $ROOT_SRV/foxglove - -# install web server -COPY --from=caddyer /usr/bin/caddy /usr/bin/caddy - -# download media files -RUN mkdir -p $ROOT_SRV/media && cd /tmp && \ - export ICONS="icons.tar.gz" && wget https://github.com/ros-planning/navigation2/files/11506823/$ICONS && \ - echo "cae5e2a5230f87b004c8232b579781edb4a72a7431405381403c6f9e9f5f7d41 $ICONS" | sha256sum -c && \ - tar xvz -C $ROOT_SRV/media -f $ICONS && rm $ICONS - -# multi-stage for exporting -FROM tester AS exporter diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 0000000000..c1cc5301a1 --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,153 @@ +variable "repo" { + default = "navigation2" +} +variable "registry" { + default = "ghcr.io" +} + +group "default" { + targets = ["tooler"] +} + +################################################################################ +// MARK: General targets +################################################################################ + +target "baser" { + target = "baser" + tags = ["nav2:baser"] + pull = true + no-cache = false + cache-from = [ + // "type=registry,ref=${registry}/${repo}:main-tooler.cache", + ] + dockerfile = ".docker/Dockerfile" +} + +target "cacher" { + inherits = ["baser"] + target = "cacher" + tags = ["nav2:cacher"] +} + +target "runner" { + inherits = ["baser"] + target = "runner" + tags = ["nav2:runner"] +} + +target "prepper" { + inherits = ["runner"] + target = "prepper" + tags = ["nav2:prepper"] +} + +target "validator" { + inherits = ["prepper"] + target = "validator" + tags = ["nav2:validator"] +} + +target "tooler" { + inherits = ["validator"] + target = "tooler" + tags = ["nav2:tooler"] +} + +################################################################################ +// MARK: Development targets +################################################################################ + +variable "DEV_FROM_STAGE" { + default = "tooler" +} + +target "dever" { + inherits = ["tooler"] + target = "dever" + tags = ["nav2:dever"] + args = { + DEV_FROM_STAGE = "${DEV_FROM_STAGE}", + } +} + +target "seeder" { + inherits = ["dever"] + target = "seeder" + tags = ["nav2:seeder"] + // no-cache-filter = ["builder"] + args = { + CLEAR_WS_CACHE = null, + // CLEAR_WS_CACHE = "${timestamp()}", + SEED_WS_CACHE = null, + // SEED_WS_CACHE = "${timestamp()}", + } +} + +target "builder" { + inherits = ["seeder"] + target = "builder" + tags = ["nav2:builder"] + // no-cache-filter = ["builder"] + args = { + BUST_BUILD_CACHE = null, + // BUST_BUILD_CACHE = "${timestamp()}", + } +} + +target "tester" { + inherits = ["builder"] + target = "tester" + tags = ["nav2:tester"] + args = { + BUST_TEST_CACHE = null, + // BUST_TEST_CACHE = "${timestamp()}", + } +} + +target "dancer" { + inherits = ["builder"] + target = "dancer" + tags = ["nav2:dancer"] +} + +target "exporter" { + inherits = ["dancer"] + target = "exporter" + tags = ["nav2:exporter"] +} + +################################################################################ +// MARK: Production targets +################################################################################ + +target "shipper" { + inherits = ["dancer"] + target = "shipper" + args = { + } +} + +target "releaser" { + inherits = ["shipper"] + target = "releaser" + tags = ["nav2:releaser"] + args = { + SHIP_FROM_STAGE = "runner", + } + cache-from = [ + // "type=registry,ref=${registry}/${repo}:main-releaser.cache", + ] +} + +target "debugger" { + inherits = ["shipper"] + target = "debugger" + tags = ["nav2:debugger"] + args = { + SHIP_FROM_STAGE = "tooler", + } + cache-from = [ + // "type=registry,ref=${registry}/${repo}:main-debugger.cache", + ] +} diff --git a/tools/underlay/nav2_minimal_turtlebot_simulation b/tools/underlay/nav2_minimal_turtlebot_simulation new file mode 160000 index 0000000000..6b64127f0e --- /dev/null +++ b/tools/underlay/nav2_minimal_turtlebot_simulation @@ -0,0 +1 @@ +Subproject commit 6b64127f0e0d677ecdaa458bce57b89119cb08ee