From 80c97cfd7e66f48d3649b2113f6bea4137350c31 Mon Sep 17 00:00:00 2001 From: Joonas Loppi Date: Thu, 24 Mar 2022 12:59:29 +0200 Subject: [PATCH] simplified build process --- .github/workflows/main.yaml | 63 ------------ .github/workflows/tii-mocap-pose.yaml | 46 +-------- Dockerfile | 54 ++++------- Dockerfile.build_env | 45 --------- build.sh | 40 -------- entrypoint.sh | 6 -- packaging/build_deps.sh | 51 ---------- packaging/package.sh | 134 -------------------------- packaging/rosdep.sh | 34 ------- packaging/rosdep.yaml | 2 +- tasks.py | 131 ------------------------- underlay.repos | 4 - 12 files changed, 27 insertions(+), 583 deletions(-) delete mode 100644 .github/workflows/main.yaml delete mode 100644 Dockerfile.build_env delete mode 100755 build.sh delete mode 100644 entrypoint.sh delete mode 100755 packaging/build_deps.sh delete mode 100755 packaging/package.sh delete mode 100755 packaging/rosdep.sh delete mode 100644 tasks.py delete mode 100644 underlay.repos diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml deleted file mode 100644 index d65e34f..0000000 --- a/.github/workflows/main.yaml +++ /dev/null @@ -1,63 +0,0 @@ -name: main - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - tii-deb-build: - runs-on: ubuntu-latest - strategy: - matrix: - ros2_distro: [galactic] - steps: - - - name: Checkout mocap_pose - uses: actions/checkout@v2 - with: - path: mocap_pose - submodules: recursive - - # Run docker build - - name: Run mocap_pose docker build - env: - ROS: 1 - ROS_DISTRO: ${{ matrix.ros2_distro }} - PACKAGE_NAME: mocap_pose - run: | - set -eux - mkdir bin - pushd mocap_pose - ./build.sh ../bin/ - popd - - - uses: jfrog/setup-jfrog-cli@v2 - env: - JF_ARTIFACTORY_1: ${{ secrets.ARTIFACTORY_TOKEN }} - - - name: Upload to Artifactory - env: - ARTIFACTORY_REPO: debian-public-local - DISTRIBUTION: focal - COMPONENT: fog-sw - ARCHITECTURE: amd64 - BUILD_NAME: mocap-pose - CI: true - if: github.event_name == 'push' - run: | - set -exu - jfrog rt ping - pkg=$(find bin -name 'ros-${{ matrix.ros2_distro }}-mocap-pose*.deb') - pkg_name=$(basename $pkg) - jfrog rt u --deb "$DISTRIBUTION/$COMPONENT/$ARCHITECTURE" \ - --target-props COMMIT="$GITHUB_SHA" \ - --build-name "$BUILD_NAME" \ - --build-number "$GITHUB_SHA" \ - "$pkg" \ - "$ARTIFACTORY_REPO/$pkg_name" - jfrog rt build-publish "$BUILD_NAME" "$GITHUB_SHA" - jfrog rt bpr "$BUILD_NAME" "$GITHUB_SHA" "$ARTIFACTORY_REPO" \ - --status dev \ - --comment "development build" diff --git a/.github/workflows/tii-mocap-pose.yaml b/.github/workflows/tii-mocap-pose.yaml index e75ca88..0c28c78 100644 --- a/.github/workflows/tii-mocap-pose.yaml +++ b/.github/workflows/tii-mocap-pose.yaml @@ -2,47 +2,17 @@ name: tii-mocap-pose on: push: - branches: main - tags: - - 'v*' - - v[0-9]+.[0-9]+.[0-9]+ - - v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+ - pull_request: - branches: [ main ] jobs: - tii-mocap-pose: + build: runs-on: ubuntu-latest - services: - registry: - image: registry:2 - ports: - - 5000:5000 steps: - - - name: Checkout mocap_pose - uses: actions/checkout@v2 + - uses: actions/checkout@v2 with: - submodules: recursive + submodules: true - uses: docker/setup-buildx-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - driver-opts: network=host - - - name: Build the builder container image - uses: docker/build-push-action@v2 - with: - context: . - file: ./Dockerfile.build_env - push: true - tags: localhost:5000/tiiuae/mocap_pose:build_env - build-args: | - PACKAGE_NAME=mocap_pose - ROS_DISTRO=galactic - - name: Docker meta id: meta uses: docker/metadata-action@v3 @@ -56,21 +26,15 @@ jobs: - name: Login to GitHub Container Registry uses: docker/login-action@v1 - if: github.event_name == 'push' with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build tii-mocap-pose image and push + - name: Build container image and push uses: docker/build-push-action@v2 with: context: . - file: ./Dockerfile - push: ${{ github.event_name != 'pull_request' }} + push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - build-args: | - PACKAGE_NAME=mocap_pose - ROS_DISTRO=galactic - FROM_IMAGE=localhost:5000/tiiuae/mocap_pose:build_env diff --git a/Dockerfile b/Dockerfile index e20505e..31c0d12 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,21 @@ -# fog-sw BUILDER -ARG FROM_IMAGE -FROM $FROM_IMAGE as fog-sw-builder -ARG ROS_DISTRO="galactic" -ARG UID=1000 -ARG GID=1000 -ARG PACKAGE_NAME - -WORKDIR /$PACKAGE_NAME/main_ws -USER root -ADD . /$PACKAGE_NAME/main_ws/src -RUN chown -R builder:builder /$PACKAGE_NAME/main_ws - -USER builder - -RUN if [ -e /$PACKAGE_NAME/deps_ws ]; then \ - . /$PACKAGE_NAME/deps_ws/install/setup.sh && \ - colcon build; \ - elif [ -e /opt/ros/${ROS_DISTRO}/setup.sh ]; then \ - . /opt/ros/${ROS_DISTRO}/setup.sh && \ - colcon build; \ - fi - -RUN sed --in-place \ - 's|^source .*|source "/'$PACKAGE_NAME'/main_ws/install/setup.bash"|' \ - /$PACKAGE_NAME/entrypoint.sh && \ - chmod +x /$PACKAGE_NAME/entrypoint.sh - -ENV PACKAGE_NAME $PACKAGE_NAME -ENV RMW_IMPLEMENTATION rmw_fastrtps_cpp - -WORKDIR /$PACKAGE_NAME -ENTRYPOINT "/"$PACKAGE_NAME"/entrypoint.sh" +FROM ghcr.io/tiiuae/fog-ros-baseimage:builder-latest AS builder + +COPY . /main_ws/src/ + +# this: +# 1) builds the application +# 2) packages the application as .deb in build_output/ + +# RUN /packaging/build-and-package-as-deb.sh -o build_output/ +RUN /packaging/build.sh + +# ▲ runtime ──┐ +# └── build ▼ + +FROM ghcr.io/tiiuae/fog-ros-baseimage:sha-d2cdcdb + +ENTRYPOINT exec ros-with-env ros2 launch mocap_pose mocap_pose.launch + +COPY --from=builder /main_ws/ros-*-mocap-pose_*_amd64.deb /mocap-pose.deb + +RUN dpkg -i /mocap-pose.deb && rm /mocap-pose.deb diff --git a/Dockerfile.build_env b/Dockerfile.build_env deleted file mode 100644 index 1db5ff3..0000000 --- a/Dockerfile.build_env +++ /dev/null @@ -1,45 +0,0 @@ -# fog-sw BUILDER -ARG ROS_DISTRO="galactic" -FROM ros:${ROS_DISTRO}-ros-base as fog-sw-builder - -ARG UID=1000 -ARG GID=1000 -ARG BUILD_NUMBER -ARG COMMIT_ID -ARG GIT_VER -ARG PACKAGE_NAME -# Install build dependencies -RUN apt-get update -y && apt-get install -y --no-install-recommends \ - curl \ - python3-bloom \ - fakeroot \ - dh-make \ - libboost-dev \ - ros-${ROS_DISTRO}-rmw-fastrtps-cpp \ - && rm -rf /var/lib/apt/lists/* - -RUN groupadd -g $GID builder && \ - useradd -m -u $UID -g $GID -g builder builder && \ - usermod -aG sudo builder && \ - echo 'builder ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -RUN echo "deb [trusted=yes] https://ssrc.jfrog.io/artifactory/ssrc-debian-public-remote focal fog-sw" >> /etc/apt/sources.list - -RUN mkdir -p /$PACKAGE_NAME/packaging - -COPY packaging/rosdep.yaml* packaging/rosdep.sh packaging/build_deps.sh /$PACKAGE_NAME/packaging/ -COPY underlay.repos package.xml /$PACKAGE_NAME/packaging/ -COPY entrypoint.sh* /$PACKAGE_NAME/ - -RUN /$PACKAGE_NAME/packaging/rosdep.sh /$PACKAGE_NAME - -RUN chown -R builder:builder /$PACKAGE_NAME - -USER builder - -RUN rosdep update - -RUN /$PACKAGE_NAME/packaging/build_deps.sh /$PACKAGE_NAME - -VOLUME /$PACKAGE_NAME/sources -WORKDIR /$PACKAGE_NAME/sources diff --git a/build.sh b/build.sh deleted file mode 100755 index fdf3b14..0000000 --- a/build.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -euxo pipefail - -output_dir=$1 - -git_commit_hash=${2:-$(git rev-parse HEAD)} - -git_version_string=${3:-$(git log --date=format:%Y%m%d --pretty=~git%cd.%h -n 1)} - -build_number=${GITHUB_RUN_NUMBER:=0} - -ros_distro=${ROS_DISTRO:=galactic} - -iname=${PACKAGE_NAME:=mocap_pose} - -iversion=${PACKAGE_VERSION:=latest} - -docker build \ - --build-arg UID=$(id -u) \ - --build-arg GID=$(id -g) \ - --build-arg ROS_DISTRO=${ros_distro} \ - --build-arg PACKAGE_NAME=${iname} \ - --pull \ - -f Dockerfile.build_env -t "${iname}_build:${iversion}" . - -docker run \ - --rm \ - -v $(pwd):/${iname}/sources \ - ${iname}_build:${iversion} \ - ./packaging/package.sh \ - -b ${build_number} \ - -g ${git_commit_hash} \ - -v ${git_version_string} - -mkdir -p ${output_dir} -cp *.deb *.ddeb ${output_dir} -rm -Rf *.deb *.ddeb - -exit 0 diff --git a/entrypoint.sh b/entrypoint.sh deleted file mode 100644 index 7ff12fc..0000000 --- a/entrypoint.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash - -source /opt/ros/galactic/setup.bash - -ros2 launch mocap_pose mocap_pose.launch - diff --git a/packaging/build_deps.sh b/packaging/build_deps.sh deleted file mode 100755 index 62cd462..0000000 --- a/packaging/build_deps.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -mod_dir=${1} - -# Extract not satisfied dependencies from output, check if they are exist in ../underlay.repos -if rosdep check --from-paths ${mod_dir} 1> /dev/null 2>&1; then - echo "[INFO] Dependencies are satisfied." -else - echo "[INFO] Building dependencies using underlay.repos." - cd ${mod_dir} - - echo "[INFO] Get package dependencies." - # Dependencies from fog-sw repo - if [ -e ${mod_dir}/ros2_ws/src ]; then - echo "[INFO] Use dependencies from fog_sw." - pushd ${mod_dir}/ros2_ws > /dev/null - source /opt/ros/${ROS_DISTRO}/setup.bash - else - echo "[INFO] Use dependencies from local repository." - mkdir -p ${mod_dir}/deps_ws/src - pushd ${mod_dir}/deps_ws > /dev/null - vcs import src < ${mod_dir}/packaging/underlay.repos - rosdep install --from-paths src --ignore-src -r -y --rosdistro ${ROS_DISTRO} - source /opt/ros/${ROS_DISTRO}/setup.bash - fi - - rosdep_out=$(rosdep check -v --from-paths src 2>&1 | grep "resolving for resources" ) - ALL_PKGS=$(echo $rosdep_out | sed 's/.*\[\(.*\)\].*/\1/' | tr ',' '\n' | tr -d ' ') - echo "[INFO] All packages: $(echo $ALL_PKGS|tr '\n' ' ')" - PKGS_TO_BUILD="" - pushd src > /dev/null - - for pkg_name in ${ALL_PKGS}; do - echo "[INFO] Check if package ${pkg_name} is in the list of packages to build." - pkg_name=$(echo ${pkg_name} | sed 's/\/$//') - if ! ros2 pkg list | grep ${pkg_name} 1> /dev/null 2>&1; then - PKGS_TO_BUILD="${PKGS_TO_BUILD} ${pkg_name}" - fi - done - - echo "[INFO] Packages to build: $PKGS_TO_BUILD" - popd > /dev/null - - echo "[INFO] Build package dependencies." - colcon build --packages-select ${PKGS_TO_BUILD} - popd > /dev/null -fi - - diff --git a/packaging/package.sh b/packaging/package.sh deleted file mode 100755 index 6b653e1..0000000 --- a/packaging/package.sh +++ /dev/null @@ -1,134 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -usage() { - echo " -Usage: $(basename "$0") [-h] [-b nbr] [-d dist] - -- Generate debian package from fog_sw module. -Params: - -h Show help text. - -b Build number. This will be tha last digit of version string (x.x.N). - -d Distribution string in debian changelog. - -g Git commit hash. - -v Git version string -" - exit 0 -} - -check_arg() { - if [ "$(echo $1 | cut -c1)" = "-" ]; then - return 1 - else - return 0 - fi -} - -error_arg() { - echo "$0: option requires an argument -- $1" - usage -} - -mod_dir="$(realpath $(dirname $0)/..)" -build_nbr=0 -distr="" -version="" -git_commit_hash="" -git_version_string="" - -while getopts "hb:d:g:v:" opt -do - case $opt in - h) - usage - ;; - b) - check_arg $OPTARG && build_nbr=$OPTARG || error_arg $opt - ;; - d) - check_arg $OPTARG && distr=$OPTARG || error_arg $opt - ;; - g) - check_arg $OPTARG && git_commit_hash=$OPTARG || error_arg $opt - ;; - v) - check_arg $OPTARG && git_version_string=$OPTARG || error_arg $opt - ;; - \?) - usage - ;; - esac -done - -if [[ "$git_commit_hash" == "0" || -z "$git_commit_hash" ]]; then - git_commit_hash="$(git rev-parse HEAD)" -fi -if [[ "$git_version_string" == "0" || -z "$git_version_string" ]]; then - git_version_string="$(git log --date=format:%Y%m%d --pretty=~git%cd.%h -n 1)" -fi - -## Remove trailing '/' mark in module dir, if exists -mod_dir=$(echo $mod_dir | sed 's/\/$//') - -## Debug prints -echo -echo "[INFO] mod_dir: ${mod_dir}." -echo "[INFO] build_nbr: ${build_nbr}." -echo "[INFO] distr: ${distr}." -echo "[INFO] git_commit_hash: ${git_commit_hash}." -echo "[INFO] git_version_string: ${git_version_string}." - -cd $mod_dir - -## Generate package -echo "[INFO] Creating deb package..." -### ROS2 Packaging - -### Create version string -version=$(grep "" package.xml | sed 's/[^>]*>\([^<"]*\).*/\1/') - -echo "[INFO] Version: ${version}." - -#title="$version ($(date +%Y-%m-%d))" -#cat << EOF_CHANGELOG > CHANGELOG.rst -#$title -#$(printf '%*s' "${#title}" | tr ' ' "-") -#* commit: ${git_commit_hash} -#EOF_CHANGELOG - -if [ -e ${mod_dir}/ros2_ws ]; then - # From fog-sw repo. - source ${mod_dir}/ros2_ws/install/setup.bash -fi -if [ -e ${mod_dir}/../deps_ws ]; then - source ${mod_dir}/../deps_ws/install/setup.bash -fi - -if [ -e ${mod_dir}/debian ]; then - cp -r debian debian_bak -fi - -bloom-generate rosdebian --os-name ubuntu --os-version focal --ros-distro ${ROS_DISTRO} --place-template-files \ - && sed -i "s/@(DebianInc)@(Distribution)/@(DebianInc)/" debian/changelog.em \ - && [ ! "$distr" = "" ] && sed -i "s/@(Distribution)/${distr}/" debian/changelog.em || : \ - && bloom-generate rosdebian --os-name ubuntu --os-version focal --ros-distro ${ROS_DISTRO} --process-template-files -i ${build_nbr}${git_version_string} \ - && sed -i 's/^\tdh_shlibdeps.*/& --dpkg-shlibdeps-params=--ignore-missing-info/g' debian/rules \ - && sed -i "s/\=\([0-9]*\.[0-9]*\.[0-9]*\*\)//g" debian/control \ - && fakeroot debian/rules clean \ - && fakeroot debian/rules binary || exit 1 - -echo "[INFO] Clean up." - -rm -rf obj-x86_64-linux-gnu debian - -if [ -e ${mod_dir}/debian_bak ]; then - cp -r debian_bak debian - rm -rf debian_bak -fi - - -echo "[INFO] Move debian packages to volume." -mv ${mod_dir}/../*.deb ${mod_dir}/../*.ddeb ${mod_dir} - -echo "[INFO] Done." -exit 0 diff --git a/packaging/rosdep.sh b/packaging/rosdep.sh deleted file mode 100755 index 44fbf0b..0000000 --- a/packaging/rosdep.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set -eu - -## for installing additional non-ROS2 dependencies to debian package generated by bloom -if [ ! -e /etc/ros/rosdep/sources.list.d/20-default.list ]; then - echo "[INFO] Initialize rosdep" - sudo rosdep init -fi - -mod_dir=$(echo ${1} | sed 's/\/$//') -yamlpath=${mod_dir}/packaging/rosdep.yaml -# Find ROS_DISTRO text in the yaml file and replace it with env variable -if [ -e ${yamlpath} ]; then - echo "[INFO] Replace ROS_DISTRO in rosdep.yaml" - sudo sed -i "s/ROS_DISTRO/${ROS_DISTRO}/g" ${yamlpath} - echo "[INFO] Add module specific dependencies" - cat $yamlpath - mkdir -p /etc/ros/rosdep/sources.list.d - echo "yaml file://${yamlpath}" > /etc/ros/rosdep/sources.list.d/51-fogsw-module.list -fi - -echo "[INFO] Updating rosdep" -rosdep update - -apt update -echo "[INFO] Running rosdep install.." -if rosdep install --from-paths ${mod_dir}/packaging -r -y --rosdistro ${ROS_DISTRO} 1> /dev/null 2>&1; then - echo "[INFO] rosdep install finished successfully." -else - echo "[ERROR] Some dependencies missing. It will be built using underlay.repos." -fi - -exit 0 diff --git a/packaging/rosdep.yaml b/packaging/rosdep.yaml index ebddb89..be6bcf2 100644 --- a/packaging/rosdep.yaml +++ b/packaging/rosdep.yaml @@ -1,2 +1,2 @@ px4_msgs: - ubuntu: ros-ROS_DISTRO-px4-msgs=3.0.0* + ubuntu: ros-galactic-px4-msgs=3.0.0* diff --git a/tasks.py b/tasks.py deleted file mode 100644 index 9913473..0000000 --- a/tasks.py +++ /dev/null @@ -1,131 +0,0 @@ -import glob -import os -from invoke import task, Collection, call - -THISDIR = os.path.dirname(os.path.realpath(__file__)) -MODULE_NAME = os.path.basename(THISDIR) - -def get_submodules(c): - """ - return repository submodule names - """ - submodules = [] - with c.cd(THISDIR): - result = c.run("git submodule status", hide=True) - for line in result.stdout.splitlines(): - submodules.append(line.split()[1]) - return submodules - -def get_iname_tag(image_name): - """ - return tuple with image name and tag - """ - if ":" in image_name: - iname, tag = image_name.split(":") - else: - iname, tag = image_name, "latest" - return iname, tag - - -@task -def init(c): - """ - Init submodules. - """ - print("init submodules") - with c.cd(THISDIR): - c.run("git submodule init", hide=True) - -@task(init) -def clone(c): - """ - Clone this repository submodules. - """ - submodules = get_submodules(c) - with c.cd(THISDIR): - for sub in submodules: - c.run("git submodule update --init --recursive %s" %sub) - -@task( - help={'nocache': "do not use cache when building the image", - 'pull': "always attempt to pull a newer version of the image", - 'ros_distro': "ROS distro to use (Available [foxy, galactic])"} -) -def build_env(c, nocache=False, pull=False, ros_distro="galactic", image_name=MODULE_NAME): - """ - Create Docker build environment. - """ - iname, tag = get_iname_tag(image_name) - - args = [] - args.append("--build-arg UID=$(id -u)") - args.append("--build-arg GID=$(id -g)") - args.append("--build-arg ROS_DISTRO=%s" % ros_distro) - args.append("--build-arg PACKAGE_NAME=%s" % iname) - args.append("-f Dockerfile.build_env") - args.append("-t %s_build:%s" % (iname, tag)) - if nocache: - args.append("--no-cache") - elif pull: - args.append("--pull") - with c.cd(THISDIR): - c.run("docker build %s ." % " ".join(args)) - -@task( - help={'reallyclean': "remove & reload all submodules"} -) -def clean(c, reallyclean=False): - """ - Clean workspace. - """ - with c.cd(THISDIR): - if reallyclean: - c.run("git submodule deinit -f --all") - clone(c) - else: - c.run("git submodule foreach git clean -xdf") - c.run("git submodule foreach git checkout .") - c.run("git clean -xdf") - -@task( - help={'out_dir': "output directory for the generated deb files", - 'ros_distro': "ROS distro to use (Available [foxy, galactic])"} -) -def create_deb_package(c, out_dir="../bin/", ros_distro="galactic", image_name=MODULE_NAME): - """ - Build debian package - """ - iname, tag = get_iname_tag(image_name) - c.run("ROS_DISTRO={0} PACKAGE_NAME={1} PACKAGE_VERSION={2} ./build.sh {3}" - .format(ros_distro, iname, tag, out_dir)) - -@task(help={'nocache': "do not use cache when building the image", - 'pull': "always attempt to pull a newer version of the image", - 'ros_distro': "ROS distro to use (Available [foxy, galactic])", - 'image_name': "name of output docker image"} -) -def build_docker(c, nocache=False, pull=False, ros_distro="galactic", image_name=MODULE_NAME): - """ - Build Docker image of this component - """ - col = Collection() - col.add_task(build_env) - col['build_env'](c, nocache=nocache, pull=pull, ros_distro=ros_distro, image_name=image_name) - - iname, tag = get_iname_tag(image_name) - args = [] - args.append("--build-arg UID=$(id -u)") - args.append("--build-arg GID=$(id -g)") - args.append("--build-arg ROS_DISTRO=%s" % ros_distro) - args.append("--build-arg PACKAGE_NAME=%s" % iname) - args.append("--build-arg FROM_IMAGE=%s_build:%s" % (iname, tag)) - args.append("-f Dockerfile") - args.append("-t %s:%s" % (iname, tag)) - if nocache: - args.append("--no-cache") - elif pull: - args.append("--pull") - with c.cd(THISDIR): - print("docker build %s ." % " ".join(args)) - c.run("docker build %s ." % " ".join(args)) - diff --git a/underlay.repos b/underlay.repos deleted file mode 100644 index 140e1fe..0000000 --- a/underlay.repos +++ /dev/null @@ -1,4 +0,0 @@ -px4_msgs: - type: git - url: https://github.com/tiiuae/PX4-msgs.git - version: tags/3.0.0