diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 568e457..8007d6e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,33 +9,33 @@ on: jobs: tii-deb-build: runs-on: ubuntu-latest + strategy: + matrix: + ros2_distro: [galactic] steps: - - name: Checkout mocap_pose + - name: Checkout moca_pose uses: actions/checkout@v2 with: path: mocap_pose - submodules: recursive - - - name: Checkout ci-scripts - uses: actions/checkout@v2 - with: - repository: tiiuae/fogsw-ci-scripts - path: fogsw-ci-scripts # Run docker build - - name: Run fog-sw docker build + - name: Run mocap_pose docker build + env: + ROS: 1 + ROS_DISTRO: ${{ matrix.ros2_distro }} + PACKAGE_NAME: mocap_pose run: | set -eux mkdir bin - pushd fogsw-ci-scripts - ./generate_deb.sh ../mocap_pose ../bin/ + pushd mocap_pose + ./build.sh ../bin/ popd - name: Install jfrog CLI tool env: - JFROG_CLI_URL: https://artifactory.ssrc.fi/artifactory/gen-public-local/tools/jfrog/jfrog-1.45.2.tar.gz - if: github.event_name == 'push' + JFROG_CLI_URL: https://ssrc.jfrog.io/artifactory/ssrc-gen-public-remote-cache/tools/jfrog/jfrog-1.45.2.tar.gz + if: github.event_name == 'push' || github.event.action == 'rebuild' run: | set -exu mkdir -p "$GITHUB_WORKSPACE/.jfrog/bin" @@ -53,12 +53,12 @@ jobs: ARCHITECTURE: amd64 BUILD_NAME: mocap-pose CI: true - if: github.event_name == 'push' + if: github.event_name == 'push' || github.event.action == 'rebuild' run: | set -exu jfrog rt c import "$ARTIFACTORY_TOKEN" jfrog rt ping - pkg=$(find bin -name 'ros-foxy-mocap-pose*.deb') + pkg=$(find bin -name 'ros-${{ matrix.ros2_distro }}-mocap-pose*.deb') jfrog rt u --deb "$DISTRIBUTION/$COMPONENT/$ARCHITECTURE" \ --target-props COMMIT="$GITHUB_SHA" \ --build-name "$BUILD_NAME" \ @@ -68,4 +68,4 @@ jobs: jfrog rt build-publish "$BUILD_NAME" "$GITHUB_SHA" jfrog rt bpr "$BUILD_NAME" "$GITHUB_SHA" "$ARTIFACTORY_REPO" \ --status dev \ - --comment "development build" + --comment "development build" \ No newline at end of file diff --git a/.github/workflows/tii-mocap-pose.yaml b/.github/workflows/tii-mocap-pose.yaml new file mode 100644 index 0000000..aee3422 --- /dev/null +++ b/.github/workflows/tii-mocap-pose.yaml @@ -0,0 +1,72 @@ +name: tii-mocap-pose + +on: + push: + branches: main + tags: + - 'v*' + - v[0-9]+.[0-9]+.[0-9]+ + - v[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+ + pull_request: + branches: [ main ] + +jobs: + tii-mocap-pose: + runs-on: ubuntu-latest + services: + registry: + image: registry:2 + ports: + - 5000:5000 + steps: + + - uses: actions/checkout@v2 + + - uses: docker/setup-buildx-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: network=host + + - name: Build the builder container image + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile.build_env + push: true + tags: localhost:5000/tiiuae/mocap_pose:build_env + build-args: | + PACKAGE_NAME=mocap_pose + ROS_DISTRO=galactic + + - name: Docker meta + id: meta + uses: docker/metadata-action@v3 + with: + images: ghcr.io/tiiuae/tii-mocap-pose + tags: | + type=ref,event=branch + type=semver,pattern={{version}} + type=sha + type=raw,value=latest + + - name: Login to GitHub Container Registry + uses: docker/login-action@v1 + if: github.event_name == 'push' + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build tii-mocap-pose image and push + uses: docker/build-push-action@v2 + with: + context: . + file: ./Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + build-args: | + PACKAGE_NAME=mocap_pose + ROS_DISTRO=galactic + FROM_IMAGE=localhost:5000/tiiuae/mocap_pose:build_env diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7258092 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +# fog-sw BUILDER +ARG FROM_IMAGE +FROM $FROM_IMAGE as fog-sw-builder +ARG ROS_DISTRO="galactic" +ARG UID=1000 +ARG GID=1000 +ARG PACKAGE_NAME + +WORKDIR /$PACKAGE_NAME/main_ws +USER root +ADD . /$PACKAGE_NAME/main_ws/src +RUN chown -R builder:builder /$PACKAGE_NAME/main_ws + +USER builder + +RUN if [ -e /$PACKAGE_NAME/deps_ws ]; then \ + . /$PACKAGE_NAME/deps_ws/install/setup.sh && \ + colcon build; \ + elif [ -e /opt/ros/${ROS_DISTRO}/setup.sh ]; then \ + . /opt/ros/${ROS_DISTRO}/setup.sh && \ + colcon build; \ + fi + +RUN sed --in-place \ + 's|^source .*|source "/'$PACKAGE_NAME'/main_ws/install/setup.bash"|' \ + /$PACKAGE_NAME/entrypoint.sh && \ + chmod +x /$PACKAGE_NAME/entrypoint.sh + +ENV PACKAGE_NAME $PACKAGE_NAME +WORKDIR /$PACKAGE_NAME +ENTRYPOINT "/"$PACKAGE_NAME"/entrypoint.sh" diff --git a/Dockerfile.build_env b/Dockerfile.build_env new file mode 100644 index 0000000..5a084b2 --- /dev/null +++ b/Dockerfile.build_env @@ -0,0 +1,44 @@ +# fog-sw BUILDER +ARG ROS_DISTRO="galactic" +FROM ros:${ROS_DISTRO}-ros-base as fog-sw-builder + +ARG UID=1000 +ARG GID=1000 +ARG BUILD_NUMBER +ARG COMMIT_ID +ARG GIT_VER +ARG PACKAGE_NAME +# Install build dependencies +RUN apt-get update -y && apt-get install -y --no-install-recommends \ + curl \ + python3-bloom \ + fakeroot \ + dh-make \ + libboost-dev \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd -g $GID builder && \ + useradd -m -u $UID -g $GID -g builder builder && \ + usermod -aG sudo builder && \ + echo 'builder ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers + +RUN echo "deb [trusted=yes] https://ssrc.jfrog.io/artifactory/ssrc-debian-public-remote focal fog-sw" >> /etc/apt/sources.list + +RUN mkdir -p /$PACKAGE_NAME/packaging + +COPY packaging/rosdep.yaml* packaging/rosdep.sh packaging/build_deps.sh /$PACKAGE_NAME/packaging/ +COPY underlay.repos package.xml /$PACKAGE_NAME/packaging/ +COPY entrypoint.sh* /$PACKAGE_NAME/ + +RUN /$PACKAGE_NAME/packaging/rosdep.sh /$PACKAGE_NAME + +RUN chown -R builder:builder /$PACKAGE_NAME + +USER builder + +RUN rosdep update + +RUN /$PACKAGE_NAME/packaging/build_deps.sh /$PACKAGE_NAME + +VOLUME /$PACKAGE_NAME/sources +WORKDIR /$PACKAGE_NAME/sources diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..fdf3b14 --- /dev/null +++ b/build.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +set -euxo pipefail + +output_dir=$1 + +git_commit_hash=${2:-$(git rev-parse HEAD)} + +git_version_string=${3:-$(git log --date=format:%Y%m%d --pretty=~git%cd.%h -n 1)} + +build_number=${GITHUB_RUN_NUMBER:=0} + +ros_distro=${ROS_DISTRO:=galactic} + +iname=${PACKAGE_NAME:=mocap_pose} + +iversion=${PACKAGE_VERSION:=latest} + +docker build \ + --build-arg UID=$(id -u) \ + --build-arg GID=$(id -g) \ + --build-arg ROS_DISTRO=${ros_distro} \ + --build-arg PACKAGE_NAME=${iname} \ + --pull \ + -f Dockerfile.build_env -t "${iname}_build:${iversion}" . + +docker run \ + --rm \ + -v $(pwd):/${iname}/sources \ + ${iname}_build:${iversion} \ + ./packaging/package.sh \ + -b ${build_number} \ + -g ${git_commit_hash} \ + -v ${git_version_string} + +mkdir -p ${output_dir} +cp *.deb *.ddeb ${output_dir} +rm -Rf *.deb *.ddeb + +exit 0 diff --git a/debian/postinst b/debian/postinst.em similarity index 98% rename from debian/postinst rename to debian/postinst.em index a80233f..285d9f6 100755 --- a/debian/postinst +++ b/debian/postinst.em @@ -10,3 +10,5 @@ SUBSYSTEM=="usb", ATTRS{idVendor}=="2357", ATTRS{idProduct}=="011e", MODE="0666" EOF chmod 644 /etc/udev/rules.d/81-tplink.rules + +exit 0 diff --git a/debian/prerm b/debian/prerm.em similarity index 93% rename from debian/prerm rename to debian/prerm.em index 86e4817..aa761d1 100755 --- a/debian/prerm +++ b/debian/prerm.em @@ -3,3 +3,5 @@ if [ -e /etc/udev/rules.d/81-tplink.rules ]; then rm /etc/udev/rules.d/81-tplink.rules fi + +exit 0 diff --git a/debian/shlibs.local b/debian/shlibs.local deleted file mode 100644 index 8b13789..0000000 --- a/debian/shlibs.local +++ /dev/null @@ -1 +0,0 @@ - diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100644 index 0000000..f6f1a54 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +source /opt/ros/galactic/setup.bash + diff --git a/packaging/build_deps.sh b/packaging/build_deps.sh new file mode 100755 index 0000000..62cd462 --- /dev/null +++ b/packaging/build_deps.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +set -eo pipefail + +mod_dir=${1} + +# Extract not satisfied dependencies from output, check if they are exist in ../underlay.repos +if rosdep check --from-paths ${mod_dir} 1> /dev/null 2>&1; then + echo "[INFO] Dependencies are satisfied." +else + echo "[INFO] Building dependencies using underlay.repos." + cd ${mod_dir} + + echo "[INFO] Get package dependencies." + # Dependencies from fog-sw repo + if [ -e ${mod_dir}/ros2_ws/src ]; then + echo "[INFO] Use dependencies from fog_sw." + pushd ${mod_dir}/ros2_ws > /dev/null + source /opt/ros/${ROS_DISTRO}/setup.bash + else + echo "[INFO] Use dependencies from local repository." + mkdir -p ${mod_dir}/deps_ws/src + pushd ${mod_dir}/deps_ws > /dev/null + vcs import src < ${mod_dir}/packaging/underlay.repos + rosdep install --from-paths src --ignore-src -r -y --rosdistro ${ROS_DISTRO} + source /opt/ros/${ROS_DISTRO}/setup.bash + fi + + rosdep_out=$(rosdep check -v --from-paths src 2>&1 | grep "resolving for resources" ) + ALL_PKGS=$(echo $rosdep_out | sed 's/.*\[\(.*\)\].*/\1/' | tr ',' '\n' | tr -d ' ') + echo "[INFO] All packages: $(echo $ALL_PKGS|tr '\n' ' ')" + PKGS_TO_BUILD="" + pushd src > /dev/null + + for pkg_name in ${ALL_PKGS}; do + echo "[INFO] Check if package ${pkg_name} is in the list of packages to build." + pkg_name=$(echo ${pkg_name} | sed 's/\/$//') + if ! ros2 pkg list | grep ${pkg_name} 1> /dev/null 2>&1; then + PKGS_TO_BUILD="${PKGS_TO_BUILD} ${pkg_name}" + fi + done + + echo "[INFO] Packages to build: $PKGS_TO_BUILD" + popd > /dev/null + + echo "[INFO] Build package dependencies." + colcon build --packages-select ${PKGS_TO_BUILD} + popd > /dev/null +fi + + diff --git a/packaging/package.sh b/packaging/package.sh new file mode 100755 index 0000000..6b653e1 --- /dev/null +++ b/packaging/package.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +set -eo pipefail + +usage() { + echo " +Usage: $(basename "$0") [-h] [-b nbr] [-d dist] + -- Generate debian package from fog_sw module. +Params: + -h Show help text. + -b Build number. This will be tha last digit of version string (x.x.N). + -d Distribution string in debian changelog. + -g Git commit hash. + -v Git version string +" + exit 0 +} + +check_arg() { + if [ "$(echo $1 | cut -c1)" = "-" ]; then + return 1 + else + return 0 + fi +} + +error_arg() { + echo "$0: option requires an argument -- $1" + usage +} + +mod_dir="$(realpath $(dirname $0)/..)" +build_nbr=0 +distr="" +version="" +git_commit_hash="" +git_version_string="" + +while getopts "hb:d:g:v:" opt +do + case $opt in + h) + usage + ;; + b) + check_arg $OPTARG && build_nbr=$OPTARG || error_arg $opt + ;; + d) + check_arg $OPTARG && distr=$OPTARG || error_arg $opt + ;; + g) + check_arg $OPTARG && git_commit_hash=$OPTARG || error_arg $opt + ;; + v) + check_arg $OPTARG && git_version_string=$OPTARG || error_arg $opt + ;; + \?) + usage + ;; + esac +done + +if [[ "$git_commit_hash" == "0" || -z "$git_commit_hash" ]]; then + git_commit_hash="$(git rev-parse HEAD)" +fi +if [[ "$git_version_string" == "0" || -z "$git_version_string" ]]; then + git_version_string="$(git log --date=format:%Y%m%d --pretty=~git%cd.%h -n 1)" +fi + +## Remove trailing '/' mark in module dir, if exists +mod_dir=$(echo $mod_dir | sed 's/\/$//') + +## Debug prints +echo +echo "[INFO] mod_dir: ${mod_dir}." +echo "[INFO] build_nbr: ${build_nbr}." +echo "[INFO] distr: ${distr}." +echo "[INFO] git_commit_hash: ${git_commit_hash}." +echo "[INFO] git_version_string: ${git_version_string}." + +cd $mod_dir + +## Generate package +echo "[INFO] Creating deb package..." +### ROS2 Packaging + +### Create version string +version=$(grep "" package.xml | sed 's/[^>]*>\([^<"]*\).*/\1/') + +echo "[INFO] Version: ${version}." + +#title="$version ($(date +%Y-%m-%d))" +#cat << EOF_CHANGELOG > CHANGELOG.rst +#$title +#$(printf '%*s' "${#title}" | tr ' ' "-") +#* commit: ${git_commit_hash} +#EOF_CHANGELOG + +if [ -e ${mod_dir}/ros2_ws ]; then + # From fog-sw repo. + source ${mod_dir}/ros2_ws/install/setup.bash +fi +if [ -e ${mod_dir}/../deps_ws ]; then + source ${mod_dir}/../deps_ws/install/setup.bash +fi + +if [ -e ${mod_dir}/debian ]; then + cp -r debian debian_bak +fi + +bloom-generate rosdebian --os-name ubuntu --os-version focal --ros-distro ${ROS_DISTRO} --place-template-files \ + && sed -i "s/@(DebianInc)@(Distribution)/@(DebianInc)/" debian/changelog.em \ + && [ ! "$distr" = "" ] && sed -i "s/@(Distribution)/${distr}/" debian/changelog.em || : \ + && bloom-generate rosdebian --os-name ubuntu --os-version focal --ros-distro ${ROS_DISTRO} --process-template-files -i ${build_nbr}${git_version_string} \ + && sed -i 's/^\tdh_shlibdeps.*/& --dpkg-shlibdeps-params=--ignore-missing-info/g' debian/rules \ + && sed -i "s/\=\([0-9]*\.[0-9]*\.[0-9]*\*\)//g" debian/control \ + && fakeroot debian/rules clean \ + && fakeroot debian/rules binary || exit 1 + +echo "[INFO] Clean up." + +rm -rf obj-x86_64-linux-gnu debian + +if [ -e ${mod_dir}/debian_bak ]; then + cp -r debian_bak debian + rm -rf debian_bak +fi + + +echo "[INFO] Move debian packages to volume." +mv ${mod_dir}/../*.deb ${mod_dir}/../*.ddeb ${mod_dir} + +echo "[INFO] Done." +exit 0 diff --git a/packaging/rosdep.sh b/packaging/rosdep.sh new file mode 100755 index 0000000..44fbf0b --- /dev/null +++ b/packaging/rosdep.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +set -eu + +## for installing additional non-ROS2 dependencies to debian package generated by bloom +if [ ! -e /etc/ros/rosdep/sources.list.d/20-default.list ]; then + echo "[INFO] Initialize rosdep" + sudo rosdep init +fi + +mod_dir=$(echo ${1} | sed 's/\/$//') +yamlpath=${mod_dir}/packaging/rosdep.yaml +# Find ROS_DISTRO text in the yaml file and replace it with env variable +if [ -e ${yamlpath} ]; then + echo "[INFO] Replace ROS_DISTRO in rosdep.yaml" + sudo sed -i "s/ROS_DISTRO/${ROS_DISTRO}/g" ${yamlpath} + echo "[INFO] Add module specific dependencies" + cat $yamlpath + mkdir -p /etc/ros/rosdep/sources.list.d + echo "yaml file://${yamlpath}" > /etc/ros/rosdep/sources.list.d/51-fogsw-module.list +fi + +echo "[INFO] Updating rosdep" +rosdep update + +apt update +echo "[INFO] Running rosdep install.." +if rosdep install --from-paths ${mod_dir}/packaging -r -y --rosdistro ${ROS_DISTRO} 1> /dev/null 2>&1; then + echo "[INFO] rosdep install finished successfully." +else + echo "[ERROR] Some dependencies missing. It will be built using underlay.repos." +fi + +exit 0 diff --git a/packaging/rosdep.yaml b/packaging/rosdep.yaml new file mode 100644 index 0000000..ebddb89 --- /dev/null +++ b/packaging/rosdep.yaml @@ -0,0 +1,2 @@ +px4_msgs: + ubuntu: ros-ROS_DISTRO-px4-msgs=3.0.0* diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..63f447b --- /dev/null +++ b/tasks.py @@ -0,0 +1,131 @@ +import glob +import os +from invoke import task, Collection, call + +THISDIR = os.path.dirname(os.path.realpath(__file__)) +MODULE_NAME = os.path.basename(THISDIR) + +def get_submodules(c): + """ + return repository submodule names + """ + submodules = [] + with c.cd(THISDIR): + result = c.run("git submodule status", hide=True) + for line in result.stdout.splitlines(): + submodules.append(line.split()[1]) + return submodules + +def get_iname_tag(image_name): + """ + return tuple with image name and tag + """ + if ":" in image_name: + iname, tag = image_name.split(":") + else: + iname, tag = image_name, "latest" + return iname, tag + + +@task +def init(c): + """ + Init submodules. + """ + print("init submodules") + with c.cd(THISDIR): + c.run("git submodule init", hide=True) + +@task(init) +def clone(c): + """ + Clone this repository submodules. + """ + submodules = get_submodules(c) + with c.cd(THISDIR): + for sub in submodules: + c.run("git submodule update --init --recursive %s" %sub) + +@task( + help={'nocache': "do not use cache when building the image", + 'pull': "always attempt to pull a newer version of the image", + 'ros_distro': "ROS distro to use (Available [foxy, galactic])"} +) +def build_env(c, nocache=False, pull=False, ros_distro="foxy", image_name=MODULE_NAME): + """ + Create Docker build environment. + """ + iname, tag = get_iname_tag(image_name) + + args = [] + args.append("--build-arg UID=$(id -u)") + args.append("--build-arg GID=$(id -g)") + args.append("--build-arg ROS_DISTRO=%s" % ros_distro) + args.append("--build-arg PACKAGE_NAME=%s" % iname) + args.append("-f Dockerfile.build_env") + args.append("-t %s_build:%s" % (iname, tag)) + if nocache: + args.append("--no-cache") + elif pull: + args.append("--pull") + with c.cd(THISDIR): + c.run("docker build %s ." % " ".join(args)) + +@task( + help={'reallyclean': "remove & reload all submodules"} +) +def clean(c, reallyclean=False): + """ + Clean workspace. + """ + with c.cd(THISDIR): + if reallyclean: + c.run("git submodule deinit -f --all") + clone(c) + else: + c.run("git submodule foreach git clean -xdf") + c.run("git submodule foreach git checkout .") + c.run("git clean -xdf") + +@task( + help={'out_dir': "output directory for the generated deb files", + 'ros_distro': "ROS distro to use (Available [foxy, galactic])"} +) +def create_deb_package(c, out_dir="../bin/", ros_distro="foxy", image_name=MODULE_NAME): + """ + Build debian package + """ + iname, tag = get_iname_tag(image_name) + c.run("ROS_DISTRO={0} PACKAGE_NAME={1} PACKAGE_VERSION={2} ./build.sh {3}" + .format(ros_distro, iname, tag, out_dir)) + +@task(help={'nocache': "do not use cache when building the image", + 'pull': "always attempt to pull a newer version of the image", + 'ros_distro': "ROS distro to use (Available [foxy, galactic])", + 'image_name': "name of output docker image"} +) +def build_docker(c, nocache=False, pull=False, ros_distro="foxy", image_name=MODULE_NAME): + """ + Build Docker image of this component + """ + col = Collection() + col.add_task(build_env) + col['build_env'](c, nocache=nocache, pull=pull, ros_distro=ros_distro, image_name=image_name) + + iname, tag = get_iname_tag(image_name) + args = [] + args.append("--build-arg UID=$(id -u)") + args.append("--build-arg GID=$(id -g)") + args.append("--build-arg ROS_DISTRO=%s" % ros_distro) + args.append("--build-arg PACKAGE_NAME=%s" % iname) + args.append("--build-arg FROM_IMAGE=%s_build:%s" % (iname, tag)) + args.append("-f Dockerfile") + args.append("-t %s:%s" % (iname, tag)) + if nocache: + args.append("--no-cache") + elif pull: + args.append("--pull") + with c.cd(THISDIR): + print("docker build %s ." % " ".join(args)) + c.run("docker build %s ." % " ".join(args)) + diff --git a/underlay.repos b/underlay.repos new file mode 100644 index 0000000..140e1fe --- /dev/null +++ b/underlay.repos @@ -0,0 +1,4 @@ +px4_msgs: + type: git + url: https://github.com/tiiuae/PX4-msgs.git + version: tags/3.0.0