From c38a7ef75b73b79fe2d55be22a11c7cadfd65266 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Denis=20=C5=A0togl?= <denis@stoglrobotics.de>
Date: Tue, 27 Sep 2022 09:21:34 +0200
Subject: [PATCH] Update CI setup in general and add more detailed explanations
 into docs. (#49)

Co-authored-by: Manuel M <mamueluth@gmail.com>
---
 .../ros_packages/configure_repository.rst     | 39 ++++++++++-
 scripts/_Team_Defines.bash                    |  2 +-
 scripts/setup-repository-ci.bash              | 68 +++++++++++++------
 templates/package/.pre-commit-config.yaml     | 12 ++--
 templates/package/.pre-commit_wrapper.shb     |  6 ++
 .../package/CI-github_abi-compatibility.yml   |  7 +-
 ...ld.yml => CI-github_binary-build-main.yml} | 10 ++-
 .../CI-github_binary-build-testing.yml        | 26 +++++++
 .../package/CI-github_ci-coverage-build.yml   | 18 ++---
 .../package/CI-github_ci-format-private.yml   |  2 +-
 templates/package/CI-github_ci-format.yml     |  2 +-
 templates/package/CI-github_ci-ros-lint.yml   |  4 +-
 .../CI-github_docs-sphinx-build-check.yml     |  2 +-
 .../CI-github_docs-sphinx-make-page.yml       |  2 +-
 ...thub_reusable-industrial-ci-with-cache.yml | 18 ++---
 ...thub_reusable-ros-tooling-source-build.yml | 48 +++++++++++++
 .../package/CI-github_rhel-binary-build.yml   |  9 ++-
 ...l => CI-github_semi-binary-build-main.yml} | 11 +--
 .../CI-github_semi-binary-build-testing.yml   | 25 +++++++
 templates/package/CI-github_source-build.yml  |  6 +-
 .../package/_append_to_README_ROS_Intro.md    |  9 +--
 .../package/_append_to_README_build_status.md |  7 +-
 .../_append_to_README_build_status_table.md   |  2 +-
 ..._to_workflows_README_build_status_table.md |  4 ++
 24 files changed, 260 insertions(+), 79 deletions(-)
 create mode 100644 templates/package/.pre-commit_wrapper.shb
 rename templates/package/{CI-github_binary-build.yml => CI-github_binary-build-main.yml} (68%)
 create mode 100644 templates/package/CI-github_binary-build-testing.yml
 create mode 100644 templates/package/CI-github_reusable-ros-tooling-source-build.yml
 rename templates/package/{CI-github_semi-binary-build.yml => CI-github_semi-binary-build-main.yml} (67%)
 create mode 100644 templates/package/CI-github_semi-binary-build-testing.yml
 create mode 100644 templates/package/_append_to_workflows_README_build_status_table.md

diff --git a/docs/use-cases/ros_packages/configure_repository.rst b/docs/use-cases/ros_packages/configure_repository.rst
index e8d61519..8d0217c4 100644
--- a/docs/use-cases/ros_packages/configure_repository.rst
+++ b/docs/use-cases/ros_packages/configure_repository.rst
@@ -10,10 +10,10 @@ Setup Repository CI configuration (GitHub)
 The script **has to be executed** from the *main* folder of your package.
 
 .. code-block:: bash
-   :caption: Usage of script for setting up new packages.
+   :caption: Usage of the script for setting up new packages.
    :name: setup-package-ci
 
-   setup-repository-ci "repo_name" "repo_namespace" ["first_package" "second_package" ...]
+   setup-repository-ci "repo_name" "repo_namespace" ["first_package second_package ..."]
 
 When executing the script, read all output carefully.
 If you make any wrong decision or enter incorrect data, use <CTRL>+C keys to terminate the script.
@@ -22,5 +22,38 @@ The script can add setup for multiple ROS2 versions.
 Simply follow the output.
 
 After a setup is created, you should go through the files and check if they are correct.
-Then try to push this to a GitHub-repository and open a PR.
+Please remember that each workflow you would like to run on schedule has to be in the default branch of your repository.
+Then try to push new files to a GitHub-repository and open a PR.
 You should then already see new workflows active.
+
+
+Details about CI setup and created workflows
+---------------------------------------------
+Code formatters and linters
+,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+For executing formatters ``pre-commit`` program is used and automatically setup.
+To use it in a cloned repositors (if there is ``.pre-commit-conifg.yaml`` file) execute once ``pre-commit install``.
+Clang-Format is used as the main code formatting program.
+
+
+Building packages
+,,,,,,,,,,,,,,,,,,
+This script creates three stages of build configuration for each ROS2 version.
+Each of those stages test different compatibility levels where users can experience issues with actively developed repositories and ensures future compatibility of the repository.
+In the following each stage is explained.
+
+``binary``
+  Building against released packages in a ROS distribution. The stage ensures that isolated build on a local machine and build-farm is possible. Unreleased dependencies can be defined in ``.repos``-file with suffix ``-not-released.<ros-distro>.repos``. This functionality is useful for new projects where packages are located in multiple repositories and when packages are released in a new ROS distribution (and you are not releasing regularly to ``rolling``).
+  The ``binary`` build has two workflows to build against ``main`` and ``testing`` ROS debian repositories. This is useful when for example API changes are done that involve multiple packages. Then it is expected that build against binary packages in the ``main`` repository fails, but if all the changes are released, building against ``testing`` repository has to work. Otherwise it means that the code has some regressions and cannot be synced to the ``main`` repositories (This is what is happening after maintainers from OpenRobotics announce a "Sync" or a ros distribution).
+  Scheduled ``binary`` build warns you that your package is incompatible with released dependencies (``main``) or that a package is breaking changes that will be released (``testing``). The first case should actually never happen if you are using all three building stages.
+
+``semi-binary``
+  Building against released core packages in a ROS distribution, but the main dependencies are build from source. The dependencies are defined in ``.repos``-file with suffix ``.<ros-distro>.repos``. This functionality is useful during development process when you have to sync changes in multiple repositories which are not released yet. This is the stage which should never fail if you are keeping your code up to date. Scheduled ``semi-binary`` builds warn you that some important dependency has changed something in their code so you can prepare yourself better for the next release.
+  There are two workflows building against ``main`` and ``testing`` ROS repositories with released packages.
+
+``source``
+  Core ROS packages are build from source. Expect the ``.repos`` files for ROS core it uses is the ``.repos``-file with suffix ``.<ros-distro>.repos`` for the package-specific dependencies. This stage makes mostly sense when working with ``rolling`` since other distribution of ROS should be API stable once released. This stage helps to anticipate braking changes in the ROS API that will be relevant the mid-term future.
+
+
+For the better overview and to avoid confusion for users there are two different tables with the workflow overview.
+One in the README file showing only the ``binary`` and ``semi-binary`` builds against ``main`` ROS repository, and another in the ``.github/workflows/README.md`` that shows all the workflows.
diff --git a/scripts/_Team_Defines.bash b/scripts/_Team_Defines.bash
index 4faebf1f..b1e160bd 100644
--- a/scripts/_Team_Defines.bash
+++ b/scripts/_Team_Defines.bash
@@ -55,7 +55,7 @@ alias create-new-package=$RosTeamWS_FRAMEWORK_SCRIPTS_PATH/create-new-package.ba
 
 alias setup-repository=$RosTeamWS_FRAMEWORK_SCRIPTS_PATH/setup-repository.bash
 
-alias setup-repository-ci=$RosTeamWS_FRAMEWORK_SCRIPTS_PATH/setup-ci-config.bash
+alias setup-repository-ci=$RosTeamWS_FRAMEWORK_SCRIPTS_PATH/setup-repository-ci.bash
 
 alias setup-ros-workspace=$RosTeamWS_FRAMEWORK_SCRIPTS_PATH/setup-ros-workspace.bash
 
diff --git a/scripts/setup-repository-ci.bash b/scripts/setup-repository-ci.bash
index 104b1dff..0d4cc7a0 100755
--- a/scripts/setup-repository-ci.bash
+++ b/scripts/setup-repository-ci.bash
@@ -15,13 +15,11 @@
 # limitations under the License.
 
 
-usage='setup-ci-config.bash "repo_name" "repo_namespace"'
-#
+usage='setup-repository-ci.bash "repo_name" "repo_namespace" ["first_package second_package ..."]'
 
 # Load Framework defines
 script_own_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )"
 source $script_own_dir/../setup.bash
-check_and_set_ros_distro_and_version ${ROS_DISTRO}
 
 repo_name=$1
 if [ -z "$1" ]; then
@@ -43,7 +41,7 @@ read -p "${RAW_TERMINAL_COLOR_BROWN}Are you setting CI for private repository?${
 private=${private:="no"}
 
 CI_FORMAT="ci-format"
-if  [[ "$choice" == "yes" ]]; then
+if  [[ "$private" == "yes" ]]; then
   CI_FORMAT="ci-format-private"
 fi
 
@@ -55,8 +53,10 @@ CI_REUSABLE_ROS_TOOLING="reusable-ros-tooling-source-build"
 PRE_RELEASE="prerelease-check"
 
 CI_ABI_COMPATIBILITY="abi-compatibility"
-CI_BINARY_BUILD="binary-build"
-CI_SEMI_BINARY_BUILD="semi-binary-build"
+CI_BINARY_BUILD_MAIN="binary-build-main"
+CI_BINARY_BUILD_TESTING="binary-build-testing"
+CI_SEMI_BINARY_BUILD_MAIN="semi-binary-build-main"
+CI_SEMI_BINARY_BUILD_TESTING="semi-binary-build-testing"
 CI_SOURCE_BUILD="source-build"
 CI_RHEL_BINARY_BUILD="rhel-binary-build"
 
@@ -73,14 +73,18 @@ CI_GENERAL_FILES=(
 
 CI_DISTRIBUTION_FILES=(
   $CI_ABI_COMPATIBILITY
-  $CI_BINARY_BUILD
-  $CI_SEMI_BINARY_BUILD
+  $CI_BINARY_BUILD_MAIN
+  $CI_BINARY_BUILD_TESTING
+  $CI_SEMI_BINARY_BUILD_MAIN
+  $CI_SEMI_BINARY_BUILD_TESTING
   $CI_SOURCE_BUILD
   $CI_RHEL_BINARY_BUILD
 )
 
-read -p "${RAW_TERMINAL_COLOR_BROWN}Name of the default ROS distro?${RAW_TERMINAL_COLOR_NC} [default: rolling] " default_ros_distro
+read -p "${RAW_TERMINAL_COLOR_BROWN}Name of the default ROS distro? Supported distributions(${rtw_supported_ros_distributions[*]})${RAW_TERMINAL_COLOR_NC} [default: rolling] " default_ros_distro
 default_ros_distro=${default_ros_distro:="rolling"}
+check_and_set_ros_distro_and_version "$default_ros_distro"
+default_ros_distro=${ros_distro}
 read -p "${RAW_TERMINAL_COLOR_BROWN}Name of the repository default branch?${RAW_TERMINAL_COLOR_NC} [default: master]: " default_branch
 default_branch=${default_branch:="master"}
 
@@ -90,7 +94,7 @@ mkdir -p .github/workflows
 # Coping and SEDing general CI files
 for CI_FILE in "${CI_GENERAL_FILES[@]}"; do
   cp -n ${PACKAGE_TEMPLATES}/CI-github_${CI_FILE}.yml .github/workflows/${CI_FILE}.yml
-  sed -i 's/\$ROS_DISTRO\$/'${default_ros_distro}'/g' .github/workflows/${CI_FILE}.yml
+  sed -i 's/\$ros_distro\$/'${default_ros_distro}'/g' .github/workflows/${CI_FILE}.yml
   sed -i 's/\$branch\$/'${default_branch}'/g' .github/workflows/${CI_FILE}.yml
   sed -i 's/\$NAME\$/'${repo_name}'/g' .github/workflows/${CI_FILE}.yml
   if [[ "$package_names" != "<none>" ]]; then
@@ -112,13 +116,15 @@ some_ros_distro=${default_ros_distro}
 some_branch=${default_branch}
 
 TMP_FILE=".f_tmp"
+TMP_FILE_2=".f_tmp2"
 touch $TMP_FILE
+touch $TMP_FILE_2
 
 while true; do
   for CI_FILE in "${CI_DISTRIBUTION_FILES[@]}"; do
     output_file_name="${some_ros_distro}-${CI_FILE}"
     cp -n ${PACKAGE_TEMPLATES}/CI-github_${CI_FILE}.yml .github/workflows/${output_file_name}.yml
-    sed -i 's/\$ROS_DISTRO\$/'${some_ros_distro}'/g' .github/workflows/${output_file_name}.yml
+    sed -i 's/\$ros_distro\$/'${some_ros_distro}'/g' .github/workflows/${output_file_name}.yml
     sed -i 's/\$Ros_distro\$/'${some_ros_distro^}'/g' .github/workflows/${output_file_name}.yml
     sed -i 's/\$branch\$/'${some_branch}'/g' .github/workflows/${output_file_name}.yml
     sed -i 's/\$NAME\$/'${repo_name}'/g' .github/workflows/${output_file_name}.yml
@@ -149,17 +155,30 @@ while true; do
   head -$CUT_LINE README.md >> $TMP_FILE
   cat ${PACKAGE_TEMPLATES}/_append_to_README_build_status_table.md >> $TMP_FILE
 
-  sed -i 's/\$branch\$/'${some_branch}'/g' $TMP_FILE
-  sed -i 's/\$ros_distro\$/'${some_ros_distro}'/g' $TMP_FILE
-  sed -i 's/\$Ros_distro\$/'${some_ros_distro^}'/g' $TMP_FILE
-  sed -i 's/\$NAME\$/'${repo_name}'/g' $TMP_FILE
-  sed -i 's/\$NAMESPACE\$/'${repo_namespace}'/g' $TMP_FILE
+  # Update .github/workflows/README.md file
+  cat ${PACKAGE_TEMPLATES}/_append_to_workflows_README_build_status_table.md > $TMP_FILE_2
+
+  # SED temp files
+  FILES_TO_UPDATE=("$TMP_FILE" "$TMP_FILE_2")
+  for SED_FILE in "${FILES_TO_UPDATE[@]}"; do
+    sed -i 's/\$branch\$/'${some_branch}'/g' $SED_FILE
+    sed -i 's/\$ros_distro\$/'${some_ros_distro}'/g' $SED_FILE
+    sed -i 's/\$Ros_distro\$/'${some_ros_distro^}'/g' $SED_FILE
+    sed -i 's/\$NAME\$/'${repo_name}'/g' $SED_FILE
+    sed -i 's/\$NAMESPACE\$/'${repo_namespace}'/g' $SED_FILE
+  done
 
+  # append to readme files
   tail -n +$CUT_LINE README.md >> $TMP_FILE
   mv $TMP_FILE README.md
 
+  cat ${PACKAGE_TEMPLATES}/_append_to_README_build_status.md >> $TMP_FILE_2
+
+  cat $TMP_FILE_2 >> .github/workflows/README.md
+  rm $TMP_FILE_2
+
   echo ""
-  echo -e "${TERMINAL_COLOR_BLUE}Added setup for ros distro '${some_ros_distro} on branch ${some_branch}.${TERMINAL_COLOR_NC}"
+  echo -e "${TERMINAL_COLOR_BLUE}Added setup for ros distro '${RAW_TERMINAL_COLOR_BROWN}${some_ros_distro}${TERMINAL_COLOR_BLUE}' on branch '${RAW_TERMINAL_COLOR_BROWN}${some_branch}${TERMINAL_COLOR_BLUE}'.${TERMINAL_COLOR_NC}"
   echo ""
 
   read -p "${RAW_TERMINAL_COLOR_BROWN}Do you want to configure ci for another ros distro?${RAW_TERMINAL_COLOR_NC} (yes/no) [no] " another
@@ -170,17 +189,22 @@ while true; do
   fi
 
   echo "" > $TMP_FILE
+  echo "" > $TMP_FILE_2
 
   read -p "${RAW_TERMINAL_COLOR_BROWN}Name of the ROS distro? ${RAW_TERMINAL_COLOR_NC}" some_ros_distro
   read -p "${RAW_TERMINAL_COLOR_BROWN}Name of the repository branch?: ${RAW_TERMINAL_COLOR_NC}" some_branch
 done
 
 # Setting up formatting
-cp -n ${PACKAGE_TEMPLATES}/.clang-format .
-cp -n ${PACKAGE_TEMPLATES}/.pre-commit-config.yaml .
-pre-commit install
-pre-commit autoupdate
-
+read -p "${RAW_TERMINAL_COLOR_BROWN}Do you want to setup formatting using pre-commit?${RAW_TERMINAL_COLOR_NC} (yes/no) [no]: " formatting
+formatting=${formatting:="no"}
+
+if  [[ "$formatting" == "yes" ]]; then
+  cp -n ${PACKAGE_TEMPLATES}/.clang-format .
+  cp -n ${PACKAGE_TEMPLATES}/.pre-commit-config.yaml .
+  pre-commit install
+  pre-commit autoupdate
+fi
 
 echo ""
 echo -e "${TERMINAL_COLOR_BLUE}FINISHED: Please check the generated file and setup package names.${TERMINAL_COLOR_NC}"
diff --git a/templates/package/.pre-commit-config.yaml b/templates/package/.pre-commit-config.yaml
index d7cf2daf..fa8d7510 100644
--- a/templates/package/.pre-commit-config.yaml
+++ b/templates/package/.pre-commit-config.yaml
@@ -15,7 +15,7 @@
 repos:
   # Standard hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.3.0
     hooks:
       - id: check-added-large-files
       - id: check-ast
@@ -33,13 +33,13 @@ repos:
 
   # Python hooks
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.31.0
+    rev: v2.37.3
     hooks:
     -   id: pyupgrade
         args: [--py36-plus]
 
   - repo: https://github.com/psf/black
-    rev: 21.12b0
+    rev: 22.6.0
     hooks:
       - id: black
         args: ["--line-length=99"]
@@ -52,7 +52,7 @@ repos:
       args: ["--ignore=D100,D101,D102,D103,D104,D105,D106,D107,D203,D212,D404"]
 
   - repo: https://github.com/pycqa/flake8
-    rev: 4.0.1
+    rev: 5.0.4
     hooks:
     - id: flake8
       args: ["--ignore=E501"]
@@ -63,7 +63,7 @@ repos:
       - id: clang-format
         name: clang-format
         description: Format files with ClangFormat.
-        entry: clang-format-10
+        entry: clang-format-12
         language: system
         files: \.(c|cc|cxx|cpp|frag|glsl|h|hpp|hxx|ih|ispc|ipp|java|js|m|proto|vert)$
         args: ['-fallback-style=none', '-i']
@@ -119,7 +119,7 @@ repos:
 
   # Docs - RestructuredText hooks
   - repo: https://github.com/PyCQA/doc8
-    rev: 0.10.1
+    rev: v1.0.0
     hooks:
       - id: doc8
         args: ['--max-line-length=100', '--ignore=D001']
diff --git a/templates/package/.pre-commit_wrapper.shb b/templates/package/.pre-commit_wrapper.shb
new file mode 100644
index 00000000..4eefebe8
--- /dev/null
+++ b/templates/package/.pre-commit_wrapper.shb
@@ -0,0 +1,6 @@
+#!/bin/bash
+source /opt/ros/rolling/setup.bash
+
+cmd=$1
+shift
+$cmd "$@"
diff --git a/templates/package/CI-github_abi-compatibility.yml b/templates/package/CI-github_abi-compatibility.yml
index f6c53d8a..2d4b1735 100644
--- a/templates/package/CI-github_abi-compatibility.yml
+++ b/templates/package/CI-github_abi-compatibility.yml
@@ -1,5 +1,8 @@
-name: ABI Compatibility Check
+name: $Ros_distro$ - ABI Compatibility Check
 on:
+  workflow_dispatch:
+    branches:
+      - $branch$
   pull_request:
     branches:
       - $branch$
@@ -11,7 +14,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ros-industrial/industrial_ci@master
         env:
-          ROS_DISTRO: $ROS_DISTRO$
+          ROS_DISTRO: $ros_distro$
           ROS_REPO: main
           ABICHECK_URL: github:${{ github.repository }}#${{ github.base_ref }}
           NOT_TEST_BUILD: true
diff --git a/templates/package/CI-github_binary-build.yml b/templates/package/CI-github_binary-build-main.yml
similarity index 68%
rename from templates/package/CI-github_binary-build.yml
rename to templates/package/CI-github_binary-build-main.yml
index 5210ca55..eb9a59c4 100644
--- a/templates/package/CI-github_binary-build.yml
+++ b/templates/package/CI-github_binary-build-main.yml
@@ -1,8 +1,11 @@
-name: $Ros_distro$ Binary Build
+name: $Ros_distro$ Binary Build - main
 # author: Denis Štogl <denis@stoglrobotics.de>
 # description: 'Build & test all dependencies from released (binary) packages.'
 
 on:
+  workflow_dispatch:
+    branches:
+      - $branch$
   pull_request:
     branches:
       - $branch$
@@ -17,6 +20,7 @@ jobs:
   binary:
     uses: ./.github/workflows/reusable-industrial-ci-with-cache.yml
     with:
-      ros_distro: $ROS_DISTRO$
-      upstream_workspace: $NAME$-not-released.$ROS_DISTRO$.repos
+      ros_distro: $ros_distro$
+      ros_repo: main
+      upstream_workspace: $NAME$-not-released.$ros_distro$.repos
       ref_for_scheduled_build: $branch$
diff --git a/templates/package/CI-github_binary-build-testing.yml b/templates/package/CI-github_binary-build-testing.yml
new file mode 100644
index 00000000..c6764060
--- /dev/null
+++ b/templates/package/CI-github_binary-build-testing.yml
@@ -0,0 +1,26 @@
+name: $Ros_distro$ Binary Build - testing
+# author: Denis Štogl <denis@stoglrobotics.de>
+# description: 'Build & test all dependencies from released (binary) packages.'
+
+on:
+  workflow_dispatch:
+    branches:
+      - $branch$
+  pull_request:
+    branches:
+      - $branch$
+  push:
+    branches:
+      - $branch$
+  schedule:
+    # Run every morning to detect flakiness and broken dependencies
+    - cron: '03 1 * * *'
+
+jobs:
+  binary:
+    uses: ./.github/workflows/reusable-industrial-ci-with-cache.yml
+    with:
+      ros_distro: $ros_distro$
+      ros_repo: testing
+      upstream_workspace: $NAME$-not-released.$ros_distro$.repos
+      ref_for_scheduled_build: $branch$
diff --git a/templates/package/CI-github_ci-coverage-build.yml b/templates/package/CI-github_ci-coverage-build.yml
index e765b419..09fb3b7c 100644
--- a/templates/package/CI-github_ci-coverage-build.yml
+++ b/templates/package/CI-github_ci-coverage-build.yml
@@ -1,5 +1,8 @@
 name: Coverage Build
 on:
+  workflow_dispatch:
+    branches:
+      - $branch$
   pull_request:
     branches:
       - $branch$
@@ -7,17 +10,17 @@ on:
 jobs:
   coverage:
     name: coverage build
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
     env:
-      ROS_DISTRO: $ROS_DISTRO$
+      ROS_DISTRO: $ros_distro$
     steps:
-      - uses: ros-tooling/setup-ros@v0.2
+      - uses: ros-tooling/setup-ros@0.3.4
         with:
           required-ros-distributions: ${{ env.ROS_DISTRO }}
       - uses: actions/checkout@v3
-      - uses: ros-tooling/action-ros-ci@v0.2
+      - uses: ros-tooling/action-ros-ci@0.2.6
         with:
           target-ros2-distro: ${{ env.ROS_DISTRO }}
           import-token: ${{ secrets.GITHUB_TOKEN }}
@@ -33,13 +36,12 @@ jobs:
               }
             }
           colcon-mixin-repository: https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml
-          skip-tests: true
-      - uses: codecov/codecov-action@v1.0.14
+      - uses: codecov/codecov-action@v3.1.0
         with:
           file: ros_ws/lcov/total_coverage.info
           flags: unittests
           name: codecov-umbrella
-      - uses: actions/upload-artifact@v1
+      - uses: actions/upload-artifact@v3.1.0
         with:
-          name: colcon-logs-${{ matrix.os }}
+          name: colcon-logs-coverage-rolling
           path: ros_ws/log
diff --git a/templates/package/CI-github_ci-format-private.yml b/templates/package/CI-github_ci-format-private.yml
index 74b114ef..05ecb08a 100644
--- a/templates/package/CI-github_ci-format-private.yml
+++ b/templates/package/CI-github_ci-format-private.yml
@@ -23,7 +23,7 @@ jobs:
       with:
         python-version: 3.9.7
     - name: Install system hooks
-      run: sudo apt install -qq clang-format-11 cppcheck
+      run: sudo apt install -qq clang-format-12 cppcheck
     - uses: pre-commit/action@v2.0.3
       with:
         extra_args: --all-files --hook-stage manual
diff --git a/templates/package/CI-github_ci-format.yml b/templates/package/CI-github_ci-format.yml
index 77c35301..4db758fe 100644
--- a/templates/package/CI-github_ci-format.yml
+++ b/templates/package/CI-github_ci-format.yml
@@ -17,7 +17,7 @@ jobs:
       with:
         python-version: 3.9.7
     - name: Install system hooks
-      run: sudo apt install -qq clang-format-11 cppcheck
+      run: sudo apt install -qq clang-format-12 cppcheck
     - uses: pre-commit/action@v2.0.3
       with:
         extra_args: --all-files --hook-stage manual
diff --git a/templates/package/CI-github_ci-ros-lint.yml b/templates/package/CI-github_ci-ros-lint.yml
index 2872a841..ca5c081a 100644
--- a/templates/package/CI-github_ci-ros-lint.yml
+++ b/templates/package/CI-github_ci-ros-lint.yml
@@ -15,7 +15,7 @@ jobs:
     - uses: ros-tooling/setup-ros@v0.2
     - uses: ros-tooling/action-ros-lint@v0.1
       with:
-        distribution: $ROS_DISTRO$
+        distribution: $ros_distro$
         linter: ${{ matrix.linter }}
         package-name:
           $PKG_NAME$
@@ -32,7 +32,7 @@ jobs:
     - uses: ros-tooling/setup-ros@v0.2
     - uses: ros-tooling/action-ros-lint@v0.1
       with:
-        distribution: $ROS_DISTRO$
+        distribution: $ros_distro$
         linter: cpplint
         arguments: "--linelength=100 --filter=-whitespace/newline"
         package-name:
diff --git a/templates/package/CI-github_docs-sphinx-build-check.yml b/templates/package/CI-github_docs-sphinx-build-check.yml
index e049b597..791c48a7 100644
--- a/templates/package/CI-github_docs-sphinx-build-check.yml
+++ b/templates/package/CI-github_docs-sphinx-build-check.yml
@@ -3,7 +3,7 @@ on:
   pull_request:
   push:
     branches:
-      - $ROS_DISTRO$
+      - $ros_distro$
   schedule:
     # Run every day to detect flakiness and broken dependencies
     - cron: '23 5 * * *'
diff --git a/templates/package/CI-github_docs-sphinx-make-page.yml b/templates/package/CI-github_docs-sphinx-make-page.yml
index f47a7a36..5d93dcf4 100644
--- a/templates/package/CI-github_docs-sphinx-make-page.yml
+++ b/templates/package/CI-github_docs-sphinx-make-page.yml
@@ -2,7 +2,7 @@ name: "Build & Deploy Page"
 on:
   push:
     branches:
-      - $ROS_DISTRO$
+      - $ros_distro$
 
 jobs:
   build:
diff --git a/templates/package/CI-github_reusable-industrial-ci-with-cache.yml b/templates/package/CI-github_reusable-industrial-ci-with-cache.yml
index 1a9818f3..0ff359d0 100644
--- a/templates/package/CI-github_reusable-industrial-ci-with-cache.yml
+++ b/templates/package/CI-github_reusable-industrial-ci-with-cache.yml
@@ -54,18 +54,18 @@ jobs:
     env:
       CCACHE_DIR: ${{ github.workspace }}/${{ inputs.ccache_dir }}
       BASEDIR: ${{ github.workspace }}/${{ inputs.basedir }}
-      CACHE_PREFIX: ${{ inputs.ros_distro }}-${{ inputs.os_code_name }}-${{ matrix.ROS_REPO }}-${{ github.job }}
+      CACHE_PREFIX: ${{ inputs.ros_distro }}-${{ inputs.os_code_name }}-${{ inputs.ros_repo }}-${{ github.job }}
     steps:
       - name: Checkout ${{ inputs.ref }} when build is not scheduled
-        if: ${{ (inputs.ros_repo == 'all' || inputs.ros_repo == matrix.ROS_REPO) && github.event_name != 'schedule' }}
+        if: ${{ github.event_name != 'schedule' }}
         uses: actions/checkout@v3
       - name: Checkout ${{ inputs.ref }} on scheduled build
-        if: ${{ (inputs.ros_repo == 'all' || inputs.ros_repo == matrix.ROS_REPO) && github.event_name == 'schedule' }}
+        if: ${{ github.event_name == 'schedule' }}
         uses: actions/checkout@v3
         with:
           ref: ${{ inputs.ref_for_scheduled_build }}
       - name: cache target_ws
-        if: ${{ ! matrix.env.CCOV && (inputs.ros_repo == 'all' || inputs.ros_repo == matrix.ROS_REPO) }}
+        if: ${{ ! matrix.env.CCOV }}
         uses: pat-s/always-upload-cache@v2.1.5
         with:
           path: ${{ env.BASEDIR }}/target_ws
@@ -73,7 +73,6 @@ jobs:
           restore-keys: |
             target_ws-${{ env.CACHE_PREFIX }}-${{ hashFiles('**/CMakeLists.txt', '**/package.xml') }}
       - name: cache ccache
-        if: ${{ inputs.ros_repo == 'all' || inputs.ros_repo == matrix.ROS_REPO }}
         uses: pat-s/always-upload-cache@v2.1.5
         with:
           path: ${{ env.CCACHE_DIR }}
@@ -81,8 +80,7 @@ jobs:
           restore-keys: |
             ccache-${{ env.CACHE_PREFIX }}-${{ github.sha }}
             ccache-${{ env.CACHE_PREFIX }}
-      - if: ${{ inputs.ros_repo == 'all' || inputs.ros_repo == matrix.ROS_REPO }}
-        uses: 'ros-industrial/industrial_ci@master'
+      - uses: 'ros-industrial/industrial_ci@master'
         env:
           UPSTREAM_WORKSPACE: ${{ inputs.upstream_workspace }}
           ROS_DISTRO: ${{ inputs.ros_distro }}
@@ -90,13 +88,9 @@ jobs:
           OS_CODE_NAME: ${{ inputs.os_code_name }}
           BEFORE_INSTALL_UPSTREAM_DEPENDENCIES: ${{ inputs.before_install_upstream_dependencies }}
       - name: prepare target_ws for cache
-        if: ${{ always() && ! matrix.env.CCOV && (inputs.ros_repo == 'all' || inputs.ros_repo == matrix.ROS_REPO) }}
+        if: ${{ always() && ! matrix.env.CCOV }}
         run: |
           du -sh ${{ env.BASEDIR }}/target_ws
           sudo find ${{ env.BASEDIR }}/target_ws -wholename '*/test_results/*' -delete
           sudo rm -rf ${{ env.BASEDIR }}/target_ws/src
           du -sh ${{ env.BASEDIR }}/target_ws
-      - name: Is job skipped?
-        if: ${{ ! (inputs.ros_repo == 'all' || inputs.ros_repo == matrix.ROS_REPO) }}
-        run: |
-          echo "This job is skpped!"
diff --git a/templates/package/CI-github_reusable-ros-tooling-source-build.yml b/templates/package/CI-github_reusable-ros-tooling-source-build.yml
new file mode 100644
index 00000000..92131759
--- /dev/null
+++ b/templates/package/CI-github_reusable-ros-tooling-source-build.yml
@@ -0,0 +1,48 @@
+name: Reusable industrial_ci Workflow with Cache
+# Reusable action to simplify dealing with ROS/ROS2 industrial_ci builds with cache
+# author: Denis Štogl <denis@stoglrobotics.de>
+
+on:
+  workflow_call:
+    inputs:
+      ros_distro:
+        description: 'ROS2 distribution name'
+        required: true
+        type: string
+      ref:
+        description: 'Reference on which the repo should be checkout. Usually is this name of a branch or a tag.'
+        required: true
+        type: string
+      ros2_repo_branch:
+        description: 'Branch in the ros2/ros2 repozitory from which ".repos" should be used. Possible values: master (Rolling), humble, galactic, foxy.'
+        default: 'master'
+        required: false
+        type: string
+
+jobs:
+  reusable_ros_tooling_source_build:
+    name: ${{ inputs.ros_distro }} ubuntu-22.04
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: ros-tooling/setup-ros@0.3.4
+        with:
+          required-ros-distributions: ${{ inputs.ros_distro }}
+      - uses: actions/checkout@v3
+        with:
+          ref: ${{ inputs.ref }}
+      - uses: ros-tooling/action-ros-ci@0.2.6
+        with:
+          target-ros2-distro: ${{ inputs.ros_distro }}
+          # build all packages listed in the meta package
+          package-name:
+            $PKG_NAME$
+          vcs-repo-file-url: |
+            https://raw.githubusercontent.com/ros2/ros2/${{ inputs.ros2_repo_branch }}/ros2.repos
+            https://raw.githubusercontent.com/${{ github.repository }}/${{ github.sha }}/$NAME$.${{ inputs.ros_distro }}.repos?token=${{ secrets.GITHUB_TOKEN }}
+          colcon-mixin-repository: https://raw.githubusercontent.com/colcon/colcon-mixin-repository/master/index.yaml
+      - uses: actions/upload-artifact@v1
+        with:
+          name: colcon-logs-ubuntu-22.04
+          path: ros_ws/log
diff --git a/templates/package/CI-github_rhel-binary-build.yml b/templates/package/CI-github_rhel-binary-build.yml
index 92ed1583..c3882da3 100644
--- a/templates/package/CI-github_rhel-binary-build.yml
+++ b/templates/package/CI-github_rhel-binary-build.yml
@@ -1,5 +1,8 @@
 name: $Ros_distro$ RHEL Binary Build
 on:
+  workflow_dispatch:
+    branches:
+      - $branch$
   pull_request:
     branches:
       - $branch$
@@ -12,12 +15,12 @@ on:
 
 
 jobs:
-  $ROS_DISTRO$_rhel_binary:
+  $ros_distro$_rhel_binary:
     name: $Ros_distro$ RHEL binary build
     runs-on: ubuntu-latest
     env:
-      ROS_DISTRO: $ROS_DISTRO$
-    container: jaronl/ros:$ROS_DISTRO$-alma
+      ROS_DISTRO: $ros_distro$
+    container: jaronl/ros:$ros_distro$-alma
     steps:
       - uses: actions/checkout@v3
         with:
diff --git a/templates/package/CI-github_semi-binary-build.yml b/templates/package/CI-github_semi-binary-build-main.yml
similarity index 67%
rename from templates/package/CI-github_semi-binary-build.yml
rename to templates/package/CI-github_semi-binary-build-main.yml
index bfcbdcdf..5546218a 100644
--- a/templates/package/CI-github_semi-binary-build.yml
+++ b/templates/package/CI-github_semi-binary-build-main.yml
@@ -1,7 +1,10 @@
-name: $Ros_distro$ Semi-Binary Build
+name: $Ros_distro$ Semi-Binary Build - main
 # description: 'Build & test that compiles the main dependencies from source.'
 
 on:
+  workflow_dispatch:
+    branches:
+      - $branch$
   pull_request:
     branches:
       - $branch$
@@ -16,7 +19,7 @@ jobs:
   semi_binary:
     uses: ./.github/workflows/reusable-industrial-ci-with-cache.yml
     with:
-      ros_distro: $ROS_DISTRO$
-      ros_repo: $ROS_REPO$
-      upstream_workspace: $NAME$.$ROS_DISTRO$.repos
+      ros_distro: $ros_distro$
+      ros_repo: main
+      upstream_workspace: $NAME$.$ros_distro$.repos
       ref_for_scheduled_build: $branch$
diff --git a/templates/package/CI-github_semi-binary-build-testing.yml b/templates/package/CI-github_semi-binary-build-testing.yml
new file mode 100644
index 00000000..c922473c
--- /dev/null
+++ b/templates/package/CI-github_semi-binary-build-testing.yml
@@ -0,0 +1,25 @@
+name: $Ros_distro$ Semi-Binary Build - testing
+# description: 'Build & test that compiles the main dependencies from source.'
+
+on:
+  workflow_dispatch:
+    branches:
+      - $branch$
+  pull_request:
+    branches:
+      - $branch$
+  push:
+    branches:
+      - $branch$
+  schedule:
+    # Run every morning to detect flakiness and broken dependencies
+    - cron: '33 1 * * *'
+
+jobs:
+  semi_binary:
+    uses: ./.github/workflows/reusable-industrial-ci-with-cache.yml
+    with:
+      ros_distro: $ros_distro$
+      ros_repo: testing
+      upstream_workspace: $NAME$.$ros_distro$.repos
+      ref_for_scheduled_build: $branch$
diff --git a/templates/package/CI-github_source-build.yml b/templates/package/CI-github_source-build.yml
index c1a1173a..a8959762 100644
--- a/templates/package/CI-github_source-build.yml
+++ b/templates/package/CI-github_source-build.yml
@@ -1,5 +1,8 @@
 name: $Ros_distro$ Source Build
 on:
+  workflow_dispatch:
+    branches:
+      - $branch$
   push:
     branches:
       - $branch$
@@ -11,5 +14,6 @@ jobs:
   source:
     uses: ./.github/workflows/reusable-ros-tooling-source-build.yml
     with:
-      ros_distro: $ROS_DISTRO$
+      ros_distro: $ros_distro$
       ref: $branch$
+      ros2_repo_branch: $ros_distro$
diff --git a/templates/package/_append_to_README_ROS_Intro.md b/templates/package/_append_to_README_ROS_Intro.md
index 2e3408d0..6b9f95ff 100644
--- a/templates/package/_append_to_README_ROS_Intro.md
+++ b/templates/package/_append_to_README_ROS_Intro.md
@@ -6,16 +6,17 @@ If you are familiar with ROS2, here are the quick-and-dirty build instructions.
   ```
   cd $COLCON_WS
   git clone git@github.com:$NAMESPACE$/$NAME$.git src/$NAME$
-  vcs import src --input src/$NAME$/$NAME$.repos
+  vcs import src --input src/$NAME$/$NAME$.<ros-distro>.repos
   rosdep install --ignore-src --from-paths src -y -r
   colcon build --cmake-args -DCMAKE_BUILD_TYPE=Release    # Faster and more efficient build type
+  cd ..
   ```
 If you end up with missing dependencies, install them using commands from [Setup ROS Workspace](#setup-ros-workspace) section.
 
 # How to use this Package and ROS Introduction
 
  - [Install and Build](#install-and-build)
-   * [Install ROS $Ros_distro$ and Development Tooling](#install-ros-$ROS_DISTRO$-and-development-tooling)
+   * [Install ROS $Ros_distro$ and Development Tooling](#install-ros-$ros_distro$-and-development-tooling)
    * [Setup ROS Workspace](#setup-ros-workspace)
    * [Configure and Build Workspace](#configure-and-build-workspace)
  - [Running Executables](#running-executables)
@@ -34,7 +35,7 @@ These instructions assume you are running Ubuntu 20.04:
    You can stop following along with the tutorial after you complete the section titled: [Environment setup](https://index.ros.org/doc/ros2/Installation/$Ros_distro$/Linux-Install-Debians/#environment-setup).
    Make sure you setup your environment with:
    ```
-   source /opt/ros/$ROS_DISTRO$/setup.bash
+   source /opt/ros/$ros_distro$/setup.bash
    ```
 
    > **NOTE:** You may want to add that line to your `~/.bashrc`
@@ -91,7 +92,7 @@ To configure and build workspace execute following commands:
   colcon build --symlink-install --mixin rel-with-deb-info compile-commands ccache
   ```
 
-## Running Executales
+## Running Executable
 
 See `README.md` files of the packages for information regarding running executables.
 
diff --git a/templates/package/_append_to_README_build_status.md b/templates/package/_append_to_README_build_status.md
index b62f5070..2cc57dda 100644
--- a/templates/package/_append_to_README_build_status.md
+++ b/templates/package/_append_to_README_build_status.md
@@ -2,18 +2,19 @@
 ## Build status
 
 
-
 ### Explanation of different build types
 
 **NOTE**: There are three build stages checking current and future compatibility of the package.
 
+[Detailed build status](.github/workflows/README.md)
+
 1. Binary builds - against released packages (main and testing) in ROS distributions. Shows that direct local build is possible.
 
-   Uses repos file: `src/$NAME$/$NAME$-not-released.<ros-distro>.repos`
+   Uses repos file: `$NAME$-not-released.<ros-distro>.repos`
 
 1. Semi-binary builds - against released core ROS packages (main and testing), but the immediate dependencies are pulled from source.
    Shows that local build with dependencies is possible and if fails there we can expect that after the next package sync we will not be able to build.
 
-   Uses repos file: `src/$NAME$/$NAME$.repos`
+   Uses repos file: `$NAME$.repos`
 
 1. Source build - also core ROS packages are build from source. It shows potential issues in the mid future.
diff --git a/templates/package/_append_to_README_build_status_table.md b/templates/package/_append_to_README_build_status_table.md
index c6e03737..0c266aff 100644
--- a/templates/package/_append_to_README_build_status_table.md
+++ b/templates/package/_append_to_README_build_status_table.md
@@ -1,4 +1,4 @@
 
 ROS2 Distro | Branch | Build status | Documentation | Released packages
 :---------: | :----: | :----------: | :-----------: | :---------------:
-**$Ros_distro$** | [`$ros_distro$`](https://github.com/$NAMESPACE$/$NAME$/tree/$ros_distro$) | [![$Ros_distro$ Binary Build](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-binary-build.yml/badge.svg?branch=$branch$)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-binary-build.yml?branch=$branch$) <br /> [![$Ros_distro$ Semi-Binary Build](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-semi-binary-build.yml/badge.svg?branch=$branch$)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-semi-binary-build.yml?branch=$branch$) <br /> [![$Ros_distro$ Source Build](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-source-build.yml/badge.svg?branch=$branch$)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-source-build.yml?branch=$branch$) | [![Doxygen Doc Deployment](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/doxygen-deploy.yml/badge.svg)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/doxygen-deploy.yml) <br /> [Generated Doc](https://$NAMESPACE$.github.io/$NAME$_Documentation/$ros_distro$/html/index.html) | [$NAME$](https://index.ros.org/p/$NAME$/#$ros_distro$)
+**$Ros_distro$** | [`$ros_distro$`](https://github.com/$NAMESPACE$/$NAME$/tree/$ros_distro$) | [![$Ros_distro$ Binary Build](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-binary-build-main.yml/badge.svg?branch=$branch$)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-binary-build-main.yml?branch=$branch$) <br /> [![$Ros_distro$ Semi-Binary Build](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-semi-binary-build-main.yml/badge.svg?branch=$branch$)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-semi-binary-build-main.yml?branch=$branch$) | [![Doxygen Doc Deployment](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/doxygen-deploy.yml/badge.svg)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/doxygen-deploy.yml) <br /> [Generated Doc](https://$NAMESPACE$.github.io/$NAME$_Documentation/$ros_distro$/html/index.html) | [$NAME$](https://index.ros.org/p/$NAME$/#$ros_distro$)
diff --git a/templates/package/_append_to_workflows_README_build_status_table.md b/templates/package/_append_to_workflows_README_build_status_table.md
new file mode 100644
index 00000000..8082c0fd
--- /dev/null
+++ b/templates/package/_append_to_workflows_README_build_status_table.md
@@ -0,0 +1,4 @@
+
+ROS2 Distro | Branch | Build status | Documentation | Released packages
+:---------: | :----: | :----------: | :-----------: | :---------------:
+**$Ros_distro$** | [`$ros_distro$`](https://github.com/$NAMESPACE$/$NAME$/tree/$ros_distro$) | [![$Ros_distro$ Binary Build](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-binary-build-main.yml/badge.svg?branch=$branch$)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-binary-build-main.yml?branch=$branch$) <br /> [![$Ros_distro$ Binary Build](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-binary-build-testing.yml/badge.svg?branch=$branch$)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-binary-build-testing.yml?branch=$branch$) <br /> [![$Ros_distro$ Semi-Binary Build](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-semi-binary-build-main.yml/badge.svg?branch=$branch$)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-semi-binary-build-main.yml?branch=$branch$) <br /> [![$Ros_distro$ Semi-Binary Build](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-semi-binary-build-testing.yml/badge.svg?branch=$branch$)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-semi-binary-build-testing.yml?branch=$branch$) <br /> [![$Ros_distro$ Source Build](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-source-build.yml/badge.svg?branch=$branch$)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/$ros_distro$-source-build.yml?branch=$branch$) | [![Doxygen Doc Deployment](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/doxygen-deploy.yml/badge.svg)](https://github.com/$NAMESPACE$/$NAME$/actions/workflows/doxygen-deploy.yml) <br /> [Generated Doc](https://$NAMESPACE$.github.io/$NAME$_Documentation/$ros_distro$/html/index.html) | [$NAME$](https://index.ros.org/p/$NAME$/#$ros_distro$)