From a53a926d1b300a35f948dfe02eece51ace42a24b Mon Sep 17 00:00:00 2001 From: Masaki Baba Date: Sun, 4 Aug 2024 18:25:13 +0900 Subject: [PATCH] feat(dependency_checker): add dependency checker script (#89) * add dependency_checker Signed-off-by: a-maumau * fix Signed-off-by: a-maumau * style(pre-commit): autofix * fix bag Signed-off-by: a-maumau * apply markdownlint Signed-off-by: a-maumau * fix typo Co-authored-by: Yutaka Kondo fix typo Co-authored-by: Yutaka Kondo Signed-off-by: a-maumau * add dep ament_cmake Signed-off-by: a-maumau * fix based on shellcheck Signed-off-by: a-maumau * fix words Signed-off-by: a-maumau * add cspell ignore Signed-off-by: a-maumau --------- Signed-off-by: a-maumau Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Yutaka Kondo --- autoware_dependency_checker/CMakeLists.txt | 10 ++ autoware_dependency_checker/README.md | 34 ++++ autoware_dependency_checker/package.xml | 17 ++ .../scripts/dependency_checker.sh | 157 ++++++++++++++++++ 4 files changed, 218 insertions(+) create mode 100644 autoware_dependency_checker/CMakeLists.txt create mode 100644 autoware_dependency_checker/README.md create mode 100644 autoware_dependency_checker/package.xml create mode 100755 autoware_dependency_checker/scripts/dependency_checker.sh diff --git a/autoware_dependency_checker/CMakeLists.txt b/autoware_dependency_checker/CMakeLists.txt new file mode 100644 index 00000000..534d2d1b --- /dev/null +++ b/autoware_dependency_checker/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.5) +project(autoware_dependency_checker) + +find_package(ament_cmake REQUIRED) + +ament_package() + +install(PROGRAMS + scripts/dependency_checker.sh + DESTINATION lib/${PROJECT_NAME}) diff --git a/autoware_dependency_checker/README.md b/autoware_dependency_checker/README.md new file mode 100644 index 00000000..92b3f677 --- /dev/null +++ b/autoware_dependency_checker/README.md @@ -0,0 +1,34 @@ +# autoware_dependency_checker + +This package provides a script for checking whether each package's dependencies listed in a package.xml are used or not. +Currently, it mainly checks packages that start with `autoware_`. + +## Dependency Checking + +The script will try to match the dependencies and the headers by reading the dependencies listed in package.xml and the included headers in the source files. + +Some dependency in `package.xml` and the included header might differ. +The following table shows the matching between dependency names and headers: + +| from | to | description | +| ------------------ | ------------------ | --------------------------------- | +| autoware_pkg_name | autoware/pkg_name | Usually this style should be used | +| autoware\_\*\_msgs | autoware\_\*\_msgs | For messages | +| autoware_other_pkg | autoware_other_pkg | E.g. autoware_lanelet2_extension | + +## Usage + +```Text +# build +$ cd to/autoware_tools +$ colcon build --symlink-install --cmake-args --packages-up-to autoware_dependency_checker +$ source + +# run +$ cd to/your/autoware +$ ros2 run autoware_dependency_checker dependency_checker.sh + +# run in some package +$ cd to/some/package +$ ros2 run autoware_dependency_checker dependency_checker.sh +``` diff --git a/autoware_dependency_checker/package.xml b/autoware_dependency_checker/package.xml new file mode 100644 index 00000000..8166a429 --- /dev/null +++ b/autoware_dependency_checker/package.xml @@ -0,0 +1,17 @@ + + + + autoware_dependency_checker + 0.1.0 + The autoware_dependency_checker package + Yutaka Kondo + Apache License 2.0 + Vincent Richard + Masaki Baba + + ament_cmake + + + ament_cmake + + diff --git a/autoware_dependency_checker/scripts/dependency_checker.sh b/autoware_dependency_checker/scripts/dependency_checker.sh new file mode 100755 index 00000000..7bf4b8ab --- /dev/null +++ b/autoware_dependency_checker/scripts/dependency_checker.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# This script finds recursively all package.xml in the src/ subdirectory +# Then check if all packages marked are effectively used somewhere in the package. +# Packages marked normally either provide shared libraries or custom messages + +# Packages that should always be , not +KNOWN_BUILDTOOLS="rosidl_default_generators;autoware_cmake;eigen3_cmake_module" + +current_dir=$(pwd) + +# Out of pattern packages +# These will get checked with `package_name/.*` pattern +# autoware_*_msg is handled in the following process, so you don't need to add here +EXCLUDE_PACKAGES=( + "autoware_ad_api_specs" + "autoware_lanelet2_extension" + "autoware_raw_vehicle_cmd_converter" +) + +# Find all package names under the current directory +mapfile -t ALL_PACKAGES < <(find "$current_dir" \ + -not \( -path "$current_dir/install" -prune \) \ + -not \( -path "$current_dir/build" -prune \) \ + -name "package.xml" -print0 | xargs -0 -n 1 dirname | xargs -n 1 basename | sort -u) + +# Find all autoware packages starting with "autoware_" +# These packages will get checked with `autoware/pkg_name/.*` +BASE_RULE_TARGETS=() +for pkg_name in "${ALL_PACKAGES[@]}"; do + if [[ $pkg_name == autoware_* ]]; then + filter_flag=0 + + for exclude_pkg_name in "${EXCLUDE_PACKAGES[@]}"; do + if [[ $pkg_name == "$exclude_pkg_name" ]]; then + filter_flag=1 + break + fi + done + if [[ $filter_flag -eq 0 ]]; then + BASE_RULE_TARGETS+=("$pkg_name") + fi + fi +done + +pkgs=$(find "$current_dir" \ + -not \( -path "$current_dir/install" -prune \) \ + -not \( -path "$current_dir/build" -prune \) \ + -name "package.xml") + +for pkg in $pkgs; do + echo "--- Checking $pkg ---" + # Get all packages marked . For example: + # dep_name + deps=$(grep -d skip -oP '^\s*\K[^<]+' "$pkg") + dir=$(dirname "$pkg") + + if [[ ! -f "$dir/CMakeLists.txt" ]]; then + echo "Skipping package with no CMakeLists.txt (is python package?)" + echo "" + continue + fi + + for dep in $deps; do + # filter out buildtools (should not use ) + if grep -q "$dep" <<<"$KNOWN_BUILDTOOLS"; then + echo "$dep should rather be marked as , not " + continue + fi + + # filter out ament stuff (should not use ) + if [[ $dep =~ ament_.*$ ]]; then + echo "$dep should rather be marked as either or , not " + continue + fi + + # filter out python stuff (should not use ) + if [[ $dep =~ python-.*$ ]]; then + echo "$dep should rather be marked as either or , not " + continue + fi + + # By convention, dependency headers must be prefixed with the package name. For Example: + # tier4_autoware_utils/geometry/geometry.hpp + header_regex="$dep/.*" + + # System dependencies don't follow this rule. They all have custom names. + [[ $dep =~ libpcl-.*$ ]] && header_regex="pcl/.*" + [[ $dep =~ libboost-.*$ ]] && header_regex="boost/.*" + [[ $dep =~ libqt5-.*$ ]] && header_regex="Q.*" + [[ $dep == "qtbase5-dev" ]] && header_regex="Q.*" + [[ $dep == "libopencv-dev" ]] && header_regex="opencv2/.*" + [[ $dep == "pugixml-dev" ]] && header_regex="pugixml.hpp" + [[ $dep == "yaml_cpp_vendor" ]] && header_regex="yaml-cpp/yaml.h" + [[ $dep == "nlohmann-json-dev" ]] && header_regex="nlohmann/json.hpp" + [[ $dep == "range-v3" ]] && header_regex="range/v3/.*" + [[ $dep == "libcpprest-dev" ]] && header_regex="cpprest/.*" + [[ $dep == "eigen" ]] && header_regex="Eigen/.*" + [[ $dep == "libnl-3-dev" ]] && header_regex="netlink/.*" + [[ $dep == "libpcap" ]] && header_regex="pcap.h" + [[ $dep == "cgal" ]] && header_regex="CGAL/.*" + [[ $dep == "osqp_vendor" ]] && header_regex="osqp/.*" + [[ $dep == "magic_enum" ]] && header_regex="magic_enum.hpp" + [[ $dep == "geographiclib" ]] && header_regex="GeographicLib/.*" + # Some autoware packages don't follow the convention either... + [[ $dep == "autoware_auto_common" ]] && header_regex="(common|helper_functions)/.*" + # cspell: ignore multigrid + [[ $dep == "ndt_omp" ]] && header_regex="(pclomp|multigrid_pclomp)/.*" + [[ $dep == "planning_test_utils" ]] && header_regex="planning_interface_test_manager/.*" + [[ $dep == "shape_estimation" ]] && header_regex="autoware/shape_estimation/.*" + # Add more as needed... + + # Check the dependency with the including rule: + # autoware_pkg_name -> autoware/pkg_name/.* + # autoware_*_msgs -> autoware_*_msgs/.* + for autoware_pkg in "${BASE_RULE_TARGETS[@]}"; do + if [[ $dep == "$autoware_pkg" ]]; then + # for the autoware_*_msgs + if [[ $autoware_pkg == *_msgs ]]; then + source_name="${autoware_pkg}" + else + # replace `autoware_` with `autoware/` + source_name="${autoware_pkg/autoware_/autoware/}" + fi + + header_regex="${source_name}/.*" + break + fi + done + + # Look for C/C++ includes. For example: + # #include "tier4_autoware_utils/geometry/geometry.hpp" + # Note: whether these includes are actually useful is out-of-scope of this script + # There are many great tools for that. + include_regex="^#include [<\"]${header_regex}[>\"]" + + # Dependencies defining custom messages may also be re-used in other messages or services + # By convention, these packages must be named "*_msgs", excepted for "builtin_interfaces" + if [[ $dep =~ .*_msgs$ ]] || [[ $dep == "builtin_interfaces" ]]; then + if grep -wIPrq "$dep" "$dir" --include \*.msg --include \*.srv --include \*.idl; then + continue # found! + fi + fi + + # Check if the dependency is included anywhere in the source files + if grep -wIPrq "$include_regex" "$dir" \ + --include \*.c --include \*.cc --include \*.cpp --include \*.h --include \*.hpp; then + continue # found! + fi + + echo "$dep seems not to be used" + + # /!\ WARNING - DANGEROUS /!\ + # Uncomment to delete unused dependency! + # sed -i "/$dep<\/depend>/d" $pkg + done + echo "" +done