Merge changes from master branch into main branch

.github/workflows/ci.yaml

@@ -0,0 +1,52 @@
+name: superflore-ci
+
+on:
+  workflow_dispatch:
+  push:
+    branches: ['master']
+  pull_request:
+
+jobs:
+    build:
+      strategy:
+        matrix:
+          os: ["ubuntu-20.04", "ubuntu-22.04"]
+          python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+      name: superflore tests
+      runs-on: ${{matrix.os}}
+      steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{matrix.python}}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{matrix.python}}
+      - name: Install dependencies
+        run: |
+          echo "Set locale"
+          sudo apt update && sudo apt install locales
+          sudo locale-gen en_US en_US.UTF-8
+          sudo update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8
+          export LANG=en_US.UTF-8
+
+          echo "Enable required repositories"
+          sudo apt install software-properties-common
+          sudo add-apt-repository universe
+
+          sudo apt update && sudo apt install curl -y
+          sudo curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg
+
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(. /etc/os-release && echo $UBUNTU_CODENAME) main" | sudo tee /etc/apt/sources.list.d/ros2.list > /dev/null
+
+          sudo apt-get update -qq
+          sudo apt-get install dpkg -y
+          sudo apt-get install -y python3-rosdep
+          pip install -r requirements.txt
+      - name: Run tests
+        run: |
+          sudo rosdep init
+          rosdep update
+          python -m 'nose' --verbose --exclude test_pull --exclude test_run --exclude test_logger_output --exclude test_failed_to_create  --exclude test_generate_installers --ignore-files test_ebuild.py  --ignore-files test_docker.py
+          python -m 'flake8' superflore --import-order-style=google
+
+
+

CHANGELOG.md

@@ -1,3 +1,24 @@
+# Changelog
+
+## [v0.3.3](https://github.com/ros-infrastructure/superflore/tree/v0.3.3) (2021-11-20)
+
+[Full Changelog](https://github.com/ros-infrastructure/superflore/compare/v0.3.2...v0.3.3)
+
+**Implemented enhancements:**
+
+- Release Version 0.3.2 [\#276](https://github.com/ros-infrastructure/superflore/issues/276)
+
+**Closed issues:**
+
+- License parsing doesn't parse e.g. LGPLv3 correctly [\#271](https://github.com/ros-infrastructure/superflore/issues/271)
+
+**Merged pull requests:**
+
+- Create GitHub Actions CI workflow based on travis job. [\#288](https://github.com/ros-infrastructure/superflore/pull/288) ([nuclearsandwich](https://github.com/nuclearsandwich))
+- Fix parsing of \<exports\>\<build\_type\> with conditionals [\#281](https://github.com/ros-infrastructure/superflore/pull/281) ([shr-project](https://github.com/shr-project))
+- Various improvements to support generating recipes for Noetic [\#280](https://github.com/ros-infrastructure/superflore/pull/280) ([shr-project](https://github.com/shr-project))
+- Improve license parsing [\#279](https://github.com/ros-infrastructure/superflore/pull/279) ([shr-project](https://github.com/shr-project))
+
 # Change Log
 
 ## [v0.3.2](https://github.com/ros-infrastructure/superflore/tree/v0.3.2) (2020-04-30)

README.md

@@ -195,7 +195,8 @@ $ superflore-gen-oe-recipes --ros-distro ROS_DISTRO
 This command will clone the `ros/meta-ros` repo into a subfolder under
 `/tmp/superflore`, generate the recipes and other files for the specified
 distro, commit them, and issue a pull request for `ros/meta-ros`. The
-`--ros-distro` flag must be supplied.
+`--ros-distro` flag must be supplied. ROS 1 distros prior to "melodic" are
+not supported.
 
 Generating bitbake recipes without specifying `--dry-run` is not
 supported. This is because it is almost inevitable that

requirements.txt

@@ -1,4 +1,4 @@
-nose
+pynose
 flake8
 flake8-import-order
 termcolor

setup.py

@@ -61,13 +61,13 @@ def append_local_version_label(public_version):
     'docker',
     'pyyaml',
     'pygithub',
-    'catkin_pkg >= 0.4.0',
+    'catkin_pkg >= 0.4.10',
     'rospkg >= 1.1.8',
 ]
 
 setup(
     name='superflore',
-    version=append_local_version_label('0.3.2'),
+    version=append_local_version_label('0.3.3'),
     packages=find_packages(exclude=['tests', 'tests.*']),
     author='Hunter L. Allen',
     author_email='[email protected]',

superflore/PackageMetadata.py

@@ -12,17 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import re
-
 from catkin_pkg.package import parse_package_string
 
 
 class PackageMetadata:
-    def __init__(self, pkg_xml):
+    def __init__(self, pkg_xml, evaluate_condition_context=None):
         self.upstream_email = None
         self.upstream_name = None
         self.homepage = 'https://wiki.ros.org'
         pkg = parse_package_string(pkg_xml)
+        if evaluate_condition_context:
+            pkg.evaluate_conditions(evaluate_condition_context)
         self.upstream_license = pkg.licenses
         self.description = pkg.description
         if 'website' in [url.type for url in pkg.urls]:
@@ -49,11 +49,4 @@ def __init__(self, pkg_xml):
         self.member_of_groups = [
             group.name for group in pkg.member_of_groups
         ]
-        tag_remover = re.compile('<.*?>')
-        build_type = [
-            re.sub(tag_remover, '', str(e))
-            for e in pkg.exports if 'build_type' in str(e)
-        ]
-        self.build_type = 'catkin'
-        if build_type:
-            self.build_type = build_type[0]
+        self.build_type = pkg.get_build_type()

superflore/generators/bitbake/gen_packages.py

@@ -12,8 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
-
 from catkin_pkg.package import InvalidPackage
 from rosdistro.dependency_walker import DependencyWalker
 from rosdistro.manifest_provider import get_release_tag
@@ -33,26 +31,28 @@
 
 
 def regenerate_pkg(
-    overlay, pkg, distro, preserve_existing, srcrev_cache,
+    overlay, pkg, rosdistro, preserve_existing, srcrev_cache,
     skip_keys
 ):
-    pkg_names = get_package_names(distro)[0]
+    pkg_names = get_package_names(rosdistro)[0]
     if pkg not in pkg_names:
         yoctoRecipe.not_generated_recipes.add(pkg)
-        raise RuntimeError("Unknown package '%s'" % pkg)
+        raise RuntimeError("Unknown package '%s' available packages"
+                           " in selected distro: %s" %
+                           (pkg, get_package_names(rosdistro)))
     try:
-        version = get_pkg_version(distro, pkg, is_oe=True)
+        version = get_pkg_version(rosdistro, pkg, is_oe=True)
     except KeyError as ke:
         yoctoRecipe.not_generated_recipes.add(pkg)
         raise ke
     repo_dir = overlay.repo.repo_dir
     component_name = yoctoRecipe.convert_to_oe_name(
-        distro.release_packages[pkg].repository_name)
+        rosdistro.release_packages[pkg].repository_name)
     recipe = yoctoRecipe.convert_to_oe_name(pkg)
     # check for an existing recipe which was removed by clean_ros_recipe_dirs
     prefix = 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb'.format(
-        yoctoRecipe._get_ros_version(distro.name),
-        distro.name,
+        yoctoRecipe._get_ros_version(rosdistro.name),
+        rosdistro.name,
         recipe
     )
     existing = overlay.repo.git.status('--porcelain', '--', prefix)
@@ -64,16 +64,16 @@ def regenerate_pkg(
             warn('More than 1 recipe was output by "git status --porcelain '
                  'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'
                  .format(
-                     yoctoRecipe._get_ros_version(distro.name),
-                     distro.name,
+                     yoctoRecipe._get_ros_version(rosdistro.name),
+                     rosdistro.name,
                      recipe,
                      existing))
         if existing.split()[0] != 'D':
             err('Unexpected output from "git status --porcelain '
                 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'
                 .format(
-                    yoctoRecipe._get_ros_version(distro.name),
-                    distro.name,
+                    yoctoRecipe._get_ros_version(rosdistro.name),
+                    rosdistro.name,
                     recipe,
                     existing))
 
@@ -89,8 +89,8 @@ def regenerate_pkg(
                     '--porcelain '
                     'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'
                     .format(
-                        yoctoRecipe._get_ros_version(distro.name),
-                        distro.name,
+                        yoctoRecipe._get_ros_version(rosdistro.name),
+                        rosdistro.name,
                         recipe,
                         existing))
             existing = existing[0]
@@ -106,7 +106,7 @@ def regenerate_pkg(
         previous_version = existing[idx_version:].rstrip('.bb')
     try:
         current = oe_recipe(
-            distro, pkg, srcrev_cache, skip_keys
+            rosdistro, pkg, srcrev_cache, skip_keys
         )
     except InvalidPackage as e:
         err('Invalid package: ' + str(e))
@@ -129,8 +129,8 @@ def regenerate_pkg(
     make_dir(
         "{0}/meta-ros{1}-{2}/generated-recipes/{3}".format(
             repo_dir,
-            yoctoRecipe._get_ros_version(distro.name),
-            distro.name,
+            yoctoRecipe._get_ros_version(rosdistro.name),
+            rosdistro.name,
             component_name
         )
     )
@@ -139,8 +139,8 @@ def regenerate_pkg(
     recipe_file_name = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/' \
         '{4}_{5}.bb'.format(
             repo_dir,
-            yoctoRecipe._get_ros_version(distro.name),
-            distro.name,
+            yoctoRecipe._get_ros_version(rosdistro.name),
+            rosdistro.name,
             component_name,
             recipe,
             version
@@ -159,12 +159,14 @@ def regenerate_pkg(
 
 
 def _gen_recipe_for_package(
-    distro, pkg_name, pkg, repo, ros_pkg,
+    rosdistro, pkg_name, pkg, repo, ros_pkg,
     pkg_rosinstall, srcrev_cache, skip_keys
 ):
-    pkg_names = get_package_names(distro)
-    pkg_dep_walker = DependencyWalker(distro,
-                                      evaluate_condition_context=os.environ)
+    pkg_names = get_package_names(rosdistro)
+    pkg_dep_walker = DependencyWalker(
+        rosdistro,
+        evaluate_condition_context=yoctoRecipe._get_condition_context(
+            rosdistro.name))
     pkg_buildtool_deps = pkg_dep_walker.get_depends(pkg_name, "buildtool")
     pkg_build_deps = pkg_dep_walker.get_depends(pkg_name, "build")
     pkg_build_export_deps = pkg_dep_walker.get_depends(
@@ -177,7 +179,7 @@ def _gen_recipe_for_package(
 
     # parse through package xml
     err_msg = 'Failed to fetch metadata for package {}'.format(pkg_name)
-    pkg_xml = retry_on_exception(ros_pkg.get_package_xml, distro.name,
+    pkg_xml = retry_on_exception(ros_pkg.get_package_xml, rosdistro.name,
                                  retry_msg='Could not get package xml!',
                                  error_msg=err_msg)
 
@@ -186,7 +188,7 @@ def _gen_recipe_for_package(
         len(ros_pkg.repository.package_names),
         pkg_name,
         pkg_xml,
-        distro,
+        rosdistro,
         src_uri,
         srcrev_cache,
         skip_keys,
@@ -220,18 +222,18 @@ def _gen_recipe_for_package(
 
 class oe_recipe(object):
     def __init__(
-        self, distro, pkg_name, srcrev_cache, skip_keys
+        self, rosdistro, pkg_name, srcrev_cache, skip_keys
     ):
-        pkg = distro.release_packages[pkg_name]
-        repo = distro.repositories[pkg.repository_name].release_repository
+        pkg = rosdistro.release_packages[pkg_name]
+        repo = rosdistro.repositories[pkg.repository_name].release_repository
         ros_pkg = RosPackage(pkg_name, repo)
 
         pkg_rosinstall = _generate_rosinstall(
             pkg_name, repo.url, get_release_tag(repo, pkg_name), True
         )
 
         self.recipe = _gen_recipe_for_package(
-            distro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall,
+            rosdistro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall,
             srcrev_cache, skip_keys
         )
 

superflore/generators/bitbake/ros_meta.py

@@ -45,7 +45,7 @@ def clean_ros_recipe_dirs(self, distro):
         info(
             'Cleaning up:\n{0}'
             .format(files))
-        self.repo.git.rm('-rf', '--ignore-unmatch', files)
+        self.repo.git.rm('-rf', '--ignore-unmatch', files.split())
 
     def commit_changes(self, distro, commit_msg):
         info('Commit changes...')

superflore/generators/bitbake/run.py

@@ -16,6 +16,7 @@
 import sys
 
 from rosinstall_generator.distro import get_distro
+from rosinstall_generator.distro import get_package_names
 from superflore.CacheManager import CacheManager
 from superflore.generate_installers import generate_installers
 from superflore.generators.bitbake.gen_packages import regenerate_pkg
@@ -39,7 +40,6 @@
 
 
 def main():
-    os.environ["ROS_OS_OVERRIDE"] = "openembedded"
     overlay = None
     parser = get_parser(
         'Generate OpenEmbedded recipes for ROS packages',
@@ -129,12 +129,14 @@ def main():
                             overlay,
                             pkg,
                             distro,
-                            preserve_existing,
+                            False,  # preserve_existing
                             srcrev_cache,
                             skip_keys=skip_keys,
                         )
                     except KeyError:
-                        err("No package to satisfy key '%s'" % pkg)
+                        err("No package to satisfy key '%s' available "
+                            "packages in selected distro: %s" %
+                            (pkg, get_package_names(distro)))
                         sys.exit(1)
                 # Commit changes and file pull request
                 title =\
@@ -146,6 +148,7 @@ def main():
                 regen_dict = dict()
                 regen_dict[args.ros_distro] = args.only
                 delta = "Regenerated: '%s'\n" % args.only
+                overlay.add_generated_files(args.ros_distro)
                 commit_msg = '\n'.join([get_pr_text(
                     title + '\n' + pr_comment.replace(
                         '**superflore**', 'superflore'), markup=''), delta])
@@ -162,6 +165,7 @@ def main():
             for adistro in selected_targets:
                 yoctoRecipe.reset()
                 distro = get_distro(adistro)
+
                 distro_installers, _, distro_changes =\
                     generate_installers(
                         distro,