diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..ba3b5d94 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,52 @@ +name: superflore-ci + +on: + workflow_dispatch: + push: + branches: ['master'] + pull_request: + +jobs: + build: + strategy: + matrix: + os: ["ubuntu-20.04", "ubuntu-22.04"] + python: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] + name: superflore tests + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{matrix.python}} + uses: actions/setup-python@v5 + with: + python-version: ${{matrix.python}} + - name: Install dependencies + run: | + echo "Set locale" + sudo apt update && sudo apt install locales + sudo locale-gen en_US en_US.UTF-8 + sudo update-locale LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 + export LANG=en_US.UTF-8 + + echo "Enable required repositories" + sudo apt install software-properties-common + sudo add-apt-repository universe + + sudo apt update && sudo apt install curl -y + sudo curl -sSL https://raw.githubusercontent.com/ros/rosdistro/master/ros.key -o /usr/share/keyrings/ros-archive-keyring.gpg + + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/ros-archive-keyring.gpg] http://packages.ros.org/ros2/ubuntu $(. /etc/os-release && echo $UBUNTU_CODENAME) main" | sudo tee /etc/apt/sources.list.d/ros2.list > /dev/null + + sudo apt-get update -qq + sudo apt-get install dpkg -y + sudo apt-get install -y python3-rosdep + pip install -r requirements.txt + - name: Run tests + run: | + sudo rosdep init + rosdep update + python -m 'nose' --verbose --exclude test_pull --exclude test_run --exclude test_logger_output --exclude test_failed_to_create --exclude test_generate_installers --ignore-files test_ebuild.py --ignore-files test_docker.py + python -m 'flake8' superflore --import-order-style=google + + + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e91d8aa2..00000000 --- a/.travis.yml +++ /dev/null @@ -1,25 +0,0 @@ -dist: bionic -sudo: required -language: python -services: docker -python: - - "3.6" - - "3.7" - # PyPy versions - # TODO(allenh1): Re-enable pypy3 when available for ubuntu 18.04 - # - "pypy3" -before_install: - - sudo echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" | sudo tee -a /etc/apt/sources.list.d/ros-latest.list - - sudo apt-key adv --keyserver hkp://ha.pool.sks-keyservers.net:80 --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 - - sudo apt-get update -qq - - sudo apt-get install dpkg -y - - sudo apt-get install -y python-rosdep -# command to install dependencies -install: - - pip install -r requirements.txt -# command to run tests -script: - - sudo rosdep init - - rosdep update - - python -m 'nose' --exclude test_pull --exclude test_run --exclude test_logger_output - - python -m 'flake8' superflore --import-order-style=google diff --git a/CHANGELOG.md b/CHANGELOG.md index 595ebaab..a3abae90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,24 @@ +# Changelog + +## [v0.3.3](https://github.com/ros-infrastructure/superflore/tree/v0.3.3) (2021-11-20) + +[Full Changelog](https://github.com/ros-infrastructure/superflore/compare/v0.3.2...v0.3.3) + +**Implemented enhancements:** + +- Release Version 0.3.2 [\#276](https://github.com/ros-infrastructure/superflore/issues/276) + +**Closed issues:** + +- License parsing doesn't parse e.g. LGPLv3 correctly [\#271](https://github.com/ros-infrastructure/superflore/issues/271) + +**Merged pull requests:** + +- Create GitHub Actions CI workflow based on travis job. [\#288](https://github.com/ros-infrastructure/superflore/pull/288) ([nuclearsandwich](https://github.com/nuclearsandwich)) +- Fix parsing of \\ with conditionals [\#281](https://github.com/ros-infrastructure/superflore/pull/281) ([shr-project](https://github.com/shr-project)) +- Various improvements to support generating recipes for Noetic [\#280](https://github.com/ros-infrastructure/superflore/pull/280) ([shr-project](https://github.com/shr-project)) +- Improve license parsing [\#279](https://github.com/ros-infrastructure/superflore/pull/279) ([shr-project](https://github.com/shr-project)) + # Change Log ## [v0.3.2](https://github.com/ros-infrastructure/superflore/tree/v0.3.2) (2020-04-30) diff --git a/README.md b/README.md index e7af4a1c..bb32844f 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,8 @@ $ superflore-gen-oe-recipes --ros-distro ROS_DISTRO This command will clone the `ros/meta-ros` repo into a subfolder under `/tmp/superflore`, generate the recipes and other files for the specified distro, commit them, and issue a pull request for `ros/meta-ros`. The -`--ros-distro` flag must be supplied. +`--ros-distro` flag must be supplied. ROS 1 distros prior to "melodic" are +not supported. Generating bitbake recipes without specifying `--dry-run` is not supported. This is because it is almost inevitable that diff --git a/requirements.txt b/requirements.txt index a50e25ed..282bb8fb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -nose +pynose flake8 flake8-import-order termcolor diff --git a/setup.py b/setup.py index bdda794e..0dc34dcd 100755 --- a/setup.py +++ b/setup.py @@ -61,13 +61,13 @@ def append_local_version_label(public_version): 'docker', 'pyyaml', 'pygithub', - 'catkin_pkg >= 0.4.0', + 'catkin_pkg >= 0.4.10', 'rospkg >= 1.1.8', ] setup( name='superflore', - version=append_local_version_label('0.3.2'), + version=append_local_version_label('0.3.3'), packages=find_packages(exclude=['tests', 'tests.*']), author='Hunter L. Allen', author_email='hunterlallen@protonmail.com', diff --git a/superflore/PackageMetadata.py b/superflore/PackageMetadata.py index bea2f36d..cb0f5b5d 100644 --- a/superflore/PackageMetadata.py +++ b/superflore/PackageMetadata.py @@ -12,17 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import re - from catkin_pkg.package import parse_package_string class PackageMetadata: - def __init__(self, pkg_xml): + def __init__(self, pkg_xml, evaluate_condition_context=None): self.upstream_email = None self.upstream_name = None self.homepage = 'https://wiki.ros.org' pkg = parse_package_string(pkg_xml) + if evaluate_condition_context: + pkg.evaluate_conditions(evaluate_condition_context) self.upstream_license = pkg.licenses self.description = pkg.description if 'website' in [url.type for url in pkg.urls]: @@ -49,11 +49,4 @@ def __init__(self, pkg_xml): self.member_of_groups = [ group.name for group in pkg.member_of_groups ] - tag_remover = re.compile('<.*?>') - build_type = [ - re.sub(tag_remover, '', str(e)) - for e in pkg.exports if 'build_type' in str(e) - ] - self.build_type = 'catkin' - if build_type: - self.build_type = build_type[0] + self.build_type = pkg.get_build_type() diff --git a/superflore/generators/bitbake/gen_packages.py b/superflore/generators/bitbake/gen_packages.py index 9e96ee33..7c9cd8fb 100644 --- a/superflore/generators/bitbake/gen_packages.py +++ b/superflore/generators/bitbake/gen_packages.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - from catkin_pkg.package import InvalidPackage from rosdistro.dependency_walker import DependencyWalker from rosdistro.manifest_provider import get_release_tag @@ -33,26 +31,28 @@ def regenerate_pkg( - overlay, pkg, distro, preserve_existing, srcrev_cache, + overlay, pkg, rosdistro, preserve_existing, srcrev_cache, skip_keys ): - pkg_names = get_package_names(distro)[0] + pkg_names = get_package_names(rosdistro)[0] if pkg not in pkg_names: yoctoRecipe.not_generated_recipes.add(pkg) - raise RuntimeError("Unknown package '%s'" % pkg) + raise RuntimeError("Unknown package '%s' available packages" + " in selected distro: %s" % + (pkg, get_package_names(rosdistro))) try: - version = get_pkg_version(distro, pkg, is_oe=True) + version = get_pkg_version(rosdistro, pkg, is_oe=True) except KeyError as ke: yoctoRecipe.not_generated_recipes.add(pkg) raise ke repo_dir = overlay.repo.repo_dir component_name = yoctoRecipe.convert_to_oe_name( - distro.release_packages[pkg].repository_name) + rosdistro.release_packages[pkg].repository_name) recipe = yoctoRecipe.convert_to_oe_name(pkg) # check for an existing recipe which was removed by clean_ros_recipe_dirs prefix = 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb'.format( - yoctoRecipe._get_ros_version(distro.name), - distro.name, + yoctoRecipe._get_ros_version(rosdistro.name), + rosdistro.name, recipe ) existing = overlay.repo.git.status('--porcelain', '--', prefix) @@ -64,16 +64,16 @@ def regenerate_pkg( warn('More than 1 recipe was output by "git status --porcelain ' 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"' .format( - yoctoRecipe._get_ros_version(distro.name), - distro.name, + yoctoRecipe._get_ros_version(rosdistro.name), + rosdistro.name, recipe, existing)) if existing.split()[0] != 'D': err('Unexpected output from "git status --porcelain ' 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"' .format( - yoctoRecipe._get_ros_version(distro.name), - distro.name, + yoctoRecipe._get_ros_version(rosdistro.name), + rosdistro.name, recipe, existing)) @@ -89,8 +89,8 @@ def regenerate_pkg( '--porcelain ' 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"' .format( - yoctoRecipe._get_ros_version(distro.name), - distro.name, + yoctoRecipe._get_ros_version(rosdistro.name), + rosdistro.name, recipe, existing)) existing = existing[0] @@ -106,7 +106,7 @@ def regenerate_pkg( previous_version = existing[idx_version:].rstrip('.bb') try: current = oe_recipe( - distro, pkg, srcrev_cache, skip_keys + rosdistro, pkg, srcrev_cache, skip_keys ) except InvalidPackage as e: err('Invalid package: ' + str(e)) @@ -129,8 +129,8 @@ def regenerate_pkg( make_dir( "{0}/meta-ros{1}-{2}/generated-recipes/{3}".format( repo_dir, - yoctoRecipe._get_ros_version(distro.name), - distro.name, + yoctoRecipe._get_ros_version(rosdistro.name), + rosdistro.name, component_name ) ) @@ -139,8 +139,8 @@ def regenerate_pkg( recipe_file_name = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/' \ '{4}_{5}.bb'.format( repo_dir, - yoctoRecipe._get_ros_version(distro.name), - distro.name, + yoctoRecipe._get_ros_version(rosdistro.name), + rosdistro.name, component_name, recipe, version @@ -159,12 +159,14 @@ def regenerate_pkg( def _gen_recipe_for_package( - distro, pkg_name, pkg, repo, ros_pkg, + rosdistro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall, srcrev_cache, skip_keys ): - pkg_names = get_package_names(distro) - pkg_dep_walker = DependencyWalker(distro, - evaluate_condition_context=os.environ) + pkg_names = get_package_names(rosdistro) + pkg_dep_walker = DependencyWalker( + rosdistro, + evaluate_condition_context=yoctoRecipe._get_condition_context( + rosdistro.name)) pkg_buildtool_deps = pkg_dep_walker.get_depends(pkg_name, "buildtool") pkg_build_deps = pkg_dep_walker.get_depends(pkg_name, "build") pkg_build_export_deps = pkg_dep_walker.get_depends( @@ -177,7 +179,7 @@ def _gen_recipe_for_package( # parse through package xml err_msg = 'Failed to fetch metadata for package {}'.format(pkg_name) - pkg_xml = retry_on_exception(ros_pkg.get_package_xml, distro.name, + pkg_xml = retry_on_exception(ros_pkg.get_package_xml, rosdistro.name, retry_msg='Could not get package xml!', error_msg=err_msg) @@ -186,7 +188,7 @@ def _gen_recipe_for_package( len(ros_pkg.repository.package_names), pkg_name, pkg_xml, - distro, + rosdistro, src_uri, srcrev_cache, skip_keys, @@ -220,10 +222,10 @@ def _gen_recipe_for_package( class oe_recipe(object): def __init__( - self, distro, pkg_name, srcrev_cache, skip_keys + self, rosdistro, pkg_name, srcrev_cache, skip_keys ): - pkg = distro.release_packages[pkg_name] - repo = distro.repositories[pkg.repository_name].release_repository + pkg = rosdistro.release_packages[pkg_name] + repo = rosdistro.repositories[pkg.repository_name].release_repository ros_pkg = RosPackage(pkg_name, repo) pkg_rosinstall = _generate_rosinstall( @@ -231,7 +233,7 @@ def __init__( ) self.recipe = _gen_recipe_for_package( - distro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall, + rosdistro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall, srcrev_cache, skip_keys ) diff --git a/superflore/generators/bitbake/ros_meta.py b/superflore/generators/bitbake/ros_meta.py index 9f2db85b..d2f41209 100644 --- a/superflore/generators/bitbake/ros_meta.py +++ b/superflore/generators/bitbake/ros_meta.py @@ -45,7 +45,7 @@ def clean_ros_recipe_dirs(self, distro): info( 'Cleaning up:\n{0}' .format(files)) - self.repo.git.rm('-rf', '--ignore-unmatch', files) + self.repo.git.rm('-rf', '--ignore-unmatch', files.split()) def commit_changes(self, distro, commit_msg): info('Commit changes...') diff --git a/superflore/generators/bitbake/run.py b/superflore/generators/bitbake/run.py index 74153d64..88c9c4cd 100644 --- a/superflore/generators/bitbake/run.py +++ b/superflore/generators/bitbake/run.py @@ -16,6 +16,7 @@ import sys from rosinstall_generator.distro import get_distro +from rosinstall_generator.distro import get_package_names from superflore.CacheManager import CacheManager from superflore.generate_installers import generate_installers from superflore.generators.bitbake.gen_packages import regenerate_pkg @@ -39,7 +40,6 @@ def main(): - os.environ["ROS_OS_OVERRIDE"] = "openembedded" overlay = None parser = get_parser( 'Generate OpenEmbedded recipes for ROS packages', @@ -129,12 +129,14 @@ def main(): overlay, pkg, distro, - preserve_existing, + False, # preserve_existing srcrev_cache, skip_keys=skip_keys, ) except KeyError: - err("No package to satisfy key '%s'" % pkg) + err("No package to satisfy key '%s' available " + "packages in selected distro: %s" % + (pkg, get_package_names(distro))) sys.exit(1) # Commit changes and file pull request title =\ @@ -146,6 +148,7 @@ def main(): regen_dict = dict() regen_dict[args.ros_distro] = args.only delta = "Regenerated: '%s'\n" % args.only + overlay.add_generated_files(args.ros_distro) commit_msg = '\n'.join([get_pr_text( title + '\n' + pr_comment.replace( '**superflore**', 'superflore'), markup=''), delta]) @@ -162,6 +165,7 @@ def main(): for adistro in selected_targets: yoctoRecipe.reset() distro = get_distro(adistro) + distro_installers, _, distro_changes =\ generate_installers( distro, diff --git a/superflore/generators/bitbake/yocto_recipe.py b/superflore/generators/bitbake/yocto_recipe.py index 5697b160..9e413f6a 100644 --- a/superflore/generators/bitbake/yocto_recipe.py +++ b/superflore/generators/bitbake/yocto_recipe.py @@ -25,7 +25,6 @@ from collections import defaultdict import hashlib -import re from subprocess import DEVNULL, PIPE, Popen from superflore.exceptions import NoPkgXml @@ -42,8 +41,8 @@ from superflore.utils import resolve_dep import yaml -UNRESOLVED_PLATFORM_PKG_PREFIX = 'ROS_UNRESOLVED_PLATFORM_PKG_' -UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX = '${'+UNRESOLVED_PLATFORM_PKG_PREFIX +UNRESOLVED_DEP_PREFIX = 'ROS_UNRESOLVED_DEP-' +UNRESOLVED_DEP_REF_PREFIX = '${'+UNRESOLVED_DEP_PREFIX class yoctoRecipe(object): @@ -62,7 +61,7 @@ class yoctoRecipe(object): max_component_name = 0 def __init__( - self, component_name, num_pkgs, pkg_name, pkg_xml, distro, src_uri, + self, component_name, num_pkgs, pkg_name, pkg_xml, rosdistro, src_uri, srcrev_cache, skip_keys ): self.component = component_name @@ -71,13 +70,15 @@ def __init__( self.oe_component = yoctoRecipe.convert_to_oe_name(component_name) self.num_pkgs = num_pkgs self.name = pkg_name - self.distro = distro.name - self.version = get_pkg_version(distro, pkg_name, is_oe=True) + self.distro = rosdistro.name + self.version = get_pkg_version(rosdistro, pkg_name, is_oe=True) self.src_uri = src_uri self.pkg_xml = pkg_xml self.author = None if self.pkg_xml: - pkg_fields = PackageMetadata(pkg_xml) + pkg_fields = PackageMetadata( + pkg_xml, + yoctoRecipe._get_condition_context(rosdistro.name)) maintainer_name = pkg_fields.upstream_name maintainer_email = pkg_fields.upstream_email author_name = pkg_fields.author_name @@ -89,13 +90,22 @@ def __init__( self.license = pkg_fields.upstream_license self.description = pkg_fields.description self.homepage = pkg_fields.homepage - self.build_type = pkg_fields.build_type + pkg_build_type = pkg_fields.build_type + if pkg_build_type == 'catkin' and \ + yoctoRecipe._get_ros_version(rosdistro.name) == 2: + err("Package " + pkg_name + " either doesn't have " + " element at all or it's set to 'catkin'" + " which isn't a valid option for ROS 2; changing it to" + " 'ament_cmake'") + pkg_build_type = 'ament_cmake' + self.build_type = pkg_build_type else: self.description = '' self.license = None self.homepage = None self.build_type = 'catkin' if \ - yoctoRecipe._get_ros_version(distro) == 1 else 'ament_cmake' + yoctoRecipe._get_ros_version(rosdistro.name) == 1 \ + else 'ament_cmake' self.maintainer = "OSRF" self.depends = set() self.depends_external = set() @@ -115,7 +125,6 @@ def __init__( srcrev_cache[self.src_uri] = self.get_srcrev() self.srcrev = srcrev_cache[self.src_uri] self.skip_keys = skip_keys - self.multi_hyphen_re = re.compile('-{2,}') def get_license_line(self): self.license_line = '' @@ -260,21 +269,13 @@ def get_bottom_inherit_line(self): ret = 'inherit ros_${ROS_BUILD_TYPE}\n' return ret - def trim_hyphens(self, s): - return self.multi_hyphen_re.sub('-', s) - - def translate_license(self, l): - conversion_table = {ord(' '): '-', ord('/'): '-', ord(':'): '-', - ord('+'): '-', ord('('): '-', ord(')'): '-'} - return self.trim_hyphens(l.translate(conversion_table)) - @staticmethod def modify_name_if_native(dep, is_native): """ If the name is for an unresolved platform package, move the "-native" inside the "}" so that it's part of the variable name. """ - if dep.startswith(UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX): + if dep.startswith(UNRESOLVED_DEP_REF_PREFIX): return dep[0:-len('}')] + ('-native}' if is_native else '}') else: return dep + ('-native' if is_native else '') @@ -362,13 +363,15 @@ def get_dependencies( yoctoRecipe.rosdep_cache[dep].add(res) info('External dependency add: ' + recipe) except UnresolvedDependency: - unresolved_name = UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX\ - + dep + '}' - recipe = self.convert_to_oe_name(unresolved_name, is_native) + oe_dep = self.convert_to_oe_name(dep, is_native) + recipe = UNRESOLVED_DEP_REF_PREFIX\ + + oe_dep + '}' dependencies.add(recipe) system_dependencies.add(recipe) # Never add -native. - rosdep_name = self.convert_to_oe_name(unresolved_name, False) + rosdep_dep = self.convert_to_oe_name(dep, False) + rosdep_name = UNRESOLVED_DEP_REF_PREFIX\ + + rosdep_dep + '}' yoctoRecipe.rosdep_cache[dep].add(rosdep_name) info('Unresolved external dependency add: ' + recipe) @@ -399,12 +402,17 @@ def get_recipe_text(self, distributor): # license self.get_license_line() if isinstance(self.license, str): - ret += 'LICENSE = "%s"\n' % self.translate_license( - get_license(self.license)) + oe_lic = get_license(self.license) + if oe_lic != self.license: + ret += '# Original license in package.xml:\n' + ret += '# "' + self.license + '"\n' elif isinstance(self.license, list): - ret += 'LICENSE = "' - ret += ' & '.join([self.translate_license( - get_license(l)) for l in self.license]) + '"\n' + oe_lic = ' & '.join([get_license(lic) for lic in self.license]) + if oe_lic != ' & '.join(self.license): + ret += '# Original license in package.xml, joined with ' + ret += '"&" when multiple license tags were used:\n' + ret += '# "' + ' & '.join(self.license) + '"\n' + ret += 'LICENSE = "' + oe_lic + '"\n' ret += 'LIC_FILES_CHKSUM = "file://package.xml;beginline=' ret += str(self.license_line) ret += ';endline=' @@ -480,7 +488,7 @@ def get_recipe_text(self, distributor): ret += ' staged should this package appear in another\'s DEPENDS.\n' ret += 'DEPENDS += "${ROS_EXPORT_DEPENDS} ' ret += '${ROS_BUILDTOOL_EXPORT_DEPENDS}"\n\n' - ret += 'RDEPENDS_${PN} += "${ROS_EXEC_DEPENDS}"' + '\n\n' + ret += 'RDEPENDS:${PN} += "${ROS_EXEC_DEPENDS}"' + '\n\n' # SRC_URI ret += '# matches with: ' + self.src_uri + '\n' ret += 'ROS_BRANCH ?= "branch=' + self.get_repo_branch_name() + '"\n' @@ -499,6 +507,20 @@ def _get_ros_version(distro): return 2 if distro not in distros \ else int(distros[distro]['distribution_type'][len('ros'):]) + @staticmethod + def _get_ros_python_version(distro): + return 2 if distro in ['melodic'] else 3 + + @staticmethod + def _get_condition_context(distro): + context = dict() + context["ROS_OS_OVERRIDE"] = "openembedded" + context["ROS_DISTRO"] = distro + context["ROS_VERSION"] = str(yoctoRecipe._get_ros_version(distro)) + context["ROS_PYTHON_VERSION"] = str( + yoctoRecipe._get_ros_python_version(distro)) + return context + @staticmethod def generate_superflore_datetime_inc(basepath, dist, now): datetime_dir = '{0}/meta-ros{1}-{2}/conf/ros-distro/include/{2}/' \ @@ -561,11 +583,9 @@ def generate_ros_distro_inc( '\nROS_DISTRO_TYPE = "ros{}"\n'.format(ros_version)) conf_file.write('ROS_VERSION = "{}"\n'.format(ros_version)) conf_file.write('# DO NOT OVERRIDE ROS_PYTHON_VERSION\n') - ros_python_version = 3 - if ros_version == 1: - ros_python_version = 2 conf_file.write( - 'ROS_PYTHON_VERSION = "{}"\n\n'.format(ros_python_version)) + 'ROS_PYTHON_VERSION = "{}"\n\n'.format( + yoctoRecipe._get_ros_python_version(distro))) oe_skip_keys = map( lambda skip_key: yoctoRecipe.convert_to_oe_name(skip_key), skip_keys @@ -636,7 +656,7 @@ def generate_ros_distro_inc( 'ROS_SUPERFLORE_GENERATED_BUILDTOOLS_%s' % distro.upper(), yoctoRecipe.generated_native_recipes) + '\n') - conf_file.write('ROS_SUPERFLORE_GENERATED_BUILDTOOLS_append =' + conf_file.write('ROS_SUPERFLORE_GENERATED_BUILDTOOLS:append =' ' " ${ROS_SUPERFLORE_GENERATED_BUILDTOOLS_%s}"' '\n\n' % distro.upper()) conf_file.write(yoctoRecipe.generate_multiline_variable( @@ -660,21 +680,17 @@ def generate_ros_distro_inc( + ' they are added, override\n# the settings in' + ' ros-distro.inc .\n') """ - Drop trailing "}" so that "..._foo-native" sorts after - "..._foo". + Drop UNRESOLVED_DEP_REF_PREFIX and trailing "}" + so that "..._foo-native" sorts after "..._foo". """ - unresolved = [p[0:-1] for p in yoctoRecipe.platform_deps - if p.startswith( - UNRESOLVED_PLATFORM_PKG_REFERENCE_PREFIX)] - for p in sorted(unresolved): - """ - PN is last underscore-separated field. NB the trailing '}' - has already been removed. - """ - pn = p.split('_')[-1] + unresolved = [ + p[len(UNRESOLVED_DEP_REF_PREFIX):-1] + for p in yoctoRecipe.platform_deps if p.startswith( + UNRESOLVED_DEP_REF_PREFIX)] + for dep in sorted(unresolved): conf_file.write( - UNRESOLVED_PLATFORM_PKG_PREFIX + pn + ' = "UNRESOLVED-' - + pn + '"\n') + UNRESOLVED_DEP_PREFIX + dep + ' = "' + + UNRESOLVED_DEP_PREFIX + dep + '"\n') ok('Wrote {0}'.format(conf_path)) except OSError as e: @@ -693,7 +709,8 @@ def generate_rosdep_resolve(basepath, distro): rosdep_resolve_file.write( '# {}/rosdep-resolve.yaml\n'.format(distro)) cache_as_dict_of_list = { - k: list(v) for k, v in yoctoRecipe.rosdep_cache.items()} + k: sorted(list(v)) for k, v in + yoctoRecipe.rosdep_cache.items()} rosdep_resolve_file.write(yaml.dump( cache_as_dict_of_list, default_flow_style=False)) ok('Wrote {0}'.format(rosdep_resolve_path)) diff --git a/superflore/generators/ebuild/ebuild.py b/superflore/generators/ebuild/ebuild.py index ad031314..a1f956d5 100644 --- a/superflore/generators/ebuild/ebuild.py +++ b/superflore/generators/ebuild/ebuild.py @@ -138,12 +138,13 @@ def get_ebuild_text(self, distributor, license_text): # EAPI= ret = self.get_license_line(distributor, license_text) ret += self.get_eapi_line() + if self.python_3 and not self.is_ros2: # enable python 2.7 and python 3.5 ret += self.get_python_compat(['2_7', '3_5', '3_6']) - elif self.python_3: - # only use 3.5, 3.6 for ROS 2 - ret += self.get_python_compat(['3_5', '3_6']) + elif self.python_3 or (self.distro == 'noetic'): + # only use 3.5 - 3.9 for ROS 2 or noetic + ret += self.get_python_compat(['3_5', '3_6', '3_7', '3_8', '3_9']) else: # fallback to python 2.7 ret += self.get_python_compat(['2_7']) @@ -161,13 +162,13 @@ def get_ebuild_text(self, distributor, license_text): # license -- only add if valid if len(self.upstream_license) == 1: self.upstream_license = [ - l.replace(', ', ' ') for l in self.upstream_license + lic.replace(', ', ' ') for lic in self.upstream_license ] split = self.upstream_license[0].split(',') if len(split) > 1: # they did something like "BSD,GPL,blah" ret += 'LICENSE="( ' - ret += ' '.join([get_license(l.strip()) for l in split]) + ret += ' '.join([get_license(lic.strip()) for lic in split]) ret += ' )"\n' else: ret += "LICENSE=\"" diff --git a/superflore/generators/ebuild/gen_packages.py b/superflore/generators/ebuild/gen_packages.py index 915ec519..d002b35b 100644 --- a/superflore/generators/ebuild/gen_packages.py +++ b/superflore/generators/ebuild/gen_packages.py @@ -120,8 +120,34 @@ def regenerate_pkg(overlay, pkg, distro, preserve_existing=False): return current, previous_version, pkg +def _package_condition_context(rosdistro_name): + distro_properties = get_distros()[rosdistro_name] + ros_version = None + if distro_properties['distribution_type'] == 'ros2': + ros_version = '2' + elif distro_properties['distribution_type'] == 'ros1': + ros_version = '1' + else: + err("Superflore does not handle the distribution type '{}'".format( + distro_properties['distribution_type'])) + raise 'Invalid distribution_type for {}'.format(rosdistro_name) + ros_python_version = None + if distro_properties['python_version'] == 3: + ros_python_version = '3' + elif distro_properties['python_version'] == 2: + ros_python_version = '2' + else: + err("Superflore does not handle the python version '{}'".format( + distro_properties['python_version'])) + raise 'Invalid python_version for {}'.format(rosdistro_name) + return { + 'ROS_DISTRO': rosdistro_name, + 'ROS_VERSION': ros_version, + 'ROS_PYTHON_VERSION': ros_python_version} + + def _gen_metadata_for_package( - distro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall + distro, pkg_name, repo, ros_pkg, pkg_rosinstall ): pkg_metadata_xml = metadata_xml() try: @@ -129,7 +155,8 @@ def _gen_metadata_for_package( except Exception: warn("fetch metadata for package {}".format(pkg_name)) return pkg_metadata_xml - pkg = PackageMetadata(pkg_xml) + package_condition_context = _package_condition_context(distro.name) + pkg = PackageMetadata(pkg_xml, evaluate_condition_context=package_condition_context) pkg_metadata_xml.upstream_email = pkg.upstream_email pkg_metadata_xml.upstream_name = pkg.upstream_name pkg_metadata_xml.longdescription = pkg.longdescription @@ -146,7 +173,8 @@ def _gen_ebuild_for_package( pkg_ebuild.distro = distro.name pkg_ebuild.src_uri = pkg_rosinstall[0]['tar']['uri'] pkg_names = get_package_names(distro) - pkg_dep_walker = DependencyWalker(distro) + package_condition_context = _package_condition_context(distro.name) + pkg_dep_walker = DependencyWalker(distro, evaluate_condition_context=package_condition_context) pkg_buildtool_deps = pkg_dep_walker.get_depends(pkg_name, "buildtool") pkg_build_deps = pkg_dep_walker.get_depends(pkg_name, "build") @@ -181,7 +209,7 @@ def _gen_ebuild_for_package( except Exception: warn("fetch metadata for package {}".format(pkg_name)) return pkg_ebuild - pkg = PackageMetadata(pkg_xml) + pkg = PackageMetadata(pkg_xml, evaluate_condition_context=package_condition_context) pkg_ebuild.upstream_license = pkg.upstream_license pkg_ebuild.description = pkg.description pkg_ebuild.homepage = pkg.homepage @@ -201,7 +229,7 @@ def __init__(self, distro, pkg_name, has_patches=False): self.metadata_xml =\ _gen_metadata_for_package(distro, pkg_name, - pkg, repo, ros_pkg, pkg_rosinstall) + repo, ros_pkg, pkg_rosinstall) self.ebuild =\ _gen_ebuild_for_package(distro, pkg_name, pkg, repo, ros_pkg, pkg_rosinstall) diff --git a/superflore/generators/ebuild/overlay_instance.py b/superflore/generators/ebuild/overlay_instance.py index 2ad7e055..382ba6db 100644 --- a/superflore/generators/ebuild/overlay_instance.py +++ b/superflore/generators/ebuild/overlay_instance.py @@ -60,7 +60,11 @@ def commit_changes(self, distro): self.repo.git.commit(m='{0}'.format(commit_msg)) def regenerate_manifests( - self, regen_dict, image_owner='allenh1', image_name='ros_gentoo_base' + self, + regen_dict, + image_owner='allenh1', + image_name='ros_gentoo_base', + split_limit=1000 ): info( "Pulling docker image '%s/%s:latest'..." % ( @@ -76,16 +80,31 @@ def regenerate_manifests( '/root/.gnupg' ) dock.map_directory(self.repo.repo_dir, '/tmp/ros-overlay') - for key in regen_dict.keys(): - for pkg in regen_dict[key]: - pkg_dir = '/tmp/ros-overlay/ros-{0}/{1}'.format(key, pkg) - dock.add_bash_command('cd {0}'.format(pkg_dir)) - dock.add_bash_command('repoman manifest') - try: - dock.run(show_cmd=True) - except docker.errors.ContainerError: - print(dock.log) - raise + for distro in regen_dict.keys(): + chunk_list = [] + chunk_count = 0 + pkg_list = regen_dict[distro] + while len(pkg_list) > 0: + current_chunk = list() + for x in range(split_limit): + if len(pkg_list) > 0: + current_chunk.append(pkg_list.pop()) + else: + break + chunk_list.append(current_chunk) + info("Regeneration list consists of '%d' chunks" % len(chunk_list)) + info("key_lists: '%s'" % chunk_list) + for chunk in chunk_list: + for pkg in chunk: + pkg_dir = '/tmp/ros-overlay/ros-{0}/{1}'.format(distro, pkg) + dock.add_bash_command('cd {0}'.format(pkg_dir)) + dock.add_bash_command('repoman manifest') + try: + dock.run(show_cmd=True) + dock.clear_commands() + except docker.errors.ContainerError: + print(dock.log) + raise def pull_request(self, message, overlay=None, title=''): if not title: diff --git a/superflore/utils.py b/superflore/utils.py index fbae7e78..8fb5cc71 100644 --- a/superflore/utils.py +++ b/superflore/utils.py @@ -145,67 +145,554 @@ def trim_string(string, length=80): return string[:length - len(end_string)] + end_string -def get_license(l): - bsd_re = '^(BSD)((.)*([124]))?' - gpl_re = '((([^L])*(GPL)([^0-9]*))|'\ - '(GNU(.)*GENERAL(.)*PUBLIC(.)*LICENSE([^0-9])*))([0-9])?' - lgpl_re = '(((LGPL)([^0-9]*))|'\ - '(GNU(.)*Lesser(.)*Public(.)*License([^0-9])*))([0-9]?\\.[0-9])?' - apache_re = '^(Apache)((.)*(1\\.0|1\\.1|2\\.0|2))?' - cc_re = '^(Creative(.)?Commons)((.)*)' - cc_nc_nd_re = '^((Creative(.)?Commons)|CC)((.)*)' +\ - '((Non(.)?Commercial)|NC)((.)*)((No(.)?Derivatives)|ND)' - cc_by_nc_sa_re = '^(CC(.)?BY(.)?NC(.)?SA(.)?)' - moz_re = '^(Mozilla)((.)*(1\\.1))?' - boost_re = '^(Boost)((.)*([1]))?' - pub_dom_re = '^(Public(.)?Domain)' - mit_re = '^MIT' - f = re.IGNORECASE - - if re.search(apache_re, l, f): - version = re.search(apache_re, l, f).group(4) - if version: - return 'Apache-%.1f' % (float(version)) - return 'Apache-1.0' - elif re.search(bsd_re, l, f): - version = re.search(bsd_re, l, f).group(4) - if version: - return 'BSD-{0}'.format(version) - return 'BSD' - elif re.search(lgpl_re, l, f): - version = re.search(lgpl_re, l, f) - grp = len(version.groups()) - version = version.group(grp) - if version: - return 'LGPL-{0}'.format(version) - return 'LGPL-2' - elif re.search(gpl_re, l, f): - version = re.search(gpl_re, l, f) - grp = len(version.groups()) - version = version.group(grp) - if version: - return 'GPL-{0}'.format(version) - return 'GPL-1' - elif re.search(moz_re, l, f): - version = re.search(moz_re, l, f).group(4) - if version: - return 'MPL-{0}'.format(version) - return 'MPL-2.0' - elif re.search(mit_re, l, f): - return 'MIT' - elif re.search(cc_nc_nd_re, l, f): - return 'CC-BY-NC-ND-4.0' - elif re.search(cc_by_nc_sa_re, l, f): - return 'CC-BY-NC-SA-4.0' - elif re.search(cc_re, l, f): - return 'CC-BY-SA-3.0' - elif re.search(boost_re, l, f): - return 'Boost-1.0' - elif re.search(pub_dom_re, l, f): - return 'public_domain' - else: - warn('Could not match license "{0}". Passing it through...'.format(l)) - return l +def get_license(lic): + """ + Temporary import the license modification from catkin_pkg validation. + + Once all active ROS packages are updated to have more reasonble license + values, this whole function can be dropped and the value should be the + same as what package.xml says. + """ + def is_valid_spdx_identifier(lic): + """ + Checks if the license is already one of valid SPDX Identifiers from + https://spdx.org/licenses/ + + The list was created with: + cat doc/spdx-3.10-2020-08-03.csv | cut -f 2 | grep -v ^Identifier$ \ + | sed 's/^/"/g; s/$/",/g' | xargs --delimiter='\n' + """ + + return lic in ["0BSD", + "AAL", + "Abstyles", + "Adobe-2006", + "Adobe-Glyph", + "ADSL", + "AFL-1.1", + "AFL-1.2", + "AFL-2.0", + "AFL-2.1", + "AFL-3.0", + "Afmparse", + "AGPL-1.0-only", + "AGPL-1.0-or-later", + "AGPL-3.0-only", + "AGPL-3.0-or-later", + "Aladdin", + "AMDPLPA", + "AML", + "AMPAS", + "ANTLR-PD", + "Apache-1.0", + "Apache-1.1", + "Apache-2.0", + "APAFML", + "APL-1.0", + "APSL-1.0", + "APSL-1.1", + "APSL-1.2", + "APSL-2.0", + "Artistic-1.0", + "Artistic-1.0-cl8", + "Artistic-1.0-Perl", + "Artistic-2.0", + "Bahyph", + "Barr", + "Beerware", + "BitTorrent-1.0", + "BitTorrent-1.1", + "blessing", + "BlueOak-1.0.0", + "Borceux", + "BSD-1-Clause", + "BSD-2-Clause", + "BSD-2-Clause-Patent", + "BSD-2-Clause-Views", + "BSD-3-Clause", + "BSD-3-Clause-Attribution", + "BSD-3-Clause-Clear", + "BSD-3-Clause-LBNL", + "BSD-3-Clause-No-Nuclear-License", + "BSD-3-Clause-No-Nuclear-License-2014", + "BSD-3-Clause-No-Nuclear-Warranty", + "BSD-3-Clause-Open-MPI", + "BSD-4-Clause", + "BSD-4-Clause-UC", + "BSD-Protection", + "BSD-Source-Code", + "BSL-1.0", + "bzip2-1.0.5", + "bzip2-1.0.6", + "CAL-1.0", + "CAL-1.0-Combined-Work-Exception", + "Caldera", + "CATOSL-1.1", + "CC-BY-1.0", + "CC-BY-2.0", + "CC-BY-2.5", + "CC-BY-3.0", + "CC-BY-3.0-AT", + "CC-BY-4.0", + "CC-BY-NC-1.0", + "CC-BY-NC-2.0", + "CC-BY-NC-2.5", + "CC-BY-NC-3.0", + "CC-BY-NC-4.0", + "CC-BY-NC-ND-1.0", + "CC-BY-NC-ND-2.0", + "CC-BY-NC-ND-2.5", + "CC-BY-NC-ND-3.0", + "CC-BY-NC-ND-3.0-IGO", + "CC-BY-NC-ND-4.0", + "CC-BY-NC-SA-1.0", + "CC-BY-NC-SA-2.0", + "CC-BY-NC-SA-2.5", + "CC-BY-NC-SA-3.0", + "CC-BY-NC-SA-4.0", + "CC-BY-ND-1.0", + "CC-BY-ND-2.0", + "CC-BY-ND-2.5", + "CC-BY-ND-3.0", + "CC-BY-ND-4.0", + "CC-BY-SA-1.0", + "CC-BY-SA-2.0", + "CC-BY-SA-2.5", + "CC-BY-SA-3.0", + "CC-BY-SA-3.0-AT", + "CC-BY-SA-4.0", + "CC-PDDC", + "CC0-1.0", + "CDDL-1.0", + "CDDL-1.1", + "CDLA-Permissive-1.0", + "CDLA-Sharing-1.0", + "CECILL-1.0", + "CECILL-1.1", + "CECILL-2.0", + "CECILL-2.1", + "CECILL-B", + "CECILL-C", + "CERN-OHL-1.1", + "CERN-OHL-1.2", + "CERN-OHL-P-2.0", + "CERN-OHL-S-2.0", + "CERN-OHL-W-2.0", + "ClArtistic", + "CNRI-Jython", + "CNRI-Python", + "CNRI-Python-GPL-Compatible", + "Condor-1.1", + "copyleft-next-0.3.0", + "copyleft-next-0.3.1", + "CPAL-1.0", + "CPL-1.0", + "CPOL-1.02", + "Crossword", + "CrystalStacker", + "CUA-OPL-1.0", + "Cube", + "curl", + "D-FSL-1.0", + "diffmark", + "DOC", + "Dotseqn", + "DSDP", + "dvipdfm", + "ECL-1.0", + "ECL-2.0", + "EFL-1.0", + "EFL-2.0", + "eGenix", + "Entessa", + "EPICS", + "EPL-1.0", + "EPL-2.0", + "ErlPL-1.1", + "etalab-2.0", + "EUDatagrid", + "EUPL-1.0", + "EUPL-1.1", + "EUPL-1.2", + "Eurosym", + "Fair", + "Frameworx-1.0", + "FreeImage", + "FSFAP", + "FSFUL", + "FSFULLR", + "FTL", + "GFDL-1.1-invariants-only", + "GFDL-1.1-invariants-or-later", + "GFDL-1.1-no-invariants-only", + "GFDL-1.1-no-invariants-or-later", + "GFDL-1.1-only", + "GFDL-1.1-or-later", + "GFDL-1.2-invariants-only", + "GFDL-1.2-invariants-or-later", + "GFDL-1.2-no-invariants-only", + "GFDL-1.2-no-invariants-or-later", + "GFDL-1.2-only", + "GFDL-1.2-or-later", + "GFDL-1.3-invariants-only", + "GFDL-1.3-invariants-or-later", + "GFDL-1.3-no-invariants-only", + "GFDL-1.3-no-invariants-or-later", + "GFDL-1.3-only", + "GFDL-1.3-or-later", + "Giftware", + "GL2PS", + "Glide", + "Glulxe", + "GLWTPL", + "gnuplot", + "GPL-1.0-only", + "GPL-1.0-or-later", + "GPL-2.0-only", + "GPL-2.0-or-later", + "GPL-3.0-only", + "GPL-3.0-or-later", + "gSOAP-1.3b", + "HaskellReport", + "Hippocratic-2.1", + "HPND", + "HPND-sell-variant", + "IBM-pibs", + "ICU", + "IJG", + "ImageMagick", + "iMatix", + "Imlib2", + "Info-ZIP", + "Intel", + "Intel-ACPI", + "Interbase-1.0", + "IPA", + "IPL-1.0", + "ISC", + "JasPer-2.0", + "JPNIC", + "JSON", + "LAL-1.2", + "LAL-1.3", + "Latex2e", + "Leptonica", + "LGPL-2.0-only", + "LGPL-2.0-or-later", + "LGPL-2.1-only", + "LGPL-2.1-or-later", + "LGPL-3.0-only", + "LGPL-3.0-or-later", + "LGPLLR", + "Libpng", + "libpng-2.0", + "libselinux-1.0", + "libtiff", + "LiLiQ-P-1.1", + "LiLiQ-R-1.1", + "LiLiQ-Rplus-1.1", + "Linux-OpenIB", + "LPL-1.0", + "LPL-1.02", + "LPPL-1.0", + "LPPL-1.1", + "LPPL-1.2", + "LPPL-1.3a", + "LPPL-1.3c", + "MakeIndex", + "MirOS", + "MIT", + "MIT-0", + "MIT-advertising", + "MIT-CMU", + "MIT-enna", + "MIT-feh", + "MITNFA", + "Motosoto", + "mpich2", + "MPL-1.0", + "MPL-1.1", + "MPL-2.0", + "MPL-2.0-no-copyleft-exception", + "MS-PL", + "MS-RL", + "MTLL", + "MulanPSL-1.0", + "MulanPSL-2.0", + "Multics", + "Mup", + "NASA-1.3", + "Naumen", + "NBPL-1.0", + "NCGL-UK-2.0", + "NCSA", + "Net-SNMP", + "NetCDF", + "Newsletr", + "NGPL", + "NIST-PD", + "NIST-PD-fallback", + "NLOD-1.0", + "NLPL", + "Nokia", + "NOSL", + "Noweb", + "NPL-1.0", + "NPL-1.1", + "NPOSL-3.0", + "NRL", + "NTP", + "NTP-0", + "O-UDA-1.0", + "OCCT-PL", + "OCLC-2.0", + "ODbL-1.0", + "ODC-By-1.0", + "OFL-1.0", + "OFL-1.0-no-RFN", + "OFL-1.0-RFN", + "OFL-1.1", + "OFL-1.1-no-RFN", + "OFL-1.1-RFN", + "OGC-1.0", + "OGL-Canada-2.0", + "OGL-UK-1.0", + "OGL-UK-2.0", + "OGL-UK-3.0", + "OGTSL", + "OLDAP-1.1", + "OLDAP-1.2", + "OLDAP-1.3", + "OLDAP-1.4", + "OLDAP-2.0", + "OLDAP-2.0.1", + "OLDAP-2.1", + "OLDAP-2.2", + "OLDAP-2.2.1", + "OLDAP-2.2.2", + "OLDAP-2.3", + "OLDAP-2.4", + "OLDAP-2.5", + "OLDAP-2.6", + "OLDAP-2.7", + "OLDAP-2.8", + "OML", + "OpenSSL", + "OPL-1.0", + "OSET-PL-2.1", + "OSL-1.0", + "OSL-1.1", + "OSL-2.0", + "OSL-2.1", + "OSL-3.0", + "Parity-6.0.0", + "Parity-7.0.0", + "PDDL-1.0", + "PHP-3.0", + "PHP-3.01", + "Plexus", + "PolyForm-Noncommercial-1.0.0", + "PolyForm-Small-Business-1.0.0", + "PostgreSQL", + "PSF-2.0", + "psfrag", + "psutils", + "Python-2.0", + "Qhull", + "QPL-1.0", + "Rdisc", + "RHeCos-1.1", + "RPL-1.1", + "RPL-1.5", + "RPSL-1.0", + "RSA-MD", + "RSCPL", + "Ruby", + "SAX-PD", + "Saxpath", + "SCEA", + "Sendmail", + "Sendmail-8.23", + "SGI-B-1.0", + "SGI-B-1.1", + "SGI-B-2.0", + "SHL-0.5", + "SHL-0.51", + "SimPL-2.0", + "SISSL", + "SISSL-1.2", + "Sleepycat", + "SMLNJ", + "SMPPL", + "SNIA", + "Spencer-86", + "Spencer-94", + "Spencer-99", + "SPL-1.0", + "SSH-OpenSSH", + "SSH-short", + "SSPL-1.0", + "SugarCRM-1.1.3", + "SWL", + "TAPR-OHL-1.0", + "TCL", + "TCP-wrappers", + "TMate", + "TORQUE-1.1", + "TOSL", + "TU-Berlin-1.0", + "TU-Berlin-2.0", + "UCL-1.0", + "Unicode-DFS-2015", + "Unicode-DFS-2016", + "Unicode-TOU", + "Unlicense", + "UPL-1.0", + "Vim", + "VOSTROM", + "VSL-1.0", + "W3C", + "W3C-19980720", + "W3C-20150513", + "Watcom-1.0", + "Wsuipa", + "WTFPL", + "X11", + "Xerox", + "XFree86-1.1", + "xinetd", + "Xnet", + "xpp", + "XSkat", + "YPL-1.0", + "YPL-1.1", + "Zed", + "Zend-2.0", + "Zimbra-1.3", + "Zimbra-1.4", + "Zlib", + "zlib-acknowledgement", + "ZPL-1.1", + "ZPL-2.0", + "ZPL-2.1"] + + def map_license_to_spdx(lic): + """ + Map some commonly used license values to one of valid SPDX Identifiers + from https://spdx.org/licenses/ + + This is mapping only whatever value is listed in package.xml without + any knowledge about the actual license used in the source files - it + can map only the clear unambiguous cases (while triggering an warning) + The rest needs to be fixed in package.xml, so it will trigger an error + + This is similar to what e.g. Openembedded is doing in: + http://git.openembedded.org/openembedded-core/tree/meta/conf/licenses.conf + """ + return { + 'Apache License Version 2.0': 'Apache-2.0', + 'Apachi 2': 'Apache-2.0', + 'Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)': 'Apache-2.0', # noqa: E501 + 'Apache v2': 'Apache-2.0', + 'Apache v2.0': 'Apache-2.0', + 'Apache2.0': 'Apache-2.0', + 'APACHE2.0': 'Apache-2.0', + 'Apache2': 'Apache-2.0', + 'Apache License, Version 2.0': 'Apache-2.0', + 'Apache 2': 'Apache-2.0', + 'Apache 2.0': 'Apache-2.0', + 'Apache License 2.0': 'Apache-2.0', + 'LGPL v2': 'LGPL-2.0-only', + 'LGPL v2.1 or later': 'LGPL-2.1-or-later', + 'LGPL v2.1': 'LGPL-2.1-only', + 'LGPL-2.1': 'LGPL-2.1-only', + 'LGPLv2.1': 'LGPL-2.1-only', + 'GNU Lesser Public License 2.1': 'LGPL-2.1-only', + 'LGPL3': 'LGPL-3.0-only', + 'LGPLv3': 'LGPL-3.0-only', + 'GPL-2.0': 'GPL-2.0-only', + 'GPLv2': 'GPL-2.0-only', + 'GNU General Public License v2.0': 'GPL-2.0-only', + 'GNU GPL v3.0': 'GPL-3.0-only', + 'GPL v3': 'GPL-3.0-only', + 'GPLv3': 'GPL-3.0-only', + 'ECL2.0': 'EPL-2.0', + 'Eclipse Public License 2.0': 'EPL-2.0', + 'Mozilla Public License Version 1.1': 'MPL-1.1', + 'Creative Commons Attribution-NonCommercial-NoDerivatives 4.0 International Public License': 'CC-BY-NC-ND-4.0', # noqa: E501 + 'CreativeCommons-Attribution-NonCommercial-NoDerivatives-4.0': 'CC-BY-NC-ND-4.0', # noqa: E501 + 'CreativeCommons-Attribution-NonCommercial-ShareAlike-4.0-International': 'CC-BY-NC-SA-4.0', # noqa: E501 + 'CC BY-NC-SA 4.0': 'CC-BY-NC-SA-4.0', + 'CreativeCommons-by-nc-4.0': 'CC-BY-NC-4.0', + 'CreativeCommons-by-nc-sa-2.0': 'CC-BY-NC-SA-2.0', + 'Creative Commons BY-NC-ND 3.0': 'CC-BY-NC-ND-3.0', + 'BSD 3-clause Clear License': 'BSD-2-Clause', + 'BSD 3-clause. See license attached': 'BSD-2-Clause', + 'BSD 2-Clause License': 'BSD-2-Clause', + 'BSD2': 'BSD-2-Clause', + 'BSD-3': 'BSD-3-Clause', + 'BSD 3-Clause': 'BSD-3-Clause', + 'Boost Software License 1.0': 'BSL-1.0', + 'Boost': 'BSL-1.0', + 'Boost Software License, Version 1.0': 'BSL-1.0', + 'Boost Software License': 'BSL-1.0', + 'BSL1.0': 'BSL-1.0', + 'MIT License': 'MIT', + 'zlib License': 'Zlib', + 'zlib': 'Zlib' + }.get(lic, None) + + def map_license_to_more_common_format(lic): + """ + These aren't SPDX Identifiers, but lets unify them to use at least + similar format. + """ + return { + "Check author's website": 'Check-authors-website', + 'proprietary': 'Proprietary', + 'Public Domain': 'PD', + 'Public domain': 'PD', + 'TODO': 'TODO-CATKIN-PACKAGE-LICENSE' + }.get(lic, None) + + def map_license_to_ampersand_separated_list(lic): + """ + Check if the license tag contains multiple license values. + + Show warning about using multiple license tags when the + value is one of the listed. + """ + return { + 'LGPLv2.1, modified BSD': 'LGPL-2.1-only & modified BSD', + 'Lesser GPL and Apache License': 'Lesser GPL & Apache License', + 'BSD,GPL because of list.h; other files released under BSD,GPL': 'BSD & GPL because of list.h & other files released under BSD & GPL', # noqa: E501 + 'GPL because of list.h; other files released under BSD': 'GPL because of list.h & other files released under BSD', # noqa: E501 + 'BSD, some icons are licensed under the GNU Lesser General Public License (LGPL) or Creative Commons Attribution-Noncommercial 3.0 License': 'BSD & some icons are licensed under LGPL or CC-BY-NC-3.0', # noqa: E501 + 'BSD,LGPL,LGPL (amcl)': 'BSD & LGPL & LGPL (amcl)', + 'BSD, GPL': 'BSD & GPL', + 'BSD, Apache 2.0': 'BSD & Apache-2.0', + 'BSD, LGPL': 'BSD & LGPL', + 'BSD,LGPL,Apache 2.0': 'BSD & LGPL & Apache-2.0' + }.get(lic, None) + + def translate_license(lic): + conversion_table = {ord(' '): '-', ord('/'): '-', ord(':'): '-', + ord('+'): '-', ord('('): '-', ord(')'): '-'} + multi_hyphen_re = re.compile('-{2,}') + return multi_hyphen_re.sub('-', lic.translate(conversion_table)) + + if is_valid_spdx_identifier(lic): + return lic + common = map_license_to_more_common_format(lic) + if common: + lic = common + multiple = map_license_to_ampersand_separated_list(lic) + if multiple: + lic = multiple + spdx = map_license_to_spdx(lic) + if spdx: + lic = spdx + return translate_license(lic) def resolve_dep(pkg, os, distro=None): diff --git a/tests/test_ebuild.py b/tests/test_ebuild.py index dd092992..36c732cd 100644 --- a/tests/test_ebuild.py +++ b/tests/test_ebuild.py @@ -211,4 +211,4 @@ def test_issue_117(self): got_text = ebuild.get_ebuild_text('Open Source Robotics Foundation', 'BSD') # grab the license line license_line = [line for line in got_text.split('\n') if "LICENSE" in line][0] - self.assertEqual(license_line, 'LICENSE="( BSD LGPL-2 Apache-2.0 )"') + self.assertEqual(license_line, 'LICENSE="( BSD LGPL Apache-2.0 )"') diff --git a/tests/test_utils.py b/tests/test_utils.py index 83460161..74a3c7ab 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -93,39 +93,39 @@ def test_get_license(self): ret = get_license('Apache License, Version 2.0 (http://www.apache.org/licenses/LICENSE-2.0)') self.assertEqual(ret, 'Apache-2.0') ret = get_license('Apache') - self.assertEqual(ret, 'Apache-1.0') + self.assertEqual(ret, 'Apache') ret = get_license('BSD-3') - self.assertEqual(ret, 'BSD') - ret = get_license('Apache-2') + self.assertEqual(ret, 'BSD-3-Clause') + ret = get_license('Apache2') self.assertEqual(ret, 'Apache-2.0') ret = get_license('CreativeCommons-Attribution-NonCommercial-NoDerivatives-4.0') self.assertEqual(ret, 'CC-BY-NC-ND-4.0') ret = get_license('CC BY-NC-SA 4.0') self.assertEqual(ret, 'CC-BY-NC-SA-4.0') - ret = get_license('BoostSoftwareLicense Version1.0') - self.assertEqual(ret, 'Boost-1.0') - ret = get_license('GNU GPLv3') - self.assertEqual(ret, 'GPL-3') + ret = get_license('Boost Software License, Version 1.0') + self.assertEqual(ret, 'BSL-1.0') + ret = get_license('GNU GPL v3.0') + self.assertEqual(ret, 'GPL-3.0-only') ret = get_license('Public Domain') - self.assertEqual(ret, 'public_domain') + self.assertEqual(ret, 'PD') ret = get_license('GPL') - self.assertEqual(ret, 'GPL-1') - ret = get_license('GNU GENERAL PUBLIC LICENSE Version 3') - self.assertEqual(ret, 'GPL-3') + self.assertEqual(ret, 'GPL') + ret = get_license('GNU General Public License v2.0') + self.assertEqual(ret, 'GPL-2.0-only') ret = get_license('GNU Lesser Public License 2.1') - self.assertEqual(ret, 'LGPL-2.1') + self.assertEqual(ret, 'LGPL-2.1-only') ret = get_license('Mozilla Public License Version 1.1') self.assertEqual(ret, 'MPL-1.1') ret = get_license('Mozilla Public License') - self.assertEqual(ret, 'MPL-2.0') + self.assertEqual(ret, 'Mozilla-Public-License') ret = get_license('BSD License 2.0') - self.assertEqual(ret, 'BSD-2') + self.assertEqual(ret, 'BSD-License-2.0') ret = get_license('MIT') self.assertEqual(ret, 'MIT') ret = get_license('Creative Commons') - self.assertEqual(ret, 'CC-BY-SA-3.0') + self.assertEqual(ret, 'Creative-Commons') ret = get_license('United States Government Purpose') - self.assertEqual(ret, 'United States Government Purpose') + self.assertEqual(ret, 'United-States-Government-Purpose') def test_delta_msg(self): """Test the delta message generated for the PR"""