From 0be23cc07b49978694f36f48c729d4d8780401cd Mon Sep 17 00:00:00 2001 From: Martin Jansa Date: Mon, 3 Feb 2020 11:56:58 -0800 Subject: [PATCH] bitbake: Delete all generated files when regenerating all recipes * clean_ros_recipe_dirs() wasn't called anywhere, update it for what's now generated by superflore and call it when regenerating all of the recipes. * the overlay.repo.remove_file(existing, True) call in bitbake/gen_packages.py works fine, but only when the new recipe is being generated in the same directory as the previous version: existing = glob.glob('{}_*.bb'.format(prefix)) where prefix is: prefix = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/{4}' which will result in multiple recipes for the same package when the component that provides a package is changed like what happened to pcl-conversions in dashing and eloquent. Signed-off-by: Martin Jansa --- superflore/generators/bitbake/gen_packages.py | 52 ++++++++++++++++--- superflore/generators/bitbake/ros_meta.py | 27 ++++++---- superflore/generators/bitbake/run.py | 1 + 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/superflore/generators/bitbake/gen_packages.py b/superflore/generators/bitbake/gen_packages.py index e4a06376..9e96ee33 100644 --- a/superflore/generators/bitbake/gen_packages.py +++ b/superflore/generators/bitbake/gen_packages.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import glob import os from catkin_pkg.package import InvalidPackage @@ -28,6 +27,7 @@ from superflore.utils import make_dir from superflore.utils import ok from superflore.utils import retry_on_exception +from superflore.utils import warn org = "Open Source Robotics Foundation" @@ -49,22 +49,58 @@ def regenerate_pkg( component_name = yoctoRecipe.convert_to_oe_name( distro.release_packages[pkg].repository_name) recipe = yoctoRecipe.convert_to_oe_name(pkg) - # check for an existing recipe - prefix = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/{4}'.format( - repo_dir, + # check for an existing recipe which was removed by clean_ros_recipe_dirs + prefix = 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb'.format( yoctoRecipe._get_ros_version(distro.name), distro.name, - component_name, - recipe, + recipe ) - existing = glob.glob('{}_*.bb'.format(prefix)) + existing = overlay.repo.git.status('--porcelain', '--', prefix) + if existing: + # The git status --porcelain output will look like this: + # D meta-ros2-eloquent/generated-recipes/variants/ros-base_0.8.3-1.bb + # we want just the path with filename + if len(existing.split('\n')) > 1: + warn('More than 1 recipe was output by "git status --porcelain ' + 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"' + .format( + yoctoRecipe._get_ros_version(distro.name), + distro.name, + recipe, + existing)) + if existing.split()[0] != 'D': + err('Unexpected output from "git status --porcelain ' + 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"' + .format( + yoctoRecipe._get_ros_version(distro.name), + distro.name, + recipe, + existing)) + + existing = existing.split()[1] + else: + # If it isn't shown in git status, it could still exist as normal + # unchanged file when --only option is being used + import glob + existing = glob.glob('{0}/{1}'.format(repo_dir, prefix)) + if existing: + if len(existing) > 1: + err('More than 1 recipe was output by "git status ' + '--porcelain ' + 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"' + .format( + yoctoRecipe._get_ros_version(distro.name), + distro.name, + recipe, + existing)) + existing = existing[0] + previous_version = None if preserve_existing and existing: ok("recipe for package '%s' up to date, skipping..." % pkg) yoctoRecipe.not_generated_recipes.add(pkg) return None, [], None elif existing: - existing = existing[0] overlay.repo.remove_file(existing, True) idx_version = existing.rfind('_') + len('_') previous_version = existing[idx_version:].rstrip('.bb') diff --git a/superflore/generators/bitbake/ros_meta.py b/superflore/generators/bitbake/ros_meta.py index e609cc72..9f2db85b 100644 --- a/superflore/generators/bitbake/ros_meta.py +++ b/superflore/generators/bitbake/ros_meta.py @@ -29,16 +29,23 @@ def __init__( info('Creating new branch {0}...'.format(self.branch_name)) self.repo.create_branch(self.branch_name) - def clean_ros_recipe_dirs(self, distro=None): - if distro: - info( - 'Cleaning up meta-ros{}-{}/generated-recipes directory...' - .format(yoctoRecipe._get_ros_version(distro), distro)) - self.repo.git.rm('-rf', 'meta-ros{}-{}/generated-recipes'.format( - yoctoRecipe._get_ros_version(distro), distro)) - else: - info('Cleaning up generated-recipes directories...') - self.repo.git.rm('-rf', 'generated-recipes') + def clean_ros_recipe_dirs(self, distro): + # superflore-change-summary.txt is no longer being generated since: + # https://github.com/ros-infrastructure/superflore/pull/273 + # but remove it here to make sure it gets deleted when new distro + # release is being generated + files = 'meta-ros{0}-{1}/generated-recipes '\ + 'meta-ros{0}-{1}/conf/ros-distro/include/{1}/generated '\ + 'meta-ros{0}-{1}/files/{1}/generated/'\ + 'newer-platform-components.list '\ + 'meta-ros{0}-{1}/files/{1}/generated/rosdep-resolve.yaml '\ + 'meta-ros{0}-{1}/files/{1}/generated/'\ + 'superflore-change-summary.txt '.format( + yoctoRecipe._get_ros_version(distro), distro) + info( + 'Cleaning up:\n{0}' + .format(files)) + self.repo.git.rm('-rf', '--ignore-unmatch', files) def commit_changes(self, distro, commit_msg): info('Commit changes...') diff --git a/superflore/generators/bitbake/run.py b/superflore/generators/bitbake/run.py index af99e656..74153d64 100644 --- a/superflore/generators/bitbake/run.py +++ b/superflore/generators/bitbake/run.py @@ -158,6 +158,7 @@ def main(): ok('Successfully synchronized repositories!') sys.exit(0) + overlay.clean_ros_recipe_dirs(args.ros_distro) for adistro in selected_targets: yoctoRecipe.reset() distro = get_distro(adistro)