bitbake: Delete all generated files when regenerating all recipes

* clean_ros_recipe_dirs() wasn't called anywhere, update it for what's
  now generated by superflore and call it when regenerating all of the
  recipes.

* the overlay.repo.remove_file(existing, True) call in bitbake/gen_packages.py
  works fine, but only when the new recipe is being generated in the same directory
  as the previous version:
  existing = glob.glob('{}_*.bb'.format(prefix))
  where prefix is:
  prefix = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/{4}'
  which will result in multiple recipes for the same package when the
  component that provides a package is changed like what happened to
  pcl-conversions in dashing and eloquent.

Signed-off-by: Martin Jansa <[email protected]>

superflore/generators/bitbake/gen_packages.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import glob
 import os
 
 from catkin_pkg.package import InvalidPackage
@@ -28,6 +27,7 @@
 from superflore.utils import make_dir
 from superflore.utils import ok
 from superflore.utils import retry_on_exception
+from superflore.utils import warn
 
 org = "Open Source Robotics Foundation"
 
@@ -49,22 +49,58 @@ def regenerate_pkg(
     component_name = yoctoRecipe.convert_to_oe_name(
         distro.release_packages[pkg].repository_name)
     recipe = yoctoRecipe.convert_to_oe_name(pkg)
-    # check for an existing recipe
-    prefix = '{0}/meta-ros{1}-{2}/generated-recipes/{3}/{4}'.format(
-        repo_dir,
+    # check for an existing recipe which was removed by clean_ros_recipe_dirs
+    prefix = 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb'.format(
         yoctoRecipe._get_ros_version(distro.name),
         distro.name,
-        component_name,
-        recipe,
+        recipe
     )
-    existing = glob.glob('{}_*.bb'.format(prefix))
+    existing = overlay.repo.git.status('--porcelain', '--', prefix)
+    if existing:
+        # The git status --porcelain output will look like this:
+        # D  meta-ros2-eloquent/generated-recipes/variants/ros-base_0.8.3-1.bb
+        # we want just the path with filename
+        if len(existing.split('\n')) > 1:
+            warn('More than 1 recipe was output by "git status --porcelain '
+                 'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'
+                 .format(
+                     yoctoRecipe._get_ros_version(distro.name),
+                     distro.name,
+                     recipe,
+                     existing))
+        if existing.split()[0] != 'D':
+            err('Unexpected output from "git status --porcelain '
+                'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'
+                .format(
+                    yoctoRecipe._get_ros_version(distro.name),
+                    distro.name,
+                    recipe,
+                    existing))
+
+        existing = existing.split()[1]
+    else:
+        # If it isn't shown in git status, it could still exist as normal
+        # unchanged file when --only option is being used
+        import glob
+        existing = glob.glob('{0}/{1}'.format(repo_dir, prefix))
+        if existing:
+            if len(existing) > 1:
+                err('More than 1 recipe was output by "git status '
+                    '--porcelain '
+                    'meta-ros{0}-{1}/generated-recipes/*/{2}_*.bb": "{3}"'
+                    .format(
+                        yoctoRecipe._get_ros_version(distro.name),
+                        distro.name,
+                        recipe,
+                        existing))
+            existing = existing[0]
+
     previous_version = None
     if preserve_existing and existing:
         ok("recipe for package '%s' up to date, skipping..." % pkg)
         yoctoRecipe.not_generated_recipes.add(pkg)
         return None, [], None
     elif existing:
-        existing = existing[0]
         overlay.repo.remove_file(existing, True)
         idx_version = existing.rfind('_') + len('_')
         previous_version = existing[idx_version:].rstrip('.bb')

superflore/generators/bitbake/ros_meta.py

@@ -29,16 +29,23 @@ def __init__(
             info('Creating new branch {0}...'.format(self.branch_name))
             self.repo.create_branch(self.branch_name)
 
-    def clean_ros_recipe_dirs(self, distro=None):
-        if distro:
-            info(
-                'Cleaning up meta-ros{}-{}/generated-recipes directory...'
-                .format(yoctoRecipe._get_ros_version(distro), distro))
-            self.repo.git.rm('-rf', 'meta-ros{}-{}/generated-recipes'.format(
-                yoctoRecipe._get_ros_version(distro), distro))
-        else:
-            info('Cleaning up generated-recipes directories...')
-            self.repo.git.rm('-rf', 'generated-recipes')
+    def clean_ros_recipe_dirs(self, distro):
+        # superflore-change-summary.txt is no longer being generated since:
+        # https://github.com/ros-infrastructure/superflore/pull/273
+        # but remove it here to make sure it gets deleted when new distro
+        # release is being generated
+        files = 'meta-ros{0}-{1}/generated-recipes '\
+                'meta-ros{0}-{1}/conf/ros-distro/include/{1}/generated '\
+                'meta-ros{0}-{1}/files/{1}/generated/'\
+                'newer-platform-components.list '\
+                'meta-ros{0}-{1}/files/{1}/generated/rosdep-resolve.yaml '\
+                'meta-ros{0}-{1}/files/{1}/generated/'\
+                'superflore-change-summary.txt '.format(
+                    yoctoRecipe._get_ros_version(distro), distro)
+        info(
+            'Cleaning up:\n{0}'
+            .format(files))
+        self.repo.git.rm('-rf', '--ignore-unmatch', files)
 
     def commit_changes(self, distro, commit_msg):
         info('Commit changes...')

superflore/generators/bitbake/run.py

@@ -158,6 +158,7 @@ def main():
                 ok('Successfully synchronized repositories!')
                 sys.exit(0)
 
+            overlay.clean_ros_recipe_dirs(args.ros_distro)
             for adistro in selected_targets:
                 yoctoRecipe.reset()
                 distro = get_distro(adistro)