From b63931b10374c6c61b540b0b6be0202ed5deda79 Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Sat, 8 Jun 2019 20:41:29 +0800 Subject: [PATCH 01/23] get_package_data return list, reuse it Notice that the `get_package_data` method will return a list. so it's clear that the logic `detect_branches` method in ReleaseGenerator and RosReleaseGenerator is same. In this commit, I reuse this function and put it into ReleaseGenerator's `handle_arguments` method so that we can safely delete the duplicate part in RosReleaseGenerator. --- bloom/generators/release.py | 4 +--- bloom/generators/rosrelease.py | 20 +------------------- 2 files changed, 2 insertions(+), 22 deletions(-) diff --git a/bloom/generators/release.py b/bloom/generators/release.py index 067edfcb..a6741400 100755 --- a/bloom/generators/release.py +++ b/bloom/generators/release.py @@ -97,11 +97,9 @@ def handle_arguments(self, args): self.src = args.src self.name = args.name self.release_inc = args.release_increment + self.branch_list = self.detect_branches() def summarize(self): - self.branch_list = self.detect_branches() - if type(self.branch_list) not in [list, tuple]: - self.exit(self.branch_list if self.branch_list is not None else 1) info("Releasing package" + ('' if len(self.branch_list) == 1 else 's') + ": " + str(self.branch_list)) diff --git a/bloom/generators/rosrelease.py b/bloom/generators/rosrelease.py index f7f09acd..277ca0f9 100644 --- a/bloom/generators/rosrelease.py +++ b/bloom/generators/rosrelease.py @@ -62,22 +62,4 @@ def post_patch(self, destination): name, version, packages = get_package_data(destination) # Execute git tag execute_command('git tag -f ' + destination + '/' + version + - '-' + str(self.release_inc)) - - def detect_branches(self): - self.packages = None - with inbranch(self.src): - if self.name is not None: - self.packages = [self.name] - return [self.name] - package_data = get_package_data(self.src) - if type(package_data) not in [list, tuple]: - return package_data - name, version, packages = package_data - self.packages = packages - # Check meta packages for valid CMakeLists.txt - if isinstance(self.packages, dict): - for path, pkg in self.packages.items(): - # Check for valid CMakeLists.txt if a metapackage - self.metapackage_check(path, pkg) - return name if type(name) is list else [name] + '-' + str(self.release_inc)) \ No newline at end of file From 05363ff7d4396a7eaece3d3ccab77ae63fb39c42 Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Sun, 9 Jun 2019 08:42:54 +0800 Subject: [PATCH 02/23] reuse post_rebase in release generator --- bloom/generators/release.py | 8 ++++---- bloom/generators/rosrelease.py | 16 ---------------- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/bloom/generators/release.py b/bloom/generators/release.py index a6741400..86f5e8d5 100755 --- a/bloom/generators/release.py +++ b/bloom/generators/release.py @@ -133,14 +133,14 @@ def post_patch(self, destination): Cannot automatically tag the release because this is not a catkin project.""") warning("""\ Please checkout the release branch and then create a tag manually with:""") - warning(" git checkout release/" + str(self.name)) - warning(" git tag -f release/" + str(self.name) + "/") + warning(" git checkout " + destination) + warning(" git tag -f " + destination + "/") return with inbranch(destination): name, version, packages = get_package_data(destination) # Execute git tag - release_tag = destination + '/' + version + '-' + self.release_inc - execute_command('git tag ' + release_tag) + execute_command('git tag -f ' + destination + '/' + version + + '-' + str(self.release_inc)) def metapackage_check(self, path, pkg): if pkg.is_metapackage(): diff --git a/bloom/generators/rosrelease.py b/bloom/generators/rosrelease.py index 277ca0f9..71bde0eb 100644 --- a/bloom/generators/rosrelease.py +++ b/bloom/generators/rosrelease.py @@ -47,19 +47,3 @@ def pre_rebase(self, destination): name, self.rosdistro, destination ) ) - - def post_patch(self, destination): - # Figure out the version of the given package - if self.name is not None: - warning("""\ -Cannot automatically tag the release because this is not a catkin project.""") - warning("""\ -Please checkout the release branch and then create a tag manually with:""") - warning(" git checkout " + destination) - warning(" git tag -f " + destination + "/") - return - with inbranch(destination): - name, version, packages = get_package_data(destination) - # Execute git tag - execute_command('git tag -f ' + destination + '/' + version + - '-' + str(self.release_inc)) \ No newline at end of file From 41037cbb9bce4b2bf868b10067876044f708b482 Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Sun, 9 Jun 2019 09:22:10 +0800 Subject: [PATCH 03/23] change resolver into class's staticmethod We need to put the `missing_dep_resolver` in class as staticmethod in case successor of Generator want to reuse this function. And I also add os_name, os_versino, ros_distro if oneday the resolver need more dependency information. --- bloom/generators/common.py | 2 +- bloom/generators/debian/generator.py | 20 +++++++++++++------- bloom/generators/rosdebian.py | 22 +++++++--------------- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/bloom/generators/common.py b/bloom/generators/common.py index c2bfccaa..80d559cd 100644 --- a/bloom/generators/common.py +++ b/bloom/generators/common.py @@ -203,7 +203,7 @@ def resolve_dependencies( # Do not compare the installer key here since this is a general purpose function # They installer is verified in the OS specific generator, when the keys are pre-checked. if resolved_key is None: - resolved_key = fallback_resolver(key, peer_packages) + resolved_key = fallback_resolver(key, peer_packages, os_name, os_version, ros_distro) resolved_keys[key] = resolved_key return resolved_keys diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index 0f9e24f9..bc5a295d 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -274,12 +274,6 @@ def get_changelogs(package, releaser_history=None): return [] -def missing_dep_resolver(key, peer_packages): - if key in peer_packages: - return [sanitize_package_name(key)] - return default_fallback_resolver(key, peer_packages) - - def generate_substitutions_from_package( package, os_name, @@ -876,6 +870,18 @@ def set_releaser_history(self, history): if has_changes(): execute_command('git commit -m "Store releaser history"') + @staticmethod + def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): + """ + This should be a staticmethod since we will use it when we call + `generate_substitutions_from_package` in `get_subs` + Notice that os_name, os_version, ros_distro maybe useful when we + want to add new resolver in the future + """ + if key in peer_packages: + return [sanitize_package_name(key)] + return default_fallback_resolver(key, peer_packages) + def get_subs(self, package, debian_distro, releaser_history=None): return generate_substitutions_from_package( package, @@ -886,7 +892,7 @@ def get_subs(self, package, debian_distro, releaser_history=None): self.debian_inc, [p.name for p in self.packages.values()], releaser_history=releaser_history, - fallback_resolver=missing_dep_resolver + fallback_resolver=self.missing_dep_resolver ) def generate_debian(self, package, debian_distro): diff --git a/bloom/generators/rosdebian.py b/bloom/generators/rosdebian.py index 12e217c4..0238797f 100644 --- a/bloom/generators/rosdebian.py +++ b/bloom/generators/rosdebian.py @@ -71,22 +71,14 @@ def summarize(self): info("Releasing for rosdistro: " + self.rosdistro) return ret + @staticmethod + def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): + if key in peer_packages: + return [sanitize_package_name(rosify_package_name(key, ros_distro))] + return default_fallback_resolver(key, peer_packages) + def get_subs(self, package, debian_distro, releaser_history, native=False): - def fallback_resolver(key, peer_packages, rosdistro=self.rosdistro): - if key in peer_packages: - return [sanitize_package_name(rosify_package_name(key, rosdistro))] - return default_fallback_resolver(key, peer_packages) - subs = generate_substitutions_from_package( - package, - self.os_name, - debian_distro, - self.rosdistro, - self.install_prefix, - self.debian_inc, - [p.name for p in self.packages.values()], - releaser_history=releaser_history, - fallback_resolver=fallback_resolver - ) + subs = DebianGenerator.get_subs(self, package, debian_distro, releaser_history) subs['Package'] = rosify_package_name(subs['Package'], self.rosdistro) # ROS 2 specific bloom extensions. From 0495da6ae6b3da46dfafd6904a9aad23badf1c74 Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Sun, 9 Jun 2019 21:28:55 +0800 Subject: [PATCH 04/23] use common Inc and package system --- bloom/generators/debian/generate_cmd.py | 11 +- bloom/generators/debian/generator.py | 172 +++++++++--------- .../debian/templates/ament_cmake/changelog.em | 2 +- .../templates/ament_python/changelog.em | 2 +- .../debian/templates/catkin/changelog.em | 2 +- .../debian/templates/cmake/changelog.em | 2 +- bloom/generators/rosdebian.py | 10 +- 7 files changed, 105 insertions(+), 96 deletions(-) diff --git a/bloom/generators/debian/generate_cmd.py b/bloom/generators/debian/generate_cmd.py index dba1a8f1..a0f450f9 100644 --- a/bloom/generators/debian/generate_cmd.py +++ b/bloom/generators/debian/generate_cmd.py @@ -45,6 +45,7 @@ from bloom.generators.debian.generator import generate_substitutions_from_package from bloom.generators.debian.generator import place_template_files from bloom.generators.debian.generator import process_template_files +from bloom.generators.debian.generator import DebianGenerator from bloom.util import get_distro_list_prompt @@ -120,20 +121,22 @@ def main(args=None, get_subs_fn=None): fmt("Generating debs for @{cf}%s:%s@| for package(s) %s" % (os_name, os_version, [p.name for p in pkgs_dict.values()]))) + # TODO: seems troublesome here + package_system = DebianGenerator.package_system for path, pkg in pkgs_dict.items(): template_files = None try: subs = get_subs_fn(pkg, os_name, os_version, ros_distro, args.native) if _place_template_files: # Place template files - place_template_files(path, pkg.get_build_type()) + place_template_files(path, pkg.get_build_type(), package_system) if _process_template_files: # Just process existing template files - template_files = process_template_files(path, subs) + template_files = process_template_files(path, subs, package_system) if not _place_template_files and not _process_template_files: # If neither, do both - place_template_files(path, pkg.get_build_type()) - template_files = process_template_files(path, subs) + place_template_files(path, pkg.get_build_type(), package_system) + template_files = process_template_files(path, subs, package_system) if template_files is not None: for template_file in template_files: os.remove(os.path.normpath(template_file)) diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index bc5a295d..e7740642 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -160,16 +160,15 @@ def __place_template_folder(group, src, dst, gbp=False): shutil.copystat(template_abs_path, template_dst) -def place_template_files(path, build_type, gbp=False): - info(fmt("@!@{bf}==>@| Placing templates files in the 'debian' folder.")) - debian_path = os.path.join(path, 'debian') - # Create/Clean the debian folder - if not os.path.exists(debian_path): - os.makedirs(debian_path) +def place_template_files(path, build_type, package_system, gbp=False): + info(fmt("@!@{bf}==>@| Placing templates files in the '" + package_system + "' folder.")) + dir_path = os.path.join(path, package_system) + if not os.path.exists(dir_path): + os.makedirs(dir_path) # Place template files - group = 'bloom.generators.debian' + group = 'bloom.generators.' + package_system templates = os.path.join('templates', build_type) - __place_template_folder(group, templates, debian_path, gbp) + __place_template_folder(group, templates, dir_path, gbp) def summarize_dependency_mapping(data, deps, build_deps, resolved_deps): @@ -280,7 +279,7 @@ def generate_substitutions_from_package( os_version, ros_distro, installation_prefix='/usr', - deb_inc=0, + inc=0, peer_packages=None, releaser_history=None, fallback_resolver=None, @@ -298,8 +297,8 @@ def generate_substitutions_from_package( if homepage == '': warning("No homepage set, defaulting to ''") data['Homepage'] = homepage - # Debian Increment Number - data['DebianInc'] = '' if native else '-{0}'.format(deb_inc) + # Increment Number + data['Inc'] = '' if native else '-{0}'.format(inc) # Debian Package Format data['format'] = 'native' if native else 'quilt' # Package name @@ -503,13 +502,13 @@ def __process_template_folder(path, subs): return processed_items -def process_template_files(path, subs): - info(fmt("@!@{bf}==>@| In place processing templates in 'debian' folder.")) - debian_dir = os.path.join(path, 'debian') - if not os.path.exists(debian_dir): - sys.exit("No debian directory found at '{0}', cannot process templates." - .format(debian_dir)) - return __process_template_folder(debian_dir, subs) +def process_template_files(path, subs, package_system): + info(fmt("@!@{bf}==>@| In place processing templates files in '" + package_system + "' folder.")) + dir_path = os.path.join(path, package_system) + if not os.path.exists(dir_path): + sys.exit("No {0} directory found at '{1}', cannot process templates." + .format(self.package_system, dir_path)) + return __process_template_folder(dir_path, subs) def match_branches_with_prefix(prefix, get_branches, prune=False): @@ -568,6 +567,7 @@ def sanitize_package_name(name): class DebianGenerator(BloomGenerator): title = 'debian' + package_system = 'debian' description = "Generates debians from the catkin meta data" has_run_rosdep = False default_install_prefix = '/usr' @@ -576,15 +576,15 @@ class DebianGenerator(BloomGenerator): def prepare_arguments(self, parser): # Add command line arguments for this generator add = parser.add_argument - add('-i', '--debian-inc', help="debian increment number", default='0') + add('-i', '--inc', help="increment number", default='0') add('-p', '--prefix', required=True, - help="branch prefix to match, and from which create debians" + help="branch prefix to match, and from which create packages" " hint: if you want to match 'release/foo' use 'release'") add('-a', '--match-all', default=False, action="store_true", help="match all branches with the given prefix, " "even if not in current upstream") add('--distros', nargs='+', required=False, default=[], - help='A list of debian (ubuntu) distros to generate for') + help='A list of os distros to generate for certain package system') add('--install-prefix', default=None, help="overrides the default installation prefix (/usr)") add('--os-name', default='ubuntu', @@ -595,7 +595,7 @@ def prepare_arguments(self, parser): def handle_arguments(self, args): self.interactive = args.interactive - self.debian_inc = args.debian_inc + self.inc = args.inc self.os_name = args.os_name self.distros = args.distros if self.distros in [None, []]: @@ -624,7 +624,7 @@ def handle_arguments(self, args): self.tag_names = {} self.names = [] self.branch_args = [] - self.debian_branches = [] + self.package_system_branches = [] for branch in self.branches: package = get_package_from_branch(branch) if package is None: @@ -633,14 +633,14 @@ def handle_arguments(self, args): self.packages[package.name] = package self.names.append(package.name) args = self.generate_branching_arguments(package, branch) - # First branch is debian/[/] - self.debian_branches.append(args[0][0]) + # First branch is package_system/[/] + self.package_system_branches.append(args[0][0]) self.branch_args.extend(args) def summarize(self): - info("Generating source debs for the packages: " + str(self.names)) - info("Debian Incremental Version: " + str(self.debian_inc)) - info("Debian Distributions: " + str(self.distros)) + info("Generating {0} source for the packages: {1}".format(self.os_name, str(self.names))) + info("Incremental Version: " + str(self.inc)) + info("Distributions: " + str(self.distros)) def get_branching_arguments(self): return self.branch_args @@ -688,10 +688,10 @@ def _check_all_keys_are_valid(self, peer_packages, ros_distro): error("Key '{0}' resolved to '{1}' with installer '{2}', " "which does not match the default installer '{3}'." .format(key, rule, installer_key, default_installer_key)) - BloomGenerator.exit( - "The Debian generator does not support dependencies " - "which are installed with the '{0}' installer." - .format(installer_key), + self.exit( + "The {0} generator does not support dependencies " + "which are installed with the '{1}' installer." + .format(self.package_system, installer_key), returncode=code.GENERATOR_INVALID_INSTALLER_KEY) except (GeneratorError, RuntimeError) as e: print(fmt("Failed to resolve @{cf}@!{key}@| on @{bf}{os_name}@|:@{cf}@!{os_version}@| with: {e}") @@ -703,7 +703,7 @@ def _check_all_keys_are_valid(self, peer_packages, ros_distro): return all_keys_valid def pre_modify(self): - info("\nPre-verifying Debian dependency keys...") + info("\nPre-verifying {0} dependency keys...".format(self.package_system)) # Run rosdep update is needed if not self.has_run_rosdep: self.update_rosdep() @@ -725,7 +725,7 @@ def pre_modify(self): info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") def pre_branch(self, destination, source): - if destination in self.debian_branches: + if destination in self.package_system_branches: return # Run rosdep update is needed if not self.has_run_rosdep: @@ -751,20 +751,20 @@ def post_rebase(self, destination): name = destination.split('/')[-1] # Retrieve the package package = self.packages[name] - # Handle differently if this is a debian vs distro branch - if destination in self.debian_branches: - info("Placing debian template files into '{0}' branch." - .format(destination)) - # Then this is a debian branch + # Handle differently if this is a package system vs distro branch + if destination in self.package_system_branches: + info("Placing {0} template files into '{1}' branch." + .format(self.package_system, destination)) + # Then this is a package system branch # Place the raw template files self.place_template_files(package.get_build_type()) else: - # This is a distro specific debian branch + # This is a distro specific package system branch # Determine the current package being generated distro = destination.split('/')[-2] - # Create debians for each distro + # Create package for each distro with inbranch(destination): - data = self.generate_debian(package, distro) + data = self.generate_package(package, distro) # Create the tag name for later self.tag_names[destination] = self.generate_tag_name(data) # Update the patch configs @@ -781,7 +781,7 @@ def post_rebase(self, destination): set_patch_config(patches_branch, config) def post_patch(self, destination, color='bluef'): - if destination in self.debian_branches: + if destination in self.package_system_branches: return # Tag after patches have been applied with inbranch(destination): @@ -806,10 +806,10 @@ def post_patch(self, destination, color='bluef'): info( ansi(color) + "#### " + ansi('greenf') + "Successfully" + ansi(color) + " generated '" + ansi('boldon') + distro + - ansi('boldoff') + "' debian for package" + ansi('boldoff') + "' {0} for package".format(self.package_system) + " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + " at version '" + ansi('boldon') + package.version + - "-" + str(self.debian_inc) + ansi('boldoff') + "'" + + "-" + str(self.inc) + ansi('boldoff') + "'" + ansi('reset'), use_prefix=False ) @@ -817,41 +817,44 @@ def post_patch(self, destination, color='bluef'): def store_original_config(self, config, patches_branch): with inbranch(patches_branch): - with open('debian.store', 'w+') as f: + with open('{0}.store'.format(self.package_system), 'w+') as f: f.write(json.dumps(config)) - execute_command('git add debian.store') + execute_command('git add {0}.store'.format(self.package_system)) if has_changes(): execute_command('git commit -m "Store original patch config"') def load_original_config(self, patches_branch): - config_store = show(patches_branch, 'debian.store') + config_store = show(patches_branch, '{0}.store'.format(self.package_system)) if config_store is None: return config_store return json.loads(config_store) - def place_template_files(self, build_type, debian_dir='debian'): - # Create/Clean the debian folder - if os.path.exists(debian_dir): + def place_template_files(self, build_type, dir_path=None): + # Create/Clean the package system folder + if dir_path is None: + dir_path = os.path.join(".", self.package_system) + if os.path.exists(dir_path): if self.interactive: - warning("debian directory exists: " + debian_dir) + warning("{0} directory exists: {1}".format(self.package_system, dir_path)) warning("Do you wish to overwrite it?") if not maybe_continue('y'): error("Answered no to continue, aborting.", exit=True) elif 'BLOOM_CLEAR_DEBIAN_ON_GENERATION' in os.environ: - warning("Overwriting debian directory: " + debian_dir) - execute_command('git rm -rf ' + debian_dir) - execute_command('git commit -m "Clearing previous debian folder"') - if os.path.exists(debian_dir): - shutil.rmtree(debian_dir) + warning("Overwriting {0} directory: {1}".format(self.package_system, dir_path)) + execute_command('git rm -rf ' + dir_path) + execute_command('git commit -m "Clearing previous {0} folder"' + .format(self.package_system)) + if os.path.exists(dir_path): + shutil.rmtree(dir_path) else: - warning("Not overwriting debian directory.") + warning("Not overwriting {0} directory.".format(self.package_system)) # Use generic place template files command - place_template_files('.', build_type, gbp=True) + place_template_files('.', build_type, self.package_system, gbp=True) # Commit results - execute_command('git add ' + debian_dir) + execute_command('git add ' + dir_path) _, has_files, _ = execute_command('git diff --cached --name-only', return_io=True) if has_files: - execute_command('git commit -m "Placing debian template files"') + execute_command('git commit -m "Placing {0} template files"'.format(self.package_system)) def get_releaser_history(self): # Assumes that this is called in the target branch @@ -882,60 +885,63 @@ def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): return [sanitize_package_name(key)] return default_fallback_resolver(key, peer_packages) - def get_subs(self, package, debian_distro, releaser_history=None): + def get_subs(self, package, os_distro, releaser_history=None): + # This is the common part, then the certain generator will add its specic content return generate_substitutions_from_package( package, self.os_name, - debian_distro, + os_distro, self.rosdistro, self.install_prefix, - self.debian_inc, + self.inc, [p.name for p in self.packages.values()], releaser_history=releaser_history, fallback_resolver=self.missing_dep_resolver ) - def generate_debian(self, package, debian_distro): - info("Generating debian for {0}...".format(debian_distro)) + # There we need to add post subs action and different commit action + def generate_package(self, package, os_distro): + info("Generating {0} for {1}...".format(self.package_system, os_distro)) # Try to retrieve the releaser_history releaser_history = self.get_releaser_history() # Generate substitution values - subs = self.get_subs(package, debian_distro, releaser_history) + subs = self.get_subs(package, os_distro, releaser_history) # Use subs to create and store releaser history releaser_history = [(v, (n, e)) for v, _, _, n, e in subs['changelogs']] self.set_releaser_history(dict(releaser_history)) # Handle gbp.conf subs['release_tag'] = self.get_release_tag(subs) # Template files - template_files = process_template_files('.', subs) + template_files = process_template_files(".", subs, self.package_system) # Remove any residual template files execute_command('git rm -rf ' + ' '.join("'{}'".format(t) for t in template_files)) - # Add changes to the debian folder - execute_command('git add debian') + # Add changes to the package system folder + execute_command('git add {0}'.format(self.package_system)) # Commit changes - execute_command('git commit -m "Generated debian files for ' + - debian_distro + '"') + execute_command('git commit -m "Generated {0} files for {1}"' + .format(self.package_system, os_distro)) # Return the subs for other use return subs def get_release_tag(self, data): return 'release/{0}/{1}-{2}'.format(data['Name'], data['Version'], - self.debian_inc) + self.inc) def generate_tag_name(self, data): - tag_name = '{Package}_{Version}{DebianInc}_{Distribution}' - tag_name = 'debian/' + tag_name.format(**data) + tag_name = '{Package}_{Version}{Inc}_{Distribution}' + tag_name = self.package_system + '/' + tag_name.format(**data) return tag_name def generate_branching_arguments(self, package, branch): n = package.name - # Debian branch - deb_branch = 'debian/' + n - # Branch first to the debian branch - args = [[deb_branch, branch, False]] - # Then for each debian distro, branch from the base debian branch + # package branch + package_branch = self.package_system + '/' + n + # Branch first to the package branch + args = [[package_branch, branch, False]] + # Then for each os distro, branch from the base package branch args.extend([ - ['debian/' + d + '/' + n, deb_branch, False] for d in self.distros + [self.package_system + '/' + d + '/' + n, package_branch, False] + for d in self.distros ]) return args @@ -943,10 +949,10 @@ def summarize_package(self, package, distro, color='bluef'): info(ansi(color) + "\n####" + ansi('reset'), use_prefix=False) info( ansi(color) + "#### Generating '" + ansi('boldon') + distro + - ansi('boldoff') + "' debian for package" + ansi('boldoff') + "' {0} for package".format(self.package_system) + " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + " at version '" + ansi('boldon') + package.version + - "-" + str(self.debian_inc) + ansi('boldoff') + "'" + + "-" + str(self.inc) + ansi('boldoff') + "'" + ansi('reset'), use_prefix=False ) diff --git a/bloom/generators/debian/templates/ament_cmake/changelog.em b/bloom/generators/debian/templates/ament_cmake/changelog.em index 35859090..9c4ed62c 100644 --- a/bloom/generators/debian/templates/ament_cmake/changelog.em +++ b/bloom/generators/debian/templates/ament_cmake/changelog.em @@ -1,4 +1,4 @@ -@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(DebianInc)@(Distribution)) @(Distribution); urgency=high +@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(Inc)@(Distribution)) @(Distribution); urgency=high @(changelog) diff --git a/bloom/generators/debian/templates/ament_python/changelog.em b/bloom/generators/debian/templates/ament_python/changelog.em index 35859090..9c4ed62c 100644 --- a/bloom/generators/debian/templates/ament_python/changelog.em +++ b/bloom/generators/debian/templates/ament_python/changelog.em @@ -1,4 +1,4 @@ -@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(DebianInc)@(Distribution)) @(Distribution); urgency=high +@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(Inc)@(Distribution)) @(Distribution); urgency=high @(changelog) diff --git a/bloom/generators/debian/templates/catkin/changelog.em b/bloom/generators/debian/templates/catkin/changelog.em index 35859090..9c4ed62c 100644 --- a/bloom/generators/debian/templates/catkin/changelog.em +++ b/bloom/generators/debian/templates/catkin/changelog.em @@ -1,4 +1,4 @@ -@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(DebianInc)@(Distribution)) @(Distribution); urgency=high +@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(Inc)@(Distribution)) @(Distribution); urgency=high @(changelog) diff --git a/bloom/generators/debian/templates/cmake/changelog.em b/bloom/generators/debian/templates/cmake/changelog.em index 35859090..9c4ed62c 100644 --- a/bloom/generators/debian/templates/cmake/changelog.em +++ b/bloom/generators/debian/templates/cmake/changelog.em @@ -1,4 +1,4 @@ -@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(DebianInc)@(Distribution)) @(Distribution); urgency=high +@[for change_version, change_date, changelog, main_name, main_email in changelogs]@(Package) (@(change_version)@(Inc)@(Distribution)) @(Distribution); urgency=high @(changelog) diff --git a/bloom/generators/rosdebian.py b/bloom/generators/rosdebian.py index 0238797f..fa4f9df6 100644 --- a/bloom/generators/rosdebian.py +++ b/bloom/generators/rosdebian.py @@ -114,17 +114,17 @@ def get_subs(self, package, debian_distro, releaser_history, native=False): return subs def generate_branching_arguments(self, package, branch): - deb_branch = 'debian/' + self.rosdistro + '/' + package.name - args = [[deb_branch, branch, False]] - n, r, b, ds = package.name, self.rosdistro, deb_branch, self.distros + package_branch = self.package_system + '/' + self.rosdistro + '/' + package.name + args = [[package_branch, branch, False]] + n, r, b, ds = package.name, self.rosdistro, package_branch, self.distros args.extend([ - ['debian/' + r + '/' + d + '/' + n, b, False] for d in ds + [self.package_system + '/' + r + '/' + d + '/' + n, b, False] for d in ds ]) return args def get_release_tag(self, data): return 'release/{0}/{1}/{2}-{3}'\ - .format(self.rosdistro, data['Name'], data['Version'], self.debian_inc) + .format(self.rosdistro, data['Name'], data['Version'], self.inc) def rosify_package_name(name, rosdistro): From fba38ed8d101842b8ea1254377307693d32ef67c Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Sun, 9 Jun 2019 21:41:03 +0800 Subject: [PATCH 05/23] put class relate method to where it belongs --- bloom/generators/debian/generator.py | 72 +++++++++++++++------------- 1 file changed, 38 insertions(+), 34 deletions(-) diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index e7740642..edc42f1d 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -533,26 +533,6 @@ def match_branches_with_prefix(prefix, get_branches, prune=False): return branches -def get_package_from_branch(branch): - with inbranch(branch): - try: - package_data = get_package_data(branch) - except SystemExit: - return None - if type(package_data) not in [list, tuple]: - # It is a ret code - DebianGenerator.exit(package_data) - names, version, packages = package_data - if type(names) is list and len(names) > 1: - DebianGenerator.exit( - "Debian generator does not support generating " - "from branches with multiple packages in them, use " - "the release generator first to split packages into " - "individual branches.") - if type(packages) is dict: - return list(packages.values())[0] - - def debianize_string(value): markup_remover = re.compile(r'<.*?>') value = markup_remover.sub('', value) @@ -593,23 +573,27 @@ def prepare_arguments(self, parser): help="Do not error if this os is not in the platforms " "list for rosdistro") + def get_default_distros(self): + index = rosdistro.get_index(rosdistro.get_index_url()) + distribution_file = rosdistro.get_distribution_file(index, self.rosdistro) + if self.os_name not in distribution_file.release_platforms: + if self.os_not_required: + warning("No platforms defined for os '{0}' in release file for the " + "'{1}' distro. This os was not required; continuing without error." + .format(self.os_name, self.rosdistro)) + sys.exit(0) + error("No platforms defined for os '{0}' in release file for the '{1}' distro." + .format(self.os_name, self.rosdistro), exit=True) + self.distros = distribution_file.release_platforms[self.os_name] + def handle_arguments(self, args): + self.os_not_required = args.os_not_required self.interactive = args.interactive self.inc = args.inc self.os_name = args.os_name self.distros = args.distros if self.distros in [None, []]: - index = rosdistro.get_index(rosdistro.get_index_url()) - distribution_file = rosdistro.get_distribution_file(index, self.rosdistro) - if self.os_name not in distribution_file.release_platforms: - if args.os_not_required: - warning("No platforms defined for os '{0}' in release file for the " - "'{1}' distro. This os was not required; continuing without error." - .format(self.os_name, self.rosdistro)) - sys.exit(0) - error("No platforms defined for os '{0}' in release file for the '{1}' distro." - .format(self.os_name, self.rosdistro), exit=True) - self.distros = distribution_file.release_platforms[self.os_name] + self.get_default_distros() self.install_prefix = args.install_prefix if args.install_prefix is None: self.install_prefix = self.default_install_prefix @@ -626,7 +610,7 @@ def handle_arguments(self, args): self.branch_args = [] self.package_system_branches = [] for branch in self.branches: - package = get_package_from_branch(branch) + package = self.get_package_from_branch(branch) if package is None: # This is an ignored package continue @@ -637,6 +621,26 @@ def handle_arguments(self, args): self.package_system_branches.append(args[0][0]) self.branch_args.extend(args) + def get_package_from_branch(self, branch): + with inbranch(branch): + try: + package_data = get_package_data(branch) + except SystemExit: + return None + if type(package_data) not in [list, tuple]: + # It is a ret code + self.exit(package_data) + names, version, packages = package_data + if type(names) is list and len(names) > 1: + self.exit( + "{0} generator does not support generating " + "from branches with multiple packages in them, use " + "the release generator first to split packages into " + "individual branches." + .format(self.package_system)) + if type(packages) is dict: + return list(packages.values())[0] + def summarize(self): info("Generating {0} source for the packages: {1}".format(self.os_name, str(self.names))) info("Incremental Version: " + str(self.inc)) @@ -649,12 +653,12 @@ def update_rosdep(self): update_rosdep() self.has_run_rosdep = True - def _check_all_keys_are_valid(self, peer_packages, ros_distro): + def _check_all_keys_are_valid(self, peer_packages, rosdistro): keys_to_resolve = [] key_to_packages_which_depends_on = collections.defaultdict(list) keys_to_ignore = set() for package in self.packages.values(): - package.evaluate_conditions(package_conditional_context(ros_distro)) + package.evaluate_conditions(package_conditional_context(rosdistro)) depends = [ dep for dep in (package.run_depends + package.buildtool_export_depends) if dep.evaluated_condition] From 0c9693d8cff01f3e69bdaae726de54068dd2ffad Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Wed, 12 Jun 2019 20:57:51 +0800 Subject: [PATCH 06/23] PackageSystemGenerator for collecting common process arguemnt function --- bloom/generators/common.py | 122 ++++++++++++++++++++++++++- bloom/generators/debian/generator.py | 108 ++---------------------- 2 files changed, 127 insertions(+), 103 deletions(-) diff --git a/bloom/generators/common.py b/bloom/generators/common.py index 80d559cd..733217d5 100644 --- a/bloom/generators/common.py +++ b/bloom/generators/common.py @@ -36,9 +36,15 @@ import sys import traceback +from bloom.git import inbranch +from bloom.git import get_branches + from bloom.logging import debug from bloom.logging import error from bloom.logging import info +from bloom.logging import warning + +from bloom.packages import get_package_data from bloom.rosdistro_api import get_distribution_type @@ -51,10 +57,16 @@ from rosdep2.catkin_support import get_catkin_view from rosdep2.lookup import ResolutionError import rosdep2.catkin_support -except ImportError as err: +except ImportError: debug(traceback.format_exc()) error("rosdep was not detected, please install it.", exit=True) +try: + import rosdistro +except ImportError: + debug(traceback.format_exc()) + error("rosdistro was not detected, please install it.", exit=True) + BLOOM_GROUP = 'bloom.generators' DEFAULT_ROS_DISTRO = 'indigo' @@ -208,6 +220,28 @@ def resolve_dependencies( return resolved_keys +def match_branches_with_prefix(prefix, get_branches, prune=False): + debug("match_branches_with_prefix(" + str(prefix) + ", " + + str(get_branches()) + ")") + branches = [] + # Match branches + existing_branches = get_branches() + for branch in existing_branches: + if branch.startswith('remotes/origin/'): + branch = branch.split('/', 2)[-1] + if branch.startswith(prefix): + branches.append(branch) + branches = list(set(branches)) + if prune: + # Prune listed branches by packages in latest upstream + with inbranch('upstream'): + pkg_names, version, pkgs_dict = get_package_data('upstream') + for branch in branches: + if branch.split(prefix)[-1].strip('/') not in pkg_names: + branches.remove(branch) + return branches + + class GeneratorError(Exception): def __init__(self, msg, returncode=code.UNKNOWN): super(GeneratorError, self).__init__("Error running generator: " + msg) @@ -363,3 +397,89 @@ def post_patch(self, branch_name): :returns: return code, return 0 or None for OK, anythign else on error """ return 0 + + +class PackageSystemGenerator(BloomGenerator): + package_system = 'none' + + def prepare_arguments(self, parser): + # The common command line arguments for every package system + add = parser.add_argument + add('-i', '--inc', help="increment number", default='0') + add('-p', '--prefix', required=True, + help="branch prefix to match, and from which create packages" + " hint: if you want to match 'release/foo' use 'release'") + add('-a', '--match-all', default=False, action="store_true", + help="match all branches with the given prefix, " + "even if not in current upstream") + add('--distros', nargs='+', required=False, default=[], + help='A list of os distros to generate for certain package system') + add('--install-prefix', default=None, + help="overrides the default installation prefix (/usr)") + + def get_package_from_branch(self, branch): + with inbranch(branch): + try: + package_data = get_package_data(branch) + except SystemExit: + return None + if type(package_data) not in [list, tuple]: + # It is a ret code + self.exit(package_data) + names, version, packages = package_data + if type(names) is list and len(names) > 1: + self.exit( + "{0} generator does not support generating " + "from branches with multiple packages in them, use " + "the release generator first to split packages into " + "individual branches." + .format(self.package_system)) + if type(packages) is dict: + return list(packages.values())[0] + + def get_default_distros(self): + index = rosdistro.get_index(rosdistro.get_index_url()) + distribution_file = rosdistro.get_distribution_file(index, self.rosdistro) + if self.os_name not in distribution_file.release_platforms: + if hasattr(self, "os_not_required") and self.os_not_required: + warning("No platforms defined for os '{0}' in release file for the " + "'{1}' distro. This os was not required; continuing without error." + .format(self.os_name, self.rosdistro)) + sys.exit(0) + error("No platforms defined for os '{0}' in release file for the '{1}' distro." + .format(self.os_name, self.rosdistro), exit=True) + self.distros = distribution_file.release_platforms[self.os_name] + + def handle_arguments(self, args): + self.interactive = args.interactive + self.inc = args.inc + self.os_name = args.os_name + self.distros = args.distros + if self.distros in [None, []]: + self.get_default_distros() + self.install_prefix = args.install_prefix + if args.install_prefix is None: + self.install_prefix = self.default_install_prefix + self.prefix = args.prefix + self.branches = match_branches_with_prefix(self.prefix, get_branches, prune=not args.match_all) + if len(self.branches) == 0: + error( + "No packages found, check your --prefix or --src arguments.", + exit=True + ) + self.packages = {} + self.tag_names = {} + self.names = [] + self.branch_args = [] + self.package_system_branches = [] + for branch in self.branches: + package = self.get_package_from_branch(branch) + if package is None: + # This is an ignored package + continue + self.packages[package.name] = package + self.names.append(package.name) + args = self.generate_branching_arguments(package, branch) + # First branch is package_system/[/] + self.package_system_branches.append(args[0][0]) + self.branch_args.extend(args) diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index edc42f1d..b29ea6fe 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -62,7 +62,6 @@ from bloom.generators.common import resolve_rosdep_key from bloom.git import inbranch -from bloom.git import get_branches from bloom.git import get_commit_hash from bloom.git import get_current_branch from bloom.git import has_changes @@ -70,6 +69,7 @@ from bloom.git import tag_exists from bloom.logging import ansi +from bloom.generators.common import PackageSystemGenerator from bloom.logging import debug from bloom.logging import enable_drop_first_log_prefix from bloom.logging import error @@ -507,32 +507,10 @@ def process_template_files(path, subs, package_system): dir_path = os.path.join(path, package_system) if not os.path.exists(dir_path): sys.exit("No {0} directory found at '{1}', cannot process templates." - .format(self.package_system, dir_path)) + .format(package_system, dir_path)) return __process_template_folder(dir_path, subs) -def match_branches_with_prefix(prefix, get_branches, prune=False): - debug("match_branches_with_prefix(" + str(prefix) + ", " + - str(get_branches()) + ")") - branches = [] - # Match branches - existing_branches = get_branches() - for branch in existing_branches: - if branch.startswith('remotes/origin/'): - branch = branch.split('/', 2)[-1] - if branch.startswith(prefix): - branches.append(branch) - branches = list(set(branches)) - if prune: - # Prune listed branches by packages in latest upstream - with inbranch('upstream'): - pkg_names, version, pkgs_dict = get_package_data('upstream') - for branch in branches: - if branch.split(prefix)[-1].strip('/') not in pkg_names: - branches.remove(branch) - return branches - - def debianize_string(value): markup_remover = re.compile(r'<.*?>') value = markup_remover.sub('', value) @@ -545,7 +523,7 @@ def sanitize_package_name(name): return name.replace('_', '-') -class DebianGenerator(BloomGenerator): +class DebianGenerator(PackageSystemGenerator): title = 'debian' package_system = 'debian' description = "Generates debians from the catkin meta data" @@ -554,92 +532,18 @@ class DebianGenerator(BloomGenerator): rosdistro = os.environ.get('ROS_DISTRO', 'indigo') def prepare_arguments(self, parser): - # Add command line arguments for this generator add = parser.add_argument - add('-i', '--inc', help="increment number", default='0') - add('-p', '--prefix', required=True, - help="branch prefix to match, and from which create packages" - " hint: if you want to match 'release/foo' use 'release'") - add('-a', '--match-all', default=False, action="store_true", - help="match all branches with the given prefix, " - "even if not in current upstream") - add('--distros', nargs='+', required=False, default=[], - help='A list of os distros to generate for certain package system') - add('--install-prefix', default=None, - help="overrides the default installation prefix (/usr)") add('--os-name', default='ubuntu', help="overrides os_name, set to 'ubuntu' by default") add('--os-not-required', default=False, action="store_true", help="Do not error if this os is not in the platforms " "list for rosdistro") - - def get_default_distros(self): - index = rosdistro.get_index(rosdistro.get_index_url()) - distribution_file = rosdistro.get_distribution_file(index, self.rosdistro) - if self.os_name not in distribution_file.release_platforms: - if self.os_not_required: - warning("No platforms defined for os '{0}' in release file for the " - "'{1}' distro. This os was not required; continuing without error." - .format(self.os_name, self.rosdistro)) - sys.exit(0) - error("No platforms defined for os '{0}' in release file for the '{1}' distro." - .format(self.os_name, self.rosdistro), exit=True) - self.distros = distribution_file.release_platforms[self.os_name] + return PackageSystemGenerator.prepare_arguments(self, parser) def handle_arguments(self, args): self.os_not_required = args.os_not_required - self.interactive = args.interactive - self.inc = args.inc - self.os_name = args.os_name - self.distros = args.distros - if self.distros in [None, []]: - self.get_default_distros() - self.install_prefix = args.install_prefix - if args.install_prefix is None: - self.install_prefix = self.default_install_prefix - self.prefix = args.prefix - self.branches = match_branches_with_prefix(self.prefix, get_branches, prune=not args.match_all) - if len(self.branches) == 0: - error( - "No packages found, check your --prefix or --src arguments.", - exit=True - ) - self.packages = {} - self.tag_names = {} - self.names = [] - self.branch_args = [] - self.package_system_branches = [] - for branch in self.branches: - package = self.get_package_from_branch(branch) - if package is None: - # This is an ignored package - continue - self.packages[package.name] = package - self.names.append(package.name) - args = self.generate_branching_arguments(package, branch) - # First branch is package_system/[/] - self.package_system_branches.append(args[0][0]) - self.branch_args.extend(args) - - def get_package_from_branch(self, branch): - with inbranch(branch): - try: - package_data = get_package_data(branch) - except SystemExit: - return None - if type(package_data) not in [list, tuple]: - # It is a ret code - self.exit(package_data) - names, version, packages = package_data - if type(names) is list and len(names) > 1: - self.exit( - "{0} generator does not support generating " - "from branches with multiple packages in them, use " - "the release generator first to split packages into " - "individual branches." - .format(self.package_system)) - if type(packages) is dict: - return list(packages.values())[0] + ret = PackageSystemGenerator.handle_arguments(self, args) + return ret def summarize(self): info("Generating {0} source for the packages: {1}".format(self.os_name, str(self.names))) From c10869f5dd34a81226a4633f7efa6f9aa8b57f7b Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Wed, 12 Jun 2019 21:54:53 +0800 Subject: [PATCH 07/23] common function when generate packages files --- bloom/generators/common.py | 189 +++++++++++++++++++++++++++ bloom/generators/debian/generator.py | 158 +--------------------- 2 files changed, 192 insertions(+), 155 deletions(-) diff --git a/bloom/generators/common.py b/bloom/generators/common.py index 733217d5..348dcdf7 100644 --- a/bloom/generators/common.py +++ b/bloom/generators/common.py @@ -32,23 +32,35 @@ from __future__ import print_function +import json + import pkg_resources import sys import traceback from bloom.git import inbranch from bloom.git import get_branches +from bloom.git import get_commit_hash +from bloom.git import get_current_branch +from bloom.git import has_changes +from bloom.git import show +from bloom.git import tag_exists +from bloom.logging import ansi from bloom.logging import debug from bloom.logging import error from bloom.logging import info from bloom.logging import warning +from bloom.commands.git.patch.common import get_patch_config +from bloom.commands.git.patch.common import set_patch_config + from bloom.packages import get_package_data from bloom.rosdistro_api import get_distribution_type from bloom.util import code +from bloom.util import execute_command from bloom.util import maybe_continue from bloom.util import print_exc @@ -401,6 +413,7 @@ def post_patch(self, branch_name): class PackageSystemGenerator(BloomGenerator): package_system = 'none' + has_run_rosdep = False def prepare_arguments(self, parser): # The common command line arguments for every package system @@ -483,3 +496,179 @@ def handle_arguments(self, args): # First branch is package_system/[/] self.package_system_branches.append(args[0][0]) self.branch_args.extend(args) + + def summarize(self): + info("Generating {0} source for the packages: {1}".format(self.os_name, str(self.names))) + info("Incremental Version: " + str(self.inc)) + info("Distributions: " + str(self.distros)) + + def get_branching_arguments(self): + return self.branch_args + + def update_rosdep(self): + update_rosdep() + self.has_run_rosdep = True + + def pre_branch(self, destination, source): + if destination in self.package_system_branches: + return + # Run rosdep update is needed + if not self.has_run_rosdep: + self.update_rosdep() + # Determine the current package being generated + name = destination.split('/')[-1] + distro = destination.split('/')[-2] + # Retrieve the package + package = self.packages[name] + # Report on this package + self.summarize_package(package, distro) + + def pre_rebase(self, destination): + # Get the stored configs is any + patches_branch = 'patches/' + destination + config = self.load_original_config(patches_branch) + if config is not None: + curr_config = get_patch_config(patches_branch) + if curr_config['parent'] == config['parent']: + set_patch_config(patches_branch, config) + + def post_rebase(self, destination): + name = destination.split('/')[-1] + # Retrieve the package + package = self.packages[name] + # Handle differently if this is a package system vs distro branch + if destination in self.package_system_branches: + info("Placing {0} template files into '{1}' branch." + .format(self.package_system, destination)) + # Then this is a package system branch + # Place the raw template files + self.place_template_files(package.get_build_type()) + else: + # This is a distro specific package system branch + # Determine the current package being generated + distro = destination.split('/')[-2] + # Create package for each distro + with inbranch(destination): + data = self.generate_package(package, distro) + # Create the tag name for later + self.tag_names[destination] = self.generate_tag_name(data) + # Update the patch configs + patches_branch = 'patches/' + destination + config = get_patch_config(patches_branch) + # Store it + self.store_original_config(config, patches_branch) + # Modify the base so import/export patch works + current_branch = get_current_branch() + if current_branch is None: + error("Could not determine current branch.", exit=True) + config['base'] = get_commit_hash(current_branch) + # Set it + set_patch_config(patches_branch, config) + + def post_patch(self, destination, color='bluef'): + if destination in self.package_system_branches: + return + # Tag after patches have been applied + with inbranch(destination): + # Tag + tag_name = self.tag_names[destination] + if tag_exists(tag_name): + if self.interactive: + warning("Tag exists: " + tag_name) + warning("Do you wish to overwrite it?") + if not maybe_continue('y'): + error("Answered no to continue, aborting.", exit=True) + else: + warning("Overwriting tag: " + tag_name) + else: + info("Creating tag: " + tag_name) + execute_command('git tag -f ' + tag_name) + # Report of success + name = destination.split('/')[-1] + package = self.packages[name] + distro = destination.split('/')[-2] + info(ansi(color) + "####" + ansi('reset'), use_prefix=False) + info( + ansi(color) + "#### " + ansi('greenf') + "Successfully" + + ansi(color) + " generated '" + ansi('boldon') + distro + + ansi('boldoff') + "' {0} for package".format(self.package_system) + + " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + + " at version '" + ansi('boldon') + package.version + + "-" + str(self.inc) + ansi('boldoff') + "'" + + ansi('reset'), + use_prefix=False + ) + info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False) + + def store_original_config(self, config, patches_branch): + with inbranch(patches_branch): + with open('{0}.store'.format(self.package_system), 'w+') as f: + f.write(json.dumps(config)) + execute_command('git add {0}.store'.format(self.package_system)) + if has_changes(): + execute_command('git commit -m "Store original patch config"') + + def load_original_config(self, patches_branch): + config_store = show(patches_branch, '{0}.store'.format(self.package_system)) + if config_store is None: + return config_store + return json.loads(config_store) + + def generate_branching_arguments(self, package, branch): + """ + The default branch for placing package system release data + + :param package: the package metadata extract from package.xml + :param branch: every branch match the prefix in command line input + + :return: list of (destination, source, interactive) + """ + n = package.name + # package branch + package_branch = self.package_system + '/' + n + # Branch first to the package branch + args = [[package_branch, branch, False]] + # Then for each os distro, branch from the base package branch + args.extend([ + [self.package_system + '/' + d + '/' + n, package_branch, False] + for d in self.distros + ]) + return args + + def summarize_package(self, package, distro, color='bluef'): + info(ansi(color) + "\n####" + ansi('reset'), use_prefix=False) + info( + ansi(color) + "#### Generating '" + ansi('boldon') + distro + + ansi('boldoff') + "' {0} for package".format(self.package_system) + + " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + + " at version '" + ansi('boldon') + package.version + + "-" + str(self.inc) + ansi('boldoff') + "'" + + ansi('reset'), + use_prefix=False + ) + info(ansi(color) + "####" + ansi('reset'), use_prefix=False) + + def generate_package(self, package, os_version): + """ + Assume we have the templactes file in directory + The overriten function should generate the package, including + 1. use the result of get_subs to replace template content + 2. set the newest release history + 3. some git commit operation + + :param package: the substitute for in place of the templacte content + :param os_version: the specific operate system version + + :returns: substitutes for other use + """ + raise NotImplemented + + def generate_tag_name(self, subs): + """ + Generate tag name based on the substitute, this method need be overwriten + + :param subs: the substitute for in place of the templacte content + + :returns: tag name + """ + raise NotImplemented diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index b29ea6fe..ada8cb60 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -51,7 +51,6 @@ from dateutil import tz from pkg_resources import parse_version -from bloom.generators import BloomGenerator from bloom.generators import GeneratorError from bloom.generators import resolve_dependencies from bloom.generators import update_rosdep @@ -59,17 +58,15 @@ from bloom.generators.common import default_fallback_resolver from bloom.generators.common import invalidate_view_cache from bloom.generators.common import package_conditional_context +from bloom.generators.common import PackageSystemGenerator from bloom.generators.common import resolve_rosdep_key from bloom.git import inbranch -from bloom.git import get_commit_hash from bloom.git import get_current_branch from bloom.git import has_changes from bloom.git import show -from bloom.git import tag_exists from bloom.logging import ansi -from bloom.generators.common import PackageSystemGenerator from bloom.logging import debug from bloom.logging import enable_drop_first_log_prefix from bloom.logging import error @@ -78,11 +75,6 @@ from bloom.logging import is_debug from bloom.logging import warning -from bloom.commands.git.patch.common import get_patch_config -from bloom.commands.git.patch.common import set_patch_config - -from bloom.packages import get_package_data - from bloom.util import code from bloom.util import to_unicode from bloom.util import execute_command @@ -527,7 +519,6 @@ class DebianGenerator(PackageSystemGenerator): title = 'debian' package_system = 'debian' description = "Generates debians from the catkin meta data" - has_run_rosdep = False default_install_prefix = '/usr' rosdistro = os.environ.get('ROS_DISTRO', 'indigo') @@ -545,18 +536,6 @@ def handle_arguments(self, args): ret = PackageSystemGenerator.handle_arguments(self, args) return ret - def summarize(self): - info("Generating {0} source for the packages: {1}".format(self.os_name, str(self.names))) - info("Incremental Version: " + str(self.inc)) - info("Distributions: " + str(self.distros)) - - def get_branching_arguments(self): - return self.branch_args - - def update_rosdep(self): - update_rosdep() - self.has_run_rosdep = True - def _check_all_keys_are_valid(self, peer_packages, rosdistro): keys_to_resolve = [] key_to_packages_which_depends_on = collections.defaultdict(list) @@ -632,111 +611,6 @@ def pre_modify(self): info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") - def pre_branch(self, destination, source): - if destination in self.package_system_branches: - return - # Run rosdep update is needed - if not self.has_run_rosdep: - self.update_rosdep() - # Determine the current package being generated - name = destination.split('/')[-1] - distro = destination.split('/')[-2] - # Retrieve the package - package = self.packages[name] - # Report on this package - self.summarize_package(package, distro) - - def pre_rebase(self, destination): - # Get the stored configs is any - patches_branch = 'patches/' + destination - config = self.load_original_config(patches_branch) - if config is not None: - curr_config = get_patch_config(patches_branch) - if curr_config['parent'] == config['parent']: - set_patch_config(patches_branch, config) - - def post_rebase(self, destination): - name = destination.split('/')[-1] - # Retrieve the package - package = self.packages[name] - # Handle differently if this is a package system vs distro branch - if destination in self.package_system_branches: - info("Placing {0} template files into '{1}' branch." - .format(self.package_system, destination)) - # Then this is a package system branch - # Place the raw template files - self.place_template_files(package.get_build_type()) - else: - # This is a distro specific package system branch - # Determine the current package being generated - distro = destination.split('/')[-2] - # Create package for each distro - with inbranch(destination): - data = self.generate_package(package, distro) - # Create the tag name for later - self.tag_names[destination] = self.generate_tag_name(data) - # Update the patch configs - patches_branch = 'patches/' + destination - config = get_patch_config(patches_branch) - # Store it - self.store_original_config(config, patches_branch) - # Modify the base so import/export patch works - current_branch = get_current_branch() - if current_branch is None: - error("Could not determine current branch.", exit=True) - config['base'] = get_commit_hash(current_branch) - # Set it - set_patch_config(patches_branch, config) - - def post_patch(self, destination, color='bluef'): - if destination in self.package_system_branches: - return - # Tag after patches have been applied - with inbranch(destination): - # Tag - tag_name = self.tag_names[destination] - if tag_exists(tag_name): - if self.interactive: - warning("Tag exists: " + tag_name) - warning("Do you wish to overwrite it?") - if not maybe_continue('y'): - error("Answered no to continue, aborting.", exit=True) - else: - warning("Overwriting tag: " + tag_name) - else: - info("Creating tag: " + tag_name) - execute_command('git tag -f ' + tag_name) - # Report of success - name = destination.split('/')[-1] - package = self.packages[name] - distro = destination.split('/')[-2] - info(ansi(color) + "####" + ansi('reset'), use_prefix=False) - info( - ansi(color) + "#### " + ansi('greenf') + "Successfully" + - ansi(color) + " generated '" + ansi('boldon') + distro + - ansi('boldoff') + "' {0} for package".format(self.package_system) + - " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + - " at version '" + ansi('boldon') + package.version + - "-" + str(self.inc) + ansi('boldoff') + "'" + - ansi('reset'), - use_prefix=False - ) - info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False) - - def store_original_config(self, config, patches_branch): - with inbranch(patches_branch): - with open('{0}.store'.format(self.package_system), 'w+') as f: - f.write(json.dumps(config)) - execute_command('git add {0}.store'.format(self.package_system)) - if has_changes(): - execute_command('git commit -m "Store original patch config"') - - def load_original_config(self, patches_branch): - config_store = show(patches_branch, '{0}.store'.format(self.package_system)) - if config_store is None: - return config_store - return json.loads(config_store) - def place_template_files(self, build_type, dir_path=None): # Create/Clean the package system folder if dir_path is None: @@ -835,33 +709,7 @@ def get_release_tag(self, data): return 'release/{0}/{1}-{2}'.format(data['Name'], data['Version'], self.inc) - def generate_tag_name(self, data): + def generate_tag_name(self, subs): tag_name = '{Package}_{Version}{Inc}_{Distribution}' - tag_name = self.package_system + '/' + tag_name.format(**data) + tag_name = self.package_system + '/' + tag_name.format(**subs) return tag_name - - def generate_branching_arguments(self, package, branch): - n = package.name - # package branch - package_branch = self.package_system + '/' + n - # Branch first to the package branch - args = [[package_branch, branch, False]] - # Then for each os distro, branch from the base package branch - args.extend([ - [self.package_system + '/' + d + '/' + n, package_branch, False] - for d in self.distros - ]) - return args - - def summarize_package(self, package, distro, color='bluef'): - info(ansi(color) + "\n####" + ansi('reset'), use_prefix=False) - info( - ansi(color) + "#### Generating '" + ansi('boldon') + distro + - ansi('boldoff') + "' {0} for package".format(self.package_system) + - " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + - " at version '" + ansi('boldon') + package.version + - "-" + str(self.inc) + ansi('boldoff') + "'" + - ansi('reset'), - use_prefix=False - ) - info(ansi(color) + "####" + ansi('reset'), use_prefix=False) From 95cc65b83cd56de3ed1104339c70b89ddb5a9a45 Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Wed, 3 Jul 2019 19:53:21 +0800 Subject: [PATCH 08/23] deal with specific part of template operation Firstly put the common place/process template files to common.py. Then, since every pacakge_system has its prefer way of formatting description and depends, so pass them as function. Next, rewrite get_sub_hook in DebianGenerator for debian specific substitutions attribute, and consider generate_cmd will use the hook, I make the hook as staticmethod method. At last, fix up the api change influence. --- bloom/generators/common.py | 477 +++++++++++++- bloom/generators/debian/__init__.py | 5 +- bloom/generators/debian/generate_cmd.py | 19 +- bloom/generators/debian/generator.py | 618 +++--------------- bloom/generators/rosdebian.py | 31 +- test/system_tests/test_catkin_release.py | 2 +- .../test_debian/test_generator.py | 2 +- 7 files changed, 590 insertions(+), 564 deletions(-) diff --git a/bloom/generators/common.py b/bloom/generators/common.py index 348dcdf7..445428e1 100644 --- a/bloom/generators/common.py +++ b/bloom/generators/common.py @@ -32,12 +32,18 @@ from __future__ import print_function +import collections import json - +import io +import os import pkg_resources +import shutil import sys import traceback +from bloom.commands.git.patch.common import get_patch_config +from bloom.commands.git.patch.common import set_patch_config + from bloom.git import inbranch from bloom.git import get_branches from bloom.git import get_commit_hash @@ -49,12 +55,10 @@ from bloom.logging import ansi from bloom.logging import debug from bloom.logging import error +from bloom.logging import fmt from bloom.logging import info from bloom.logging import warning -from bloom.commands.git.patch.common import get_patch_config -from bloom.commands.git.patch.common import set_patch_config - from bloom.packages import get_package_data from bloom.rosdistro_api import get_distribution_type @@ -64,6 +68,11 @@ from bloom.util import maybe_continue from bloom.util import print_exc +try: + from configparser import SafeConfigParser +except ImportError: + from ConfigParser import SafeConfigParser + try: from rosdep2 import create_default_installer_context from rosdep2.catkin_support import get_catkin_view @@ -79,8 +88,26 @@ debug(traceback.format_exc()) error("rosdistro was not detected, please install it.", exit=True) +try: + import em +except ImportError: + debug(traceback.format_exc()) + error("empy was not detected, please install it.", exit=True) + +# Fix unicode bug in empy +# This should be removed once upstream empy is fixed +# See: https://github.com/ros-infrastructure/bloom/issues/196 +try: + em.str = unicode + em.Stream.write_old = em.Stream.write + em.Stream.write = lambda self, data: em.Stream.write_old(self, data.encode('utf8')) +except NameError: + pass +# End fix + BLOOM_GROUP = 'bloom.generators' DEFAULT_ROS_DISTRO = 'indigo' +TEMPLATE_EXTENSION = '.em' def list_generators(): @@ -232,6 +259,249 @@ def resolve_dependencies( return resolved_keys +def generate_substitutions_from_package( + package, + os_name, + os_version, + ros_distro, + format_description, + format_depends, + installation_prefix='/usr', + inc=0, + peer_packages=None, + fallback_resolver=None, + native=False +): + peer_packages = peer_packages or [] + data = {} + # Name, Version, Description + data['Name'] = package.name + data['Version'] = package.version + data['Description'] = format_description(package.description) + # Websites + websites = [str(url) for url in package.urls if url.type == 'website'] + homepage = websites[0] if websites else '' + if homepage == '': + warning("No homepage set, defaulting to ''") + data['Homepage'] = homepage + # Increment Number + data['Inc'] = '' if native else '-{0}'.format(inc) + # Package name + data['Package'] = sanitize_package_name(package.name) + # Installation prefix + data['InstallationPrefix'] = installation_prefix + # Resolve dependencies + package.evaluate_conditions(package_conditional_context(ros_distro)) + depends = [ + dep for dep in (package.run_depends + package.buildtool_export_depends) + if dep.evaluated_condition] + build_depends = [ + dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) + if dep.evaluated_condition] + + unresolved_keys = [ + dep for dep in (depends + build_depends + package.replaces + package.conflicts) + if dep.evaluated_condition] + # The installer key is not considered here, but it is checked when the keys are checked before this + resolved_deps = resolve_dependencies(unresolved_keys, os_name, + os_version, ros_distro, + peer_packages + [d.name for d in package.replaces + package.conflicts], + fallback_resolver) + data['Depends'] = sorted( + set(format_depends(depends, resolved_deps)) + ) + data['BuildDepends'] = sorted( + set(format_depends(build_depends, resolved_deps)) + ) + data['Replaces'] = sorted( + set(format_depends(package.replaces, resolved_deps)) + ) + data['Conflicts'] = sorted( + set(format_depends(package.conflicts, resolved_deps)) + ) + + # Build-type specific substitutions. + build_type = package.get_build_type() + if build_type == 'catkin': + pass + elif build_type == 'cmake': + pass + elif build_type == 'ament_cmake': + pass + elif build_type == 'ament_python': + # Don't set the install-scripts flag if it's already set in setup.cfg. + package_path = os.path.abspath(os.path.dirname(package.filename)) + setup_cfg_path = os.path.join(package_path, 'setup.cfg') + data['pass_install_scripts'] = True + if os.path.isfile(setup_cfg_path): + setup_cfg = SafeConfigParser() + setup_cfg.read([setup_cfg_path]) + if ( + setup_cfg.has_option('install', 'install-scripts') or + setup_cfg.has_option('install', 'install_scripts') + ): + data['pass_install_scripts'] = False + else: + error("Build type '{}' is not supported by this version of bloom.". + format(build_type), exit=True) + + # Set the distribution + data['Distribution'] = os_version + # Maintainers + maintainers = [] + for m in package.maintainers: + maintainers.append(str(m)) + data['Maintainer'] = maintainers[0] + data['Maintainers'] = ', '.join(maintainers) + + # Summarize dependencies + summarize_dependency_mapping(data, depends, build_depends, resolved_deps) + + return data + + +def __place_template_folder(group, src, dst, gbp=False): + template_files = pkg_resources.resource_listdir(group, src) + # For each template, place + for template_file in template_files: + if not gbp and os.path.basename(template_file) == 'gbp.conf.em': + debug("Skipping template '{0}'".format(template_file)) + continue + template_path = os.path.join(src, template_file) + template_dst = os.path.join(dst, template_file) + if pkg_resources.resource_isdir(group, template_path): + debug("Recursing on folder '{0}'".format(template_path)) + __place_template_folder(group, template_path, template_dst, gbp) + else: + try: + debug("Placing template '{0}'".format(template_path)) + template = pkg_resources.resource_string(group, template_path) + template_abs_path = pkg_resources.resource_filename(group, template_path) + except IOError as err: + error("Failed to load template " + "'{0}': {1}".format(template_file, str(err)), exit=True) + if not os.path.exists(dst): + os.makedirs(dst) + if os.path.exists(template_dst): + debug("Not overwriting existing file '{0}'".format(template_dst)) + else: + with io.open(template_dst, 'w', encoding='utf-8') as f: + if not isinstance(template, str): + template = template.decode('utf-8') + # Python 2 API needs a `unicode` not a utf-8 string. + elif sys.version_info.major == 2: + template = template.decode('utf-8') + f.write(template) + shutil.copystat(template_abs_path, template_dst) + + +def convertToUnicode(obj): + if sys.version_info.major == 2: + if isinstance(obj, str): + return unicode(obj.decode('utf8')) + elif isinstance(obj, unicode): + return obj + else: + if isinstance(obj, bytes): + return str(obj.decode('utf8')) + elif isinstance(obj, str): + return obj + if isinstance(obj, list): + for i, val in enumerate(obj): + obj[i] = convertToUnicode(val) + return obj + elif isinstance(obj, type(None)): + return None + elif isinstance(obj, tuple): + obj_tmp = list(obj) + for i, val in enumerate(obj_tmp): + obj_tmp[i] = convertToUnicode(obj_tmp[i]) + return tuple(obj_tmp) + elif isinstance(obj, int): + return obj + raise RuntimeError('need to deal with type %s' % (str(type(obj)))) + + +def place_template_files(path, build_type, package_system, gbp=False): + info(fmt("@!@{bf}==>@| Placing templates files in the '" + package_system + "' folder.")) + dir_path = os.path.join(path, package_system) + if not os.path.exists(dir_path): + os.makedirs(dir_path) + # Place template files + group = 'bloom.generators.' + package_system + templates = os.path.join('templates', build_type) + __place_template_folder(group, templates, dir_path, gbp) + + +def summarize_dependency_mapping(data, deps, build_deps, resolved_deps): + if len(deps) == 0 and len(build_deps) == 0: + return + info("Package '" + data['Package'] + "' has dependencies:") + header = " " + ansi('boldoff') + ansi('ulon') + \ + "rosdep key => " + data['Distribution'] + \ + " key" + ansi('reset') + template = " " + ansi('cyanf') + "{0:<20} " + ansi('purplef') + \ + "=> " + ansi('cyanf') + "{1}" + ansi('reset') + if len(deps) != 0: + info(ansi('purplef') + "Run Dependencies:" + + ansi('reset')) + info(header) + for key in [d.name for d in deps]: + info(template.format(key, resolved_deps[key])) + if len(build_deps) != 0: + info(ansi('purplef') + + "Build and Build Tool Dependencies:" + ansi('reset')) + info(header) + for key in [d.name for d in build_deps]: + info(template.format(key, resolved_deps[key])) + + +def __process_template_folder(path, subs): + items = os.listdir(path) + processed_items = [] + for item in list(items): + item = os.path.abspath(os.path.join(path, item)) + if os.path.basename(item) in ['.', '..', '.git', '.svn']: + continue + if os.path.isdir(item): + sub_items = __process_template_folder(item, subs) + processed_items.extend([os.path.join(item, s) for s in sub_items]) + if not item.endswith(TEMPLATE_EXTENSION): + continue + with open(item, 'r') as f: + template = f.read() + # Remove extension + template_path = item[:-len(TEMPLATE_EXTENSION)] + # Expand template + info("Expanding '{0}' -> '{1}'".format( + os.path.relpath(item), + os.path.relpath(template_path))) + result = em.expand(template, **subs) + # Don't write an empty file + if len(result) == 0 and \ + os.path.basename(template_path) in ['copyright']: + processed_items.append(item) + continue + # Write the result + with io.open(template_path, 'w', encoding='utf-8') as f: + if sys.version_info.major == 2: + result = result.decode('utf-8') + f.write(result) + # Copy the permissions + shutil.copymode(item, template_path) + processed_items.append(item) + return processed_items + + +def process_template_files(path, subs, package_system): + info(fmt("@!@{bf}==>@| In place processing templates files in '" + package_system + "' folder.")) + dir_path = os.path.join(path, package_system) + if not os.path.exists(dir_path): + sys.exit("No {0} directory found at '{1}', cannot process templates." + .format(package_system, dir_path)) + return __process_template_folder(dir_path, subs) + + def match_branches_with_prefix(prefix, get_branches, prune=False): debug("match_branches_with_prefix(" + str(prefix) + ", " + str(get_branches()) + ")") @@ -254,6 +524,14 @@ def match_branches_with_prefix(prefix, get_branches, prune=False): return branches +def rosify_package_name(name, rosdistro): + return 'ros-{0}-{1}'.format(rosdistro, name) + + +def sanitize_package_name(name): + return name.replace('_', '-') + + class GeneratorError(Exception): def __init__(self, msg, returncode=code.UNKNOWN): super(GeneratorError, self).__init__("Error running generator: " + msg) @@ -509,6 +787,80 @@ def update_rosdep(self): update_rosdep() self.has_run_rosdep = True + def _check_all_keys_are_valid(self, peer_packages, rosdistro): + keys_to_resolve = [] + key_to_packages_which_depends_on = collections.defaultdict(list) + keys_to_ignore = set() + for package in self.packages.values(): + package.evaluate_conditions(package_conditional_context(rosdistro)) + depends = [ + dep for dep in (package.run_depends + package.buildtool_export_depends) + if dep.evaluated_condition] + build_depends = [ + dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) + if dep.evaluated_condition] + unresolved_keys = [ + dep for dep in (depends + build_depends + package.replaces + package.conflicts) + if dep.evaluated_condition] + keys_to_ignore = { + dep for dep in keys_to_ignore.union(package.replaces + package.conflicts) + if dep.evaluated_condition} + keys = [d.name for d in unresolved_keys] + keys_to_resolve.extend(keys) + for key in keys: + key_to_packages_which_depends_on[key].append(package.name) + + os_name = self.os_name + rosdistro = self.rosdistro + all_keys_valid = True + for key in sorted(set(keys_to_resolve)): + for os_version in self.distros: + try: + extended_peer_packages = peer_packages + [d.name for d in keys_to_ignore] + rule, installer_key, default_installer_key = \ + resolve_rosdep_key(key, os_name, os_version, rosdistro, extended_peer_packages, + retry=False) + if rule is None: + continue + if installer_key != default_installer_key: + error("Key '{0}' resolved to '{1}' with installer '{2}', " + "which does not match the default installer '{3}'." + .format(key, rule, installer_key, default_installer_key)) + self.exit( + "The {0} generator does not support dependencies " + "which are installed with the '{1}' installer." + .format(self.package_system, installer_key), + returncode=code.GENERATOR_INVALID_INSTALLER_KEY) + except (GeneratorError, RuntimeError) as e: + print(fmt("Failed to resolve @{cf}@!{key}@| on @{bf}{os_name}@|:@{cf}@!{os_version}@| with: {e}") + .format(**locals())) + print(fmt("@{cf}@!{0}@| is depended on by these packages: ").format(key) + + str(list(set(key_to_packages_which_depends_on[key])))) + print(fmt("@{kf}@!<== @{rf}@!Failed@|")) + all_keys_valid = False + return all_keys_valid + + def _pre_modify(self, key_unvalid_error_msg): + info("\nPre-verifying {0} dependency keys...".format(self.package_system)) + # Run rosdep update is needed + if not self.has_run_rosdep: + self.update_rosdep() + + peer_packages = [p.name for p in self.packages.values()] + + while not self._check_all_keys_are_valid(peer_packages, self.rosdistro): + error(key_unvalid_error_msg) + try: + if not maybe_continue(msg="Would you like to try again?"): + error("User aborted after rosdep keys were not resolved.") + sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) + except (KeyboardInterrupt, EOFError): + error("\nUser quit.", exit=True) + update_rosdep() + invalidate_view_cache() + + info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") + def pre_branch(self, destination, source): if destination in self.package_system_branches: return @@ -614,6 +966,96 @@ def load_original_config(self, patches_branch): return config_store return json.loads(config_store) + def get_releaser_history(self): + # Assumes that this is called in the target branch + patches_branch = 'patches/' + get_current_branch() + raw = show(patches_branch, 'releaser_history.json') + return None if raw is None else json.loads(raw) + + def set_releaser_history(self, history): + # Assumes that this is called in the target branch + patches_branch = 'patches/' + get_current_branch() + debug("Writing release history to '{0}' branch".format(patches_branch)) + with inbranch(patches_branch): + with open('releaser_history.json', 'w') as f: + f.write(json.dumps(history)) + execute_command('git add releaser_history.json') + if has_changes(): + execute_command('git commit -m "Store releaser history"') + + @staticmethod + def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): + """ + This should be a staticmethod since we will use it when we call + `generate_substitutions_from_package` in `get_subs` + Notice that os_name, os_version, ros_distro maybe useful when we + want to add new resolver in the future + """ + if key in peer_packages: + return [sanitize_package_name(key)] + return default_fallback_resolver(key, peer_packages) + + def place_template_files(self, build_type, dir_path=None): + # Create/Clean the package system folder + if dir_path is None: + dir_path = os.path.join(".", self.package_system) + if os.path.exists(dir_path): + if self.interactive: + warning("{0} directory exists: {1}".format(self.package_system, dir_path)) + warning("Do you wish to overwrite it?") + if not maybe_continue('y'): + error("Answered no to continue, aborting.", exit=True) + elif 'BLOOM_CLEAR_TEMPLATE_ON_GENERATION' in os.environ: + warning("Overwriting {0} directory: {1}".format(self.package_system, dir_path)) + execute_command('git rm -rf ' + dir_path) + execute_command('git commit -m "Clearing previous {0} folder"' + .format(self.package_system)) + if os.path.exists(dir_path): + shutil.rmtree(dir_path) + else: + warning("Not overwriting {0} directory.".format(self.package_system)) + # Use generic place template files command + place_template_files('.', build_type, self.package_system, gbp=True) + # Commit results + execute_command('git add ' + dir_path) + _, has_files, _ = execute_command('git diff --cached --name-only', return_io=True) + if has_files: + execute_command('git commit -m "Placing {0} template files"'.format(self.package_system)) + + def get_subs(self, package, os_version, format_description, format_depends, releaser_history=None): + # This is the common part for generate templacte substitute, then successor of + # the generator will add its specic content via define its get_subs_hook function + subs = generate_substitutions_from_package( + package, + self.os_name, + os_version, + self.rosdistro, + format_description, + format_depends, + self.install_prefix, + self.inc, + [p.name for p in self.packages.values()], + fallback_resolver=self.missing_dep_resolver + ) + subs['release_tag'] = 'release/{0}/{1}-{2}'.format(subs['Name'], subs['Version'], self.inc) + subs = self.get_subs_hook(subs, package, self.rosdistro, releaser_history=releaser_history) + for item in subs.items(): + subs[item[0]] = convertToUnicode(item[1]) + return subs + + def summarize_package(self, package, distro, color='bluef'): + info(ansi(color) + "\n####" + ansi('reset'), use_prefix=False) + info( + ansi(color) + "#### Generating '" + ansi('boldon') + distro + + ansi('boldoff') + "' {0} for package".format(self.package_system) + + " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + + " at version '" + ansi('boldon') + package.version + + "-" + str(self.inc) + ansi('boldoff') + "'" + + ansi('reset'), + use_prefix=False + ) + info(ansi(color) + "####" + ansi('reset'), use_prefix=False) + def generate_branching_arguments(self, package, branch): """ The default branch for placing package system release data @@ -635,19 +1077,6 @@ def generate_branching_arguments(self, package, branch): ]) return args - def summarize_package(self, package, distro, color='bluef'): - info(ansi(color) + "\n####" + ansi('reset'), use_prefix=False) - info( - ansi(color) + "#### Generating '" + ansi('boldon') + distro + - ansi('boldoff') + "' {0} for package".format(self.package_system) + - " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + - " at version '" + ansi('boldon') + package.version + - "-" + str(self.inc) + ansi('boldoff') + "'" + - ansi('reset'), - use_prefix=False - ) - info(ansi(color) + "####" + ansi('reset'), use_prefix=False) - def generate_package(self, package, os_version): """ Assume we have the templactes file in directory @@ -663,6 +1092,20 @@ def generate_package(self, package, os_version): """ raise NotImplemented + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + """ + The specific package system related substitute operation + + :param subs: the substitute for in place of the template content + :param package: the substitute for in place of the templacte content + :param rosdistro: the ros version + :param releaser_history: the substitute for in place of the templacte content + + :returns: the improved subs + """ + return subs + def generate_tag_name(self, subs): """ Generate tag name based on the substitute, this method need be overwriten diff --git a/bloom/generators/debian/__init__.py b/bloom/generators/debian/__init__.py index 385f4c99..daa4096a 100755 --- a/bloom/generators/debian/__init__.py +++ b/bloom/generators/debian/__init__.py @@ -1,4 +1,3 @@ -from .generator import DebianGenerator -from .generator import sanitize_package_name +from .generator import DebianGenerator, format_description, format_depends -__all__ = ['DebianGenerator', 'sanitize_package_name'] +__all__ = ['DebianGenerator', 'format_description', 'format_depends'] diff --git a/bloom/generators/debian/generate_cmd.py b/bloom/generators/debian/generate_cmd.py index a0f450f9..dd96045c 100644 --- a/bloom/generators/debian/generate_cmd.py +++ b/bloom/generators/debian/generate_cmd.py @@ -42,10 +42,13 @@ from bloom.logging import fmt from bloom.logging import info -from bloom.generators.debian.generator import generate_substitutions_from_package -from bloom.generators.debian.generator import place_template_files -from bloom.generators.debian.generator import process_template_files +from bloom.generators.common import generate_substitutions_from_package +from bloom.generators.common import place_template_files +from bloom.generators.common import process_template_files + from bloom.generators.debian.generator import DebianGenerator +from bloom.generators.debian.generator import format_description +from bloom.generators.debian.generator import format_depends from bloom.util import get_distro_list_prompt @@ -81,13 +84,18 @@ def prepare_arguments(parser): def get_subs(pkg, os_name, os_version, ros_distro, native=False): - return generate_substitutions_from_package( + subs = generate_substitutions_from_package( pkg, os_name, os_version, ros_distro, - native=native + format_description, + format_depends, ) + subs = DebianGenerator.get_subs_hook(subs, pkg, ros_distro) + # Debian Package Format + subs['format'] = 'native' if native else 'quilt' + return subs def main(args=None, get_subs_fn=None): @@ -121,7 +129,6 @@ def main(args=None, get_subs_fn=None): fmt("Generating debs for @{cf}%s:%s@| for package(s) %s" % (os_name, os_version, [p.name for p in pkgs_dict.values()]))) - # TODO: seems troublesome here package_system = DebianGenerator.package_system for path, pkg in pkgs_dict.items(): template_files = None diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index ada8cb60..fbf2b24f 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -32,50 +32,25 @@ from __future__ import print_function -import collections import datetime -import io -import json import os -import pkg_resources import re -import shutil import sys import traceback -# Python 2/3 support. -try: - from configparser import SafeConfigParser -except ImportError: - from ConfigParser import SafeConfigParser from dateutil import tz from pkg_resources import parse_version -from bloom.generators import GeneratorError -from bloom.generators import resolve_dependencies -from bloom.generators import update_rosdep - -from bloom.generators.common import default_fallback_resolver -from bloom.generators.common import invalidate_view_cache -from bloom.generators.common import package_conditional_context from bloom.generators.common import PackageSystemGenerator -from bloom.generators.common import resolve_rosdep_key +from bloom.generators.common import process_template_files -from bloom.git import inbranch -from bloom.git import get_current_branch -from bloom.git import has_changes -from bloom.git import show - -from bloom.logging import ansi from bloom.logging import debug from bloom.logging import enable_drop_first_log_prefix from bloom.logging import error -from bloom.logging import fmt from bloom.logging import info from bloom.logging import is_debug from bloom.logging import warning -from bloom.util import code from bloom.util import to_unicode from bloom.util import execute_command from bloom.util import get_rfc_2822_date @@ -88,103 +63,9 @@ debug(traceback.format_exc()) error("catkin_pkg was not detected, please install it.", exit=True) -try: - import rosdistro -except ImportError as err: - debug(traceback.format_exc()) - error("rosdistro was not detected, please install it.", exit=True) - -try: - import em -except ImportError: - debug(traceback.format_exc()) - error("empy was not detected, please install it.", exit=True) - -# Fix unicode bug in empy -# This should be removed once upstream empy is fixed -# See: https://github.com/ros-infrastructure/bloom/issues/196 -try: - em.str = unicode - em.Stream.write_old = em.Stream.write - em.Stream.write = lambda self, data: em.Stream.write_old(self, data.encode('utf8')) -except NameError: - pass -# End fix - # Drop the first log prefix for this command enable_drop_first_log_prefix(True) -TEMPLATE_EXTENSION = '.em' - - -def __place_template_folder(group, src, dst, gbp=False): - template_files = pkg_resources.resource_listdir(group, src) - # For each template, place - for template_file in template_files: - if not gbp and os.path.basename(template_file) == 'gbp.conf.em': - debug("Skipping template '{0}'".format(template_file)) - continue - template_path = os.path.join(src, template_file) - template_dst = os.path.join(dst, template_file) - if pkg_resources.resource_isdir(group, template_path): - debug("Recursing on folder '{0}'".format(template_path)) - __place_template_folder(group, template_path, template_dst, gbp) - else: - try: - debug("Placing template '{0}'".format(template_path)) - template = pkg_resources.resource_string(group, template_path) - template_abs_path = pkg_resources.resource_filename(group, template_path) - except IOError as err: - error("Failed to load template " - "'{0}': {1}".format(template_file, str(err)), exit=True) - if not os.path.exists(dst): - os.makedirs(dst) - if os.path.exists(template_dst): - debug("Not overwriting existing file '{0}'".format(template_dst)) - else: - with io.open(template_dst, 'w', encoding='utf-8') as f: - if not isinstance(template, str): - template = template.decode('utf-8') - # Python 2 API needs a `unicode` not a utf-8 string. - elif sys.version_info.major == 2: - template = template.decode('utf-8') - f.write(template) - shutil.copystat(template_abs_path, template_dst) - - -def place_template_files(path, build_type, package_system, gbp=False): - info(fmt("@!@{bf}==>@| Placing templates files in the '" + package_system + "' folder.")) - dir_path = os.path.join(path, package_system) - if not os.path.exists(dir_path): - os.makedirs(dir_path) - # Place template files - group = 'bloom.generators.' + package_system - templates = os.path.join('templates', build_type) - __place_template_folder(group, templates, dir_path, gbp) - - -def summarize_dependency_mapping(data, deps, build_deps, resolved_deps): - if len(deps) == 0 and len(build_deps) == 0: - return - info("Package '" + data['Package'] + "' has dependencies:") - header = " " + ansi('boldoff') + ansi('ulon') + \ - "rosdep key => " + data['Distribution'] + \ - " key" + ansi('reset') - template = " " + ansi('cyanf') + "{0:<20} " + ansi('purplef') + \ - "=> " + ansi('cyanf') + "{1}" + ansi('reset') - if len(deps) != 0: - info(ansi('purplef') + "Run Dependencies:" + - ansi('reset')) - info(header) - for key in [d.name for d in deps]: - info(template.format(key, resolved_deps[key])) - if len(build_deps) != 0: - info(ansi('purplef') + - "Build and Build Tool Dependencies:" + ansi('reset')) - info(header) - for key in [d.name for d in build_deps]: - info(template.format(key, resolved_deps[key])) - def format_depends(depends, resolved_deps): versions = { @@ -209,6 +90,14 @@ def format_depends(depends, resolved_deps): return formatted +def debianize_string(value): + markup_remover = re.compile(r'<.*?>') + value = markup_remover.sub('', value) + value = re.sub('\s+', ' ', value) + value = value.strip() + return value + + def format_description(value): """ Format proper string following Debian control file @@ -265,256 +154,6 @@ def get_changelogs(package, releaser_history=None): return [] -def generate_substitutions_from_package( - package, - os_name, - os_version, - ros_distro, - installation_prefix='/usr', - inc=0, - peer_packages=None, - releaser_history=None, - fallback_resolver=None, - native=False -): - peer_packages = peer_packages or [] - data = {} - # Name, Version, Description - data['Name'] = package.name - data['Version'] = package.version - data['Description'] = format_description(package.description) - # Websites - websites = [str(url) for url in package.urls if url.type == 'website'] - homepage = websites[0] if websites else '' - if homepage == '': - warning("No homepage set, defaulting to ''") - data['Homepage'] = homepage - # Increment Number - data['Inc'] = '' if native else '-{0}'.format(inc) - # Debian Package Format - data['format'] = 'native' if native else 'quilt' - # Package name - data['Package'] = sanitize_package_name(package.name) - # Installation prefix - data['InstallationPrefix'] = installation_prefix - # Resolve dependencies - package.evaluate_conditions(package_conditional_context(ros_distro)) - depends = [ - dep for dep in (package.run_depends + package.buildtool_export_depends) - if dep.evaluated_condition] - build_depends = [ - dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) - if dep.evaluated_condition] - - unresolved_keys = [ - dep for dep in (depends + build_depends + package.replaces + package.conflicts) - if dep.evaluated_condition] - # The installer key is not considered here, but it is checked when the keys are checked before this - resolved_deps = resolve_dependencies(unresolved_keys, os_name, - os_version, ros_distro, - peer_packages + [d.name for d in package.replaces + package.conflicts], - fallback_resolver) - data['Depends'] = sorted( - set(format_depends(depends, resolved_deps)) - ) - data['BuildDepends'] = sorted( - set(format_depends(build_depends, resolved_deps)) - ) - data['Replaces'] = sorted( - set(format_depends(package.replaces, resolved_deps)) - ) - data['Conflicts'] = sorted( - set(format_depends(package.conflicts, resolved_deps)) - ) - - # Build-type specific substitutions. - build_type = package.get_build_type() - if build_type == 'catkin': - pass - elif build_type == 'cmake': - pass - elif build_type == 'ament_cmake': - pass - elif build_type == 'ament_python': - # Don't set the install-scripts flag if it's already set in setup.cfg. - package_path = os.path.abspath(os.path.dirname(package.filename)) - setup_cfg_path = os.path.join(package_path, 'setup.cfg') - data['pass_install_scripts'] = True - if os.path.isfile(setup_cfg_path): - setup_cfg = SafeConfigParser() - setup_cfg.read([setup_cfg_path]) - if ( - setup_cfg.has_option('install', 'install-scripts') or - setup_cfg.has_option('install', 'install_scripts') - ): - data['pass_install_scripts'] = False - else: - error( - "Build type '{}' is not supported by this version of bloom.". - format(build_type), exit=True) - - # Set the distribution - data['Distribution'] = os_version - # Use the time stamp to set the date strings - stamp = datetime.datetime.now(tz.tzlocal()) - data['Date'] = stamp.strftime('%a, %d %b %Y %T %z') - data['YYYY'] = stamp.strftime('%Y') - # Maintainers - maintainers = [] - for m in package.maintainers: - maintainers.append(str(m)) - data['Maintainer'] = maintainers[0] - data['Maintainers'] = ', '.join(maintainers) - # Changelog - changelogs = get_changelogs(package, releaser_history) - if changelogs and package.version not in [x[0] for x in changelogs]: - warning("") - warning("A CHANGELOG.rst was found, but no changelog for this version was found.") - warning("You REALLY should have a entry (even a blank one) for each version of your package.") - warning("") - if not changelogs: - # Ensure at least a minimal changelog - changelogs = [] - if package.version not in [x[0] for x in changelogs]: - changelogs.insert(0, ( - package.version, - get_rfc_2822_date(datetime.datetime.now()), - ' * Autogenerated, no changelog for this version found in CHANGELOG.rst.', - package.maintainers[0].name, - package.maintainers[0].email - )) - bad_changelog = False - # Make sure that the first change log is the version being released - if package.version != changelogs[0][0]: - error("") - error("The version of the first changelog entry '{0}' is not the " - "same as the version being currently released '{1}'." - .format(package.version, changelogs[0][0])) - bad_changelog = True - # Make sure that the current version is the latest in the changelog - for changelog in changelogs: - if parse_version(package.version) < parse_version(changelog[0]): - error("") - error("There is at least one changelog entry, '{0}', which has a " - "newer version than the version of package '{1}' being released, '{2}'." - .format(changelog[0], package.name, package.version)) - bad_changelog = True - if bad_changelog: - error("This is almost certainly by mistake, you should really take a " - "look at the changelogs for the package you are releasing.") - error("") - if not maybe_continue('n', 'Continue anyways'): - sys.exit("User quit.") - data['changelogs'] = changelogs - # Use debhelper version 7 for oneric, otherwise 9 - data['debhelper_version'] = 7 if os_version in ['oneiric'] else 9 - # Summarize dependencies - summarize_dependency_mapping(data, depends, build_depends, resolved_deps) - # Copyright - licenses = [] - separator = '\n' + '=' * 80 + '\n\n' - for l in package.licenses: - if hasattr(l, 'file') and l.file is not None: - license_file = os.path.join(os.path.dirname(package.filename), l.file) - if not os.path.exists(license_file): - error("License file '{}' is not found.". - format(license_file), exit=True) - license_text = open(license_file, 'r').read() - if not license_text.endswith('\n'): - license_text += '\n' - licenses.append(license_text) - data['Copyright'] = separator.join(licenses) - - def convertToUnicode(obj): - if sys.version_info.major == 2: - if isinstance(obj, str): - return unicode(obj.decode('utf8')) - elif isinstance(obj, unicode): - return obj - else: - if isinstance(obj, bytes): - return str(obj.decode('utf8')) - elif isinstance(obj, str): - return obj - if isinstance(obj, list): - for i, val in enumerate(obj): - obj[i] = convertToUnicode(val) - return obj - elif isinstance(obj, type(None)): - return None - elif isinstance(obj, tuple): - obj_tmp = list(obj) - for i, val in enumerate(obj_tmp): - obj_tmp[i] = convertToUnicode(obj_tmp[i]) - return tuple(obj_tmp) - elif isinstance(obj, int): - return obj - raise RuntimeError('need to deal with type %s' % (str(type(obj)))) - - for item in data.items(): - data[item[0]] = convertToUnicode(item[1]) - - return data - - -def __process_template_folder(path, subs): - items = os.listdir(path) - processed_items = [] - for item in list(items): - item = os.path.abspath(os.path.join(path, item)) - if os.path.basename(item) in ['.', '..', '.git', '.svn']: - continue - if os.path.isdir(item): - sub_items = __process_template_folder(item, subs) - processed_items.extend([os.path.join(item, s) for s in sub_items]) - if not item.endswith(TEMPLATE_EXTENSION): - continue - with open(item, 'r') as f: - template = f.read() - # Remove extension - template_path = item[:-len(TEMPLATE_EXTENSION)] - # Expand template - info("Expanding '{0}' -> '{1}'".format( - os.path.relpath(item), - os.path.relpath(template_path))) - result = em.expand(template, **subs) - # Don't write an empty file - if len(result) == 0 and \ - os.path.basename(template_path) in ['copyright']: - processed_items.append(item) - continue - # Write the result - with io.open(template_path, 'w', encoding='utf-8') as f: - if sys.version_info.major == 2: - result = result.decode('utf-8') - f.write(result) - # Copy the permissions - shutil.copymode(item, template_path) - processed_items.append(item) - return processed_items - - -def process_template_files(path, subs, package_system): - info(fmt("@!@{bf}==>@| In place processing templates files in '" + package_system + "' folder.")) - dir_path = os.path.join(path, package_system) - if not os.path.exists(dir_path): - sys.exit("No {0} directory found at '{1}', cannot process templates." - .format(package_system, dir_path)) - return __process_template_folder(dir_path, subs) - - -def debianize_string(value): - markup_remover = re.compile(r'<.*?>') - value = markup_remover.sub('', value) - value = re.sub('\s+', ' ', value) - value = value.strip() - return value - - -def sanitize_package_name(name): - return name.replace('_', '-') - - class DebianGenerator(PackageSystemGenerator): title = 'debian' package_system = 'debian' @@ -536,163 +175,22 @@ def handle_arguments(self, args): ret = PackageSystemGenerator.handle_arguments(self, args) return ret - def _check_all_keys_are_valid(self, peer_packages, rosdistro): - keys_to_resolve = [] - key_to_packages_which_depends_on = collections.defaultdict(list) - keys_to_ignore = set() - for package in self.packages.values(): - package.evaluate_conditions(package_conditional_context(rosdistro)) - depends = [ - dep for dep in (package.run_depends + package.buildtool_export_depends) - if dep.evaluated_condition] - build_depends = [ - dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) - if dep.evaluated_condition] - unresolved_keys = [ - dep for dep in (depends + build_depends + package.replaces + package.conflicts) - if dep.evaluated_condition] - keys_to_ignore = { - dep for dep in keys_to_ignore.union(package.replaces + package.conflicts) - if dep.evaluated_condition} - keys = [d.name for d in unresolved_keys] - keys_to_resolve.extend(keys) - for key in keys: - key_to_packages_which_depends_on[key].append(package.name) - - os_name = self.os_name - rosdistro = self.rosdistro - all_keys_valid = True - for key in sorted(set(keys_to_resolve)): - for os_version in self.distros: - try: - extended_peer_packages = peer_packages + [d.name for d in keys_to_ignore] - rule, installer_key, default_installer_key = \ - resolve_rosdep_key(key, os_name, os_version, rosdistro, extended_peer_packages, - retry=False) - if rule is None: - continue - if installer_key != default_installer_key: - error("Key '{0}' resolved to '{1}' with installer '{2}', " - "which does not match the default installer '{3}'." - .format(key, rule, installer_key, default_installer_key)) - self.exit( - "The {0} generator does not support dependencies " - "which are installed with the '{1}' installer." - .format(self.package_system, installer_key), - returncode=code.GENERATOR_INVALID_INSTALLER_KEY) - except (GeneratorError, RuntimeError) as e: - print(fmt("Failed to resolve @{cf}@!{key}@| on @{bf}{os_name}@|:@{cf}@!{os_version}@| with: {e}") - .format(**locals())) - print(fmt("@{cf}@!{0}@| is depended on by these packages: ").format(key) + - str(list(set(key_to_packages_which_depends_on[key])))) - print(fmt("@{kf}@!<== @{rf}@!Failed@|")) - all_keys_valid = False - return all_keys_valid - def pre_modify(self): - info("\nPre-verifying {0} dependency keys...".format(self.package_system)) - # Run rosdep update is needed - if not self.has_run_rosdep: - self.update_rosdep() - - peer_packages = [p.name for p in self.packages.values()] - - while not self._check_all_keys_are_valid(peer_packages, self.rosdistro): - error("Some of the dependencies for packages in this repository could not be resolved by rosdep.") - error("You can try to address the issues which appear above and try again if you wish.") - try: - if not maybe_continue(msg="Would you like to try again?"): - error("User aborted after rosdep keys were not resolved.") - sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) - except (KeyboardInterrupt, EOFError): - error("\nUser quit.", exit=True) - update_rosdep() - invalidate_view_cache() - - info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") - - def place_template_files(self, build_type, dir_path=None): - # Create/Clean the package system folder - if dir_path is None: - dir_path = os.path.join(".", self.package_system) - if os.path.exists(dir_path): - if self.interactive: - warning("{0} directory exists: {1}".format(self.package_system, dir_path)) - warning("Do you wish to overwrite it?") - if not maybe_continue('y'): - error("Answered no to continue, aborting.", exit=True) - elif 'BLOOM_CLEAR_DEBIAN_ON_GENERATION' in os.environ: - warning("Overwriting {0} directory: {1}".format(self.package_system, dir_path)) - execute_command('git rm -rf ' + dir_path) - execute_command('git commit -m "Clearing previous {0} folder"' - .format(self.package_system)) - if os.path.exists(dir_path): - shutil.rmtree(dir_path) - else: - warning("Not overwriting {0} directory.".format(self.package_system)) - # Use generic place template files command - place_template_files('.', build_type, self.package_system, gbp=True) - # Commit results - execute_command('git add ' + dir_path) - _, has_files, _ = execute_command('git diff --cached --name-only', return_io=True) - if has_files: - execute_command('git commit -m "Placing {0} template files"'.format(self.package_system)) - - def get_releaser_history(self): - # Assumes that this is called in the target branch - patches_branch = 'patches/' + get_current_branch() - raw = show(patches_branch, 'releaser_history.json') - return None if raw is None else json.loads(raw) - - def set_releaser_history(self, history): - # Assumes that this is called in the target branch - patches_branch = 'patches/' + get_current_branch() - debug("Writing release history to '{0}' branch".format(patches_branch)) - with inbranch(patches_branch): - with open('releaser_history.json', 'w') as f: - f.write(json.dumps(history)) - execute_command('git add releaser_history.json') - if has_changes(): - execute_command('git commit -m "Store releaser history"') - - @staticmethod - def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): - """ - This should be a staticmethod since we will use it when we call - `generate_substitutions_from_package` in `get_subs` - Notice that os_name, os_version, ros_distro maybe useful when we - want to add new resolver in the future - """ - if key in peer_packages: - return [sanitize_package_name(key)] - return default_fallback_resolver(key, peer_packages) - - def get_subs(self, package, os_distro, releaser_history=None): - # This is the common part, then the certain generator will add its specic content - return generate_substitutions_from_package( - package, - self.os_name, - os_distro, - self.rosdistro, - self.install_prefix, - self.inc, - [p.name for p in self.packages.values()], - releaser_history=releaser_history, - fallback_resolver=self.missing_dep_resolver - ) - - # There we need to add post subs action and different commit action - def generate_package(self, package, os_distro): - info("Generating {0} for {1}...".format(self.package_system, os_distro)) + error_msg = ''.join([ + "Some of the dependencies for packages in this repository could not be resolved by rosdep.\n", + "You can try to address the issues which appear above and try again if you wish." + ]) + PackageSystemGenerator._pre_modify(self, error_msg) + + def generate_package(self, package, os_version): + info("Generating {0} for {1}...".format(self.package_system, os_version)) # Try to retrieve the releaser_history releaser_history = self.get_releaser_history() # Generate substitution values - subs = self.get_subs(package, os_distro, releaser_history) + subs = self.get_subs(package, os_version, format_description, format_depends, releaser_history) # Use subs to create and store releaser history releaser_history = [(v, (n, e)) for v, _, _, n, e in subs['changelogs']] self.set_releaser_history(dict(releaser_history)) - # Handle gbp.conf - subs['release_tag'] = self.get_release_tag(subs) # Template files template_files = process_template_files(".", subs, self.package_system) # Remove any residual template files @@ -701,13 +199,85 @@ def generate_package(self, package, os_distro): execute_command('git add {0}'.format(self.package_system)) # Commit changes execute_command('git commit -m "Generated {0} files for {1}"' - .format(self.package_system, os_distro)) + .format(self.package_system, os_version)) # Return the subs for other use return subs - def get_release_tag(self, data): - return 'release/{0}/{1}-{2}'.format(data['Name'], data['Version'], - self.inc) + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + # Use the time stamp to set the date strings + stamp = datetime.datetime.now(tz.tzlocal()) + subs['Date'] = stamp.strftime('%a, %d %b %Y %T %z') + subs['YYYY'] = stamp.strftime('%Y') + + # Changelog + changelogs = get_changelogs(package, releaser_history) + if changelogs and package.version not in [x[0] for x in changelogs]: + warning("") + warning("A CHANGELOG.rst was found, but no changelog for this version was found.") + warning("You REALLY should have a entry (even a blank one) for each version of your package.") + warning("") + if not changelogs: + # Ensure at least a minimal changelog + changelogs = [] + if package.version not in [x[0] for x in changelogs]: + changelogs.insert(0, ( + package.version, + get_rfc_2822_date(datetime.datetime.now()), + ' * Autogenerated, no changelog for this version found in CHANGELOG.rst.', + package.maintainers[0].name, + package.maintainers[0].email + )) + bad_changelog = False + # Make sure that the first change log is the version being released + if package.version != changelogs[0][0]: + error("") + error("The version of the first changelog entry '{0}' is not the " + "same as the version being currently released '{1}'." + .format(package.version, changelogs[0][0])) + bad_changelog = True + # Make sure that the current version is the latest in the changelog + for changelog in changelogs: + if parse_version(package.version) < parse_version(changelog[0]): + error("") + error("There is at least one changelog entry, '{0}', which has a " + "newer version than the version of package '{1}' being released, '{2}'." + .format(changelog[0], package.name, package.version)) + bad_changelog = True + if bad_changelog: + error("This is almost certainly by mistake, you should really take a " + "look at the changelogs for the package you are releasing.") + error("") + if not maybe_continue('n', 'Continue anyways'): + sys.exit("User quit.") + subs['changelogs'] = changelogs + + # Use debhelper version 7 for oneric, otherwise 9 + subs['debhelper_version'] = 7 if subs['Distribution'] in ['oneiric'] else 9 + + # Copyright + licenses = [] + separator = '\n' + '=' * 80 + '\n\n' + for l in package.licenses: + if hasattr(l, 'file') and l.file is not None: + license_file = os.path.join(os.path.dirname(package.filename), l.file) + if not os.path.exists(license_file): + error("License file '{}' is not found.". + format(license_file), exit=True) + license_text = open(license_file, 'r').read() + if not license_text.endswith('\n'): + license_text += '\n' + licenses.append(license_text) + subs['Copyright'] = separator.join(licenses) + + # Handle gbp.conf + release_tag = 'release/{0}/{1}-{2}'.format(subs['Name'], subs['Version'], subs['Inc']) + subs['release_tag'] = release_tag + + # Debian Package Format + subs['format'] = 'quilt' + + return subs def generate_tag_name(self, subs): tag_name = '{Package}_{Version}{Inc}_{Distribution}' diff --git a/bloom/generators/rosdebian.py b/bloom/generators/rosdebian.py index fa4f9df6..cec4bea8 100644 --- a/bloom/generators/rosdebian.py +++ b/bloom/generators/rosdebian.py @@ -34,11 +34,13 @@ from __future__ import print_function from bloom.generators.common import default_fallback_resolver +from bloom.generators.common import generate_substitutions_from_package +from bloom.generators.common import sanitize_package_name -from bloom.generators.debian.generator import sanitize_package_name from bloom.generators.debian import DebianGenerator -from bloom.generators.debian.generator import generate_substitutions_from_package +from bloom.generators.debian import format_description +from bloom.generators.debian import format_depends from bloom.generators.debian.generate_cmd import main as debian_main from bloom.generators.debian.generate_cmd import prepare_arguments @@ -77,26 +79,27 @@ def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): return [sanitize_package_name(rosify_package_name(key, ros_distro))] return default_fallback_resolver(key, peer_packages) - def get_subs(self, package, debian_distro, releaser_history, native=False): - subs = DebianGenerator.get_subs(self, package, debian_distro, releaser_history) - subs['Package'] = rosify_package_name(subs['Package'], self.rosdistro) + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + subs = DebianGenerator.get_subs_hook(subs, package, releaser_history) + subs['Package'] = rosify_package_name(subs['Package'], rosdistro) # ROS 2 specific bloom extensions. ros2_distros = [ name for name, values in get_index().distributions.items() if values.get('distribution_type') == 'ros2'] - if self.rosdistro in ros2_distros: + if rosdistro in ros2_distros: # Add ros-workspace package as a dependency to any package other # than ros_workspace and its dependencies. if package.name not in ['ament_cmake_core', 'ament_package', 'ros_workspace']: - workspace_pkg_name = rosify_package_name('ros-workspace', self.rosdistro) + workspace_pkg_name = rosify_package_name('ros-workspace', rosdistro) subs['BuildDepends'].append(workspace_pkg_name) subs['Depends'].append(workspace_pkg_name) # Add packages necessary to build vendor typesupport for rosidl_interface_packages to their # build dependencies. - if self.rosdistro in ros2_distros and \ - self.rosdistro not in ('r2b2', 'r2b3', 'ardent') and \ + if rosdistro in ros2_distros and \ + rosdistro not in ('r2b2', 'r2b3', 'ardent') and \ 'rosidl_interface_packages' in [p.name for p in package.member_of_groups]: ROS2_VENDOR_TYPESUPPORT_DEPENDENCIES = [ 'rmw-connext-cpp', @@ -110,7 +113,8 @@ def get_subs(self, package, debian_distro, releaser_history, native=False): ] subs['BuildDepends'] += [ - rosify_package_name(name, self.rosdistro) for name in ROS2_VENDOR_TYPESUPPORT_DEPENDENCIES] + rosify_package_name(name, rosdistro) for name in ROS2_VENDOR_TYPESUPPORT_DEPENDENCIES] + return subs def generate_branching_arguments(self, package, branch): @@ -138,10 +142,13 @@ def get_subs(pkg, os_name, os_version, ros_distro, native): os_name, os_version, ros_distro, + format_description, + format_depends, RosDebianGenerator.default_install_prefix + ros_distro, - native=native ) - subs['Package'] = rosify_package_name(subs['Package'], ros_distro) + subs = RosDebianGenerator.get_subs_hook(subs, pkg, ros_distro) + # Debian Package Format + subs['format'] = 'native' if native else 'quilt' return subs diff --git a/test/system_tests/test_catkin_release.py b/test/system_tests/test_catkin_release.py index d30921e6..3c6caced 100644 --- a/test/system_tests/test_catkin_release.py +++ b/test/system_tests/test_catkin_release.py @@ -31,7 +31,7 @@ from bloom.commands.git.patch import import_cmd from bloom.commands.git.patch import remove_cmd -from bloom.generators.debian.generator import sanitize_package_name +from bloom.generators.common import sanitize_package_name def create_upstream_repository(packages, directory=None, format_versions=None): diff --git a/test/unit_tests/test_generators/test_debian/test_generator.py b/test/unit_tests/test_generators/test_debian/test_generator.py index d0eb2723..317699bf 100644 --- a/test/unit_tests/test_generators/test_debian/test_generator.py +++ b/test/unit_tests/test_generators/test_debian/test_generator.py @@ -2,7 +2,7 @@ from ....utils.common import redirected_stdio -from bloom.generators.debian.generator import em +from bloom.generators.common import em from bloom.generators.debian.generator import get_changelogs from bloom.generators.debian.generator import format_description From 1f5a705105c3a9246dea97abc025fc10bd2afa51 Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Sat, 15 Jun 2019 13:20:20 +0800 Subject: [PATCH 09/23] use pacakge manager as base generator's name --- bloom/generators/common.py | 70 ++++++++++++------------- bloom/generators/debian/generate_cmd.py | 10 ++-- bloom/generators/debian/generator.py | 22 ++++---- bloom/generators/rosdebian.py | 4 +- 4 files changed, 53 insertions(+), 53 deletions(-) diff --git a/bloom/generators/common.py b/bloom/generators/common.py index 445428e1..97657552 100644 --- a/bloom/generators/common.py +++ b/bloom/generators/common.py @@ -422,13 +422,13 @@ def convertToUnicode(obj): raise RuntimeError('need to deal with type %s' % (str(type(obj)))) -def place_template_files(path, build_type, package_system, gbp=False): - info(fmt("@!@{bf}==>@| Placing templates files in the '" + package_system + "' folder.")) - dir_path = os.path.join(path, package_system) +def place_template_files(path, build_type, package_manager, gbp=False): + info(fmt("@!@{bf}==>@| Placing templates files in the '" + package_manager + "' folder.")) + dir_path = os.path.join(path, package_manager) if not os.path.exists(dir_path): os.makedirs(dir_path) # Place template files - group = 'bloom.generators.' + package_system + group = 'bloom.generators.' + package_manager templates = os.path.join('templates', build_type) __place_template_folder(group, templates, dir_path, gbp) @@ -493,12 +493,12 @@ def __process_template_folder(path, subs): return processed_items -def process_template_files(path, subs, package_system): - info(fmt("@!@{bf}==>@| In place processing templates files in '" + package_system + "' folder.")) - dir_path = os.path.join(path, package_system) +def process_template_files(path, subs, pacakge_manager): + info(fmt("@!@{bf}==>@| In place processing templates files in '" + pacakge_manager + "' folder.")) + dir_path = os.path.join(path, pacakge_manager) if not os.path.exists(dir_path): sys.exit("No {0} directory found at '{1}', cannot process templates." - .format(package_system, dir_path)) + .format(pacakge_manager, dir_path)) return __process_template_folder(dir_path, subs) @@ -689,8 +689,8 @@ def post_patch(self, branch_name): return 0 -class PackageSystemGenerator(BloomGenerator): - package_system = 'none' +class PackageManagerGenerator(BloomGenerator): + package_manager = 'none' has_run_rosdep = False def prepare_arguments(self, parser): @@ -724,7 +724,7 @@ def get_package_from_branch(self, branch): "from branches with multiple packages in them, use " "the release generator first to split packages into " "individual branches." - .format(self.package_system)) + .format(self.package_manager)) if type(packages) is dict: return list(packages.values())[0] @@ -762,7 +762,7 @@ def handle_arguments(self, args): self.tag_names = {} self.names = [] self.branch_args = [] - self.package_system_branches = [] + self.package_manager_branches = [] for branch in self.branches: package = self.get_package_from_branch(branch) if package is None: @@ -771,8 +771,8 @@ def handle_arguments(self, args): self.packages[package.name] = package self.names.append(package.name) args = self.generate_branching_arguments(package, branch) - # First branch is package_system/[/] - self.package_system_branches.append(args[0][0]) + # First branch is package_manager/[/] + self.package_manager_branches.append(args[0][0]) self.branch_args.extend(args) def summarize(self): @@ -829,7 +829,7 @@ def _check_all_keys_are_valid(self, peer_packages, rosdistro): self.exit( "The {0} generator does not support dependencies " "which are installed with the '{1}' installer." - .format(self.package_system, installer_key), + .format(self.package_manager, installer_key), returncode=code.GENERATOR_INVALID_INSTALLER_KEY) except (GeneratorError, RuntimeError) as e: print(fmt("Failed to resolve @{cf}@!{key}@| on @{bf}{os_name}@|:@{cf}@!{os_version}@| with: {e}") @@ -841,7 +841,7 @@ def _check_all_keys_are_valid(self, peer_packages, rosdistro): return all_keys_valid def _pre_modify(self, key_unvalid_error_msg): - info("\nPre-verifying {0} dependency keys...".format(self.package_system)) + info("\nPre-verifying {0} dependency keys...".format(self.package_manager)) # Run rosdep update is needed if not self.has_run_rosdep: self.update_rosdep() @@ -862,7 +862,7 @@ def _pre_modify(self, key_unvalid_error_msg): info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") def pre_branch(self, destination, source): - if destination in self.package_system_branches: + if destination in self.package_manager_branches: return # Run rosdep update is needed if not self.has_run_rosdep: @@ -889,9 +889,9 @@ def post_rebase(self, destination): # Retrieve the package package = self.packages[name] # Handle differently if this is a package system vs distro branch - if destination in self.package_system_branches: + if destination in self.package_manager_branches: info("Placing {0} template files into '{1}' branch." - .format(self.package_system, destination)) + .format(self.package_manager, destination)) # Then this is a package system branch # Place the raw template files self.place_template_files(package.get_build_type()) @@ -918,7 +918,7 @@ def post_rebase(self, destination): set_patch_config(patches_branch, config) def post_patch(self, destination, color='bluef'): - if destination in self.package_system_branches: + if destination in self.package_manager_branches: return # Tag after patches have been applied with inbranch(destination): @@ -943,7 +943,7 @@ def post_patch(self, destination, color='bluef'): info( ansi(color) + "#### " + ansi('greenf') + "Successfully" + ansi(color) + " generated '" + ansi('boldon') + distro + - ansi('boldoff') + "' {0} for package".format(self.package_system) + + ansi('boldoff') + "' {0} for package".format(self.package_manager) + " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + " at version '" + ansi('boldon') + package.version + "-" + str(self.inc) + ansi('boldoff') + "'" + @@ -954,14 +954,14 @@ def post_patch(self, destination, color='bluef'): def store_original_config(self, config, patches_branch): with inbranch(patches_branch): - with open('{0}.store'.format(self.package_system), 'w+') as f: + with open('{0}.store'.format(self.package_manager), 'w+') as f: f.write(json.dumps(config)) - execute_command('git add {0}.store'.format(self.package_system)) + execute_command('git add {0}.store'.format(self.package_manager)) if has_changes(): execute_command('git commit -m "Store original patch config"') def load_original_config(self, patches_branch): - config_store = show(patches_branch, '{0}.store'.format(self.package_system)) + config_store = show(patches_branch, '{0}.store'.format(self.package_manager)) if config_store is None: return config_store return json.loads(config_store) @@ -998,29 +998,29 @@ def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): def place_template_files(self, build_type, dir_path=None): # Create/Clean the package system folder if dir_path is None: - dir_path = os.path.join(".", self.package_system) + dir_path = os.path.join(".", self.package_manager) if os.path.exists(dir_path): if self.interactive: - warning("{0} directory exists: {1}".format(self.package_system, dir_path)) + warning("{0} directory exists: {1}".format(self.package_manager, dir_path)) warning("Do you wish to overwrite it?") if not maybe_continue('y'): error("Answered no to continue, aborting.", exit=True) elif 'BLOOM_CLEAR_TEMPLATE_ON_GENERATION' in os.environ: - warning("Overwriting {0} directory: {1}".format(self.package_system, dir_path)) + warning("Overwriting {0} directory: {1}".format(self.package_manager, dir_path)) execute_command('git rm -rf ' + dir_path) execute_command('git commit -m "Clearing previous {0} folder"' - .format(self.package_system)) + .format(self.package_manager)) if os.path.exists(dir_path): shutil.rmtree(dir_path) else: - warning("Not overwriting {0} directory.".format(self.package_system)) + warning("Not overwriting {0} directory.".format(self.package_manager)) # Use generic place template files command - place_template_files('.', build_type, self.package_system, gbp=True) + place_template_files('.', build_type, self.package_manager, gbp=True) # Commit results execute_command('git add ' + dir_path) _, has_files, _ = execute_command('git diff --cached --name-only', return_io=True) if has_files: - execute_command('git commit -m "Placing {0} template files"'.format(self.package_system)) + execute_command('git commit -m "Placing {0} template files"'.format(self.package_manager)) def get_subs(self, package, os_version, format_description, format_depends, releaser_history=None): # This is the common part for generate templacte substitute, then successor of @@ -1047,7 +1047,7 @@ def summarize_package(self, package, distro, color='bluef'): info(ansi(color) + "\n####" + ansi('reset'), use_prefix=False) info( ansi(color) + "#### Generating '" + ansi('boldon') + distro + - ansi('boldoff') + "' {0} for package".format(self.package_system) + + ansi('boldoff') + "' {0} for package".format(self.package_manager) + " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + " at version '" + ansi('boldon') + package.version + "-" + str(self.inc) + ansi('boldoff') + "'" + @@ -1067,19 +1067,19 @@ def generate_branching_arguments(self, package, branch): """ n = package.name # package branch - package_branch = self.package_system + '/' + n + package_branch = self.package_manager + '/' + n # Branch first to the package branch args = [[package_branch, branch, False]] # Then for each os distro, branch from the base package branch args.extend([ - [self.package_system + '/' + d + '/' + n, package_branch, False] + [self.package_manager + '/' + d + '/' + n, package_branch, False] for d in self.distros ]) return args def generate_package(self, package, os_version): """ - Assume we have the templactes file in directory + Assume we have the templactes file in directory The overriten function should generate the package, including 1. use the result of get_subs to replace template content 2. set the newest release history diff --git a/bloom/generators/debian/generate_cmd.py b/bloom/generators/debian/generate_cmd.py index dd96045c..ee25ffad 100644 --- a/bloom/generators/debian/generate_cmd.py +++ b/bloom/generators/debian/generate_cmd.py @@ -129,21 +129,21 @@ def main(args=None, get_subs_fn=None): fmt("Generating debs for @{cf}%s:%s@| for package(s) %s" % (os_name, os_version, [p.name for p in pkgs_dict.values()]))) - package_system = DebianGenerator.package_system + package_manager = DebianGenerator.package_manager for path, pkg in pkgs_dict.items(): template_files = None try: subs = get_subs_fn(pkg, os_name, os_version, ros_distro, args.native) if _place_template_files: # Place template files - place_template_files(path, pkg.get_build_type(), package_system) + place_template_files(path, pkg.get_build_type(), package_manager) if _process_template_files: # Just process existing template files - template_files = process_template_files(path, subs, package_system) + template_files = process_template_files(path, subs, package_manager) if not _place_template_files and not _process_template_files: # If neither, do both - place_template_files(path, pkg.get_build_type(), package_system) - template_files = process_template_files(path, subs, package_system) + place_template_files(path, pkg.get_build_type(), package_manager) + template_files = process_template_files(path, subs, package_manager) if template_files is not None: for template_file in template_files: os.remove(os.path.normpath(template_file)) diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index fbf2b24f..ea584547 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -41,7 +41,7 @@ from dateutil import tz from pkg_resources import parse_version -from bloom.generators.common import PackageSystemGenerator +from bloom.generators.common import PackageManagerGenerator from bloom.generators.common import process_template_files from bloom.logging import debug @@ -154,9 +154,9 @@ def get_changelogs(package, releaser_history=None): return [] -class DebianGenerator(PackageSystemGenerator): +class DebianGenerator(PackageManagerGenerator): title = 'debian' - package_system = 'debian' + package_manager = 'debian' description = "Generates debians from the catkin meta data" default_install_prefix = '/usr' rosdistro = os.environ.get('ROS_DISTRO', 'indigo') @@ -168,11 +168,11 @@ def prepare_arguments(self, parser): add('--os-not-required', default=False, action="store_true", help="Do not error if this os is not in the platforms " "list for rosdistro") - return PackageSystemGenerator.prepare_arguments(self, parser) + return PackageManagerGenerator.prepare_arguments(self, parser) def handle_arguments(self, args): self.os_not_required = args.os_not_required - ret = PackageSystemGenerator.handle_arguments(self, args) + ret = PackageManagerGenerator.handle_arguments(self, args) return ret def pre_modify(self): @@ -180,10 +180,10 @@ def pre_modify(self): "Some of the dependencies for packages in this repository could not be resolved by rosdep.\n", "You can try to address the issues which appear above and try again if you wish." ]) - PackageSystemGenerator._pre_modify(self, error_msg) + PackageManagerGenerator._pre_modify(self, error_msg) def generate_package(self, package, os_version): - info("Generating {0} for {1}...".format(self.package_system, os_version)) + info("Generating {0} for {1}...".format(self.package_manager, os_version)) # Try to retrieve the releaser_history releaser_history = self.get_releaser_history() # Generate substitution values @@ -192,14 +192,14 @@ def generate_package(self, package, os_version): releaser_history = [(v, (n, e)) for v, _, _, n, e in subs['changelogs']] self.set_releaser_history(dict(releaser_history)) # Template files - template_files = process_template_files(".", subs, self.package_system) + template_files = process_template_files(".", subs, self.package_manager) # Remove any residual template files execute_command('git rm -rf ' + ' '.join("'{}'".format(t) for t in template_files)) # Add changes to the package system folder - execute_command('git add {0}'.format(self.package_system)) + execute_command('git add {0}'.format(self.package_manager)) # Commit changes execute_command('git commit -m "Generated {0} files for {1}"' - .format(self.package_system, os_version)) + .format(self.package_manager, os_version)) # Return the subs for other use return subs @@ -281,5 +281,5 @@ def get_subs_hook(subs, package, rosdistro, releaser_history=None): def generate_tag_name(self, subs): tag_name = '{Package}_{Version}{Inc}_{Distribution}' - tag_name = self.package_system + '/' + tag_name.format(**subs) + tag_name = self.package_manager + '/' + tag_name.format(**subs) return tag_name diff --git a/bloom/generators/rosdebian.py b/bloom/generators/rosdebian.py index cec4bea8..2df405db 100644 --- a/bloom/generators/rosdebian.py +++ b/bloom/generators/rosdebian.py @@ -118,11 +118,11 @@ def get_subs_hook(subs, package, rosdistro, releaser_history=None): return subs def generate_branching_arguments(self, package, branch): - package_branch = self.package_system + '/' + self.rosdistro + '/' + package.name + package_branch = self.package_manager + '/' + self.rosdistro + '/' + package.name args = [[package_branch, branch, False]] n, r, b, ds = package.name, self.rosdistro, package_branch, self.distros args.extend([ - [self.package_system + '/' + r + '/' + d + '/' + n, b, False] for d in ds + [self.package_manager + '/' + r + '/' + d + '/' + n, b, False] for d in ds ]) return args From 0e1c8c05e28058fccfdc39bffecbb59a416d6e51 Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Sat, 15 Jun 2019 21:34:07 +0800 Subject: [PATCH 10/23] refactor RpmGenerator The major different part of RPM and Debian's generator as follow: 1. rpm don't have native option in local generate_cmd related command 2. rpm and debian's Date, changelogs, and License format are different 3. rpm need NoArch substitution 4. rpm has its description and depends specification And I also update the utils function like __process_template_folder, __place_template_folder, and place_tempalte_folder, the newest code in debian should also fit rpm generator's logic --- bloom/generators/common.py | 3 +- bloom/generators/debian/__init__.py | 4 +- bloom/generators/debian/generator.py | 3 +- bloom/generators/rosrpm.py | 52 +- bloom/generators/rpm/__init__.py | 5 +- bloom/generators/rpm/generate_cmd.py | 31 +- bloom/generators/rpm/generator.py | 772 ++---------------- .../rpm/templates/catkin/template.spec.em | 2 +- .../rpm/templates/cmake/template.spec.em | 2 +- 9 files changed, 123 insertions(+), 751 deletions(-) diff --git a/bloom/generators/common.py b/bloom/generators/common.py index 97657552..97ddb447 100644 --- a/bloom/generators/common.py +++ b/bloom/generators/common.py @@ -692,6 +692,8 @@ def post_patch(self, branch_name): class PackageManagerGenerator(BloomGenerator): package_manager = 'none' has_run_rosdep = False + default_install_prefix = '/usr' + rosdistro = os.environ.get('ROS_DISTRO', 'indigo') def prepare_arguments(self, parser): # The common command line arguments for every package system @@ -744,7 +746,6 @@ def get_default_distros(self): def handle_arguments(self, args): self.interactive = args.interactive self.inc = args.inc - self.os_name = args.os_name self.distros = args.distros if self.distros in [None, []]: self.get_default_distros() diff --git a/bloom/generators/debian/__init__.py b/bloom/generators/debian/__init__.py index daa4096a..a1f3f8c7 100755 --- a/bloom/generators/debian/__init__.py +++ b/bloom/generators/debian/__init__.py @@ -1,3 +1,5 @@ -from .generator import DebianGenerator, format_description, format_depends +from .generator import DebianGenerator +from .generator import format_depends +from .generator import format_description __all__ = ['DebianGenerator', 'format_description', 'format_depends'] diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index ea584547..38d4897d 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -158,8 +158,6 @@ class DebianGenerator(PackageManagerGenerator): title = 'debian' package_manager = 'debian' description = "Generates debians from the catkin meta data" - default_install_prefix = '/usr' - rosdistro = os.environ.get('ROS_DISTRO', 'indigo') def prepare_arguments(self, parser): add = parser.add_argument @@ -171,6 +169,7 @@ def prepare_arguments(self, parser): return PackageManagerGenerator.prepare_arguments(self, parser) def handle_arguments(self, args): + self.os_name = args.os_name self.os_not_required = args.os_not_required ret = PackageManagerGenerator.handle_arguments(self, args) return ret diff --git a/bloom/generators/rosrpm.py b/bloom/generators/rosrpm.py index ad8ae70f..f672ff4c 100644 --- a/bloom/generators/rosrpm.py +++ b/bloom/generators/rosrpm.py @@ -34,11 +34,13 @@ from __future__ import print_function from bloom.generators.common import default_fallback_resolver - -from bloom.generators.rpm.generator import sanitize_package_name +from bloom.generators.common import generate_substitutions_from_package +from bloom.generators.common import sanitize_package_name from bloom.generators.rpm import RpmGenerator -from bloom.generators.rpm.generator import generate_substitutions_from_package +from bloom.generators.rpm import format_depends +from bloom.generators.rpm import format_description + from bloom.generators.rpm.generate_cmd import main as rpm_main from bloom.generators.rpm.generate_cmd import prepare_arguments @@ -69,37 +71,30 @@ def summarize(self): info("Releasing for rosdistro: " + self.rosdistro) return ret - def get_subs(self, package, rpm_distro, releaser_history): - def fallback_resolver(key, peer_packages, rosdistro=self.rosdistro): - if key in peer_packages: - return [sanitize_package_name(rosify_package_name(key, rosdistro))] - return default_fallback_resolver(key, peer_packages) - subs = generate_substitutions_from_package( - package, - self.os_name, - rpm_distro, - self.rosdistro, - self.install_prefix, - self.rpm_inc, - [p.name for p in self.packages.values()], - releaser_history=releaser_history, - fallback_resolver=fallback_resolver - ) - subs['Package'] = rosify_package_name(subs['Package'], self.rosdistro) + @staticmethod + def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): + if key in peer_packages: + return [sanitize_package_name(rosify_package_name(key, ros_distro))] + return default_fallback_resolver(key, peer_packages) + + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + subs = RpmGenerator.get_subs_hook(subs, package, releaser_history) + subs['Package'] = rosify_package_name(subs['Package'], rosdistro) return subs def generate_branching_arguments(self, package, branch): - rpm_branch = 'rpm/' + self.rosdistro + '/' + package.name - args = [[rpm_branch, branch, False]] - n, r, b, ds = package.name, self.rosdistro, rpm_branch, self.distros + package_branch = self.package_manager + '/' + self.rosdistro + '/' + package.name + args = [[package_branch, branch, False]] + n, r, b, ds = package.name, self.rosdistro, package_branch, self.distros args.extend([ - ['rpm/' + r + '/' + d + '/' + n, b, False] for d in ds + [self.package_manager + '/' + r + '/' + d + '/' + n, b, False] for d in ds ]) return args def get_release_tag(self, data): return 'release/{0}/{1}/{2}-{3}'\ - .format(self.rosdistro, data['Name'], data['Version'], self.rpm_inc) + .format(self.rosdistro, data['Name'], data['Version'], self.inc) def rosify_package_name(name, rosdistro): @@ -112,9 +107,12 @@ def get_subs(pkg, os_name, os_version, ros_distro): pkg, os_name, os_version, - ros_distro + ros_distro, + format_description, + format_depends, + RosRpmGenerator.default_install_prefix + ros_distro, ) - subs['Package'] = rosify_package_name(subs['Package'], ros_distro) + subs = RosRpmGenerator.get_subs_hook(subs, pkg, ros_distro) return subs diff --git a/bloom/generators/rpm/__init__.py b/bloom/generators/rpm/__init__.py index ffa3421a..0632bc9f 100755 --- a/bloom/generators/rpm/__init__.py +++ b/bloom/generators/rpm/__init__.py @@ -1,4 +1,5 @@ from .generator import RpmGenerator -from .generator import sanitize_package_name +from .generator import format_depends +from .generator import format_description -__all__ = ['RpmGenerator', 'sanitize_package_name'] +__all__ = ['RpmGenerator', 'format_description', 'format_depends'] diff --git a/bloom/generators/rpm/generate_cmd.py b/bloom/generators/rpm/generate_cmd.py index 4d91e88b..cf01501d 100644 --- a/bloom/generators/rpm/generate_cmd.py +++ b/bloom/generators/rpm/generate_cmd.py @@ -42,9 +42,13 @@ from bloom.logging import fmt from bloom.logging import info -from bloom.generators.rpm.generator import generate_substitutions_from_package -from bloom.generators.rpm.generator import place_template_files -from bloom.generators.rpm.generator import process_template_files +from bloom.generators.common import generate_substitutions_from_package +from bloom.generators.common import place_template_files +from bloom.generators.common import process_template_files + +from bloom.generators.rpm.generator import RpmGenerator +from bloom.generators.rpm.generator import format_description +from bloom.generators.rpm.generator import format_depends from bloom.util import get_distro_list_prompt @@ -79,12 +83,18 @@ def prepare_arguments(parser): def get_subs(pkg, os_name, os_version, ros_distro): - return generate_substitutions_from_package( + # No fallback_resolver provided because peer packages not considered. + subs = generate_substitutions_from_package( pkg, os_name, os_version, - ros_distro - ) + ros_distro, + format_description, + format_depends, + RpmGenerator.default_install_prefix + ) + subs = RpmGenerator.get_subs_hook(subs, pkg, ros_distro) + return subs def main(args=None, get_subs_fn=None): @@ -118,20 +128,21 @@ def main(args=None, get_subs_fn=None): fmt("Generating RPMs for @{cf}%s:%s@| for package(s) %s" % (os_name, os_version, [p.name for p in pkgs_dict.values()]))) + package_manager = RpmGenerator.package_manager for path, pkg in pkgs_dict.items(): template_files = None try: subs = get_subs_fn(pkg, os_name, os_version, ros_distro) if _place_template_files: # Place template files - place_template_files(path, pkg.get_build_type()) + place_template_files(path, pkg.get_build_type(), package_manager) if _process_template_files: # Just process existing template files - template_files = process_template_files(path, subs) + template_files = process_template_files(path, subs, package_manager) if not _place_template_files and not _process_template_files: # If neither, do both - place_template_files(path, pkg.get_build_type()) - template_files = process_template_files(path, subs) + place_template_files(path, pkg.get_build_type(), package_manager) + template_files = process_template_files(path, subs, package_manager) if template_files is not None: for template_file in template_files: os.remove(os.path.normpath(template_file)) diff --git a/bloom/generators/rpm/generator.py b/bloom/generators/rpm/generator.py index 871f7b13..83c80d0b 100644 --- a/bloom/generators/rpm/generator.py +++ b/bloom/generators/rpm/generator.py @@ -32,143 +32,26 @@ from __future__ import print_function -import collections import datetime -import io -import json -import os -import pkg_resources import re -import shutil -import sys -import traceback import textwrap -# Python 2/3 support. -try: - from configparser import SafeConfigParser -except ImportError: - from ConfigParser import SafeConfigParser from dateutil import tz from distutils.version import LooseVersion from time import strptime -from bloom.generators import BloomGenerator -from bloom.generators import GeneratorError -from bloom.generators import resolve_dependencies -from bloom.generators import update_rosdep +from bloom.generators.common import PackageManagerGenerator +from bloom.generators.common import process_template_files -from bloom.generators.common import default_fallback_resolver -from bloom.generators.common import invalidate_view_cache -from bloom.generators.common import package_conditional_context -from bloom.generators.common import resolve_rosdep_key - -from bloom.git import inbranch -from bloom.git import get_branches -from bloom.git import get_commit_hash -from bloom.git import get_current_branch -from bloom.git import has_changes -from bloom.git import show -from bloom.git import tag_exists - -from bloom.logging import ansi -from bloom.logging import debug from bloom.logging import enable_drop_first_log_prefix from bloom.logging import error -from bloom.logging import fmt from bloom.logging import info -from bloom.logging import warning - -from bloom.commands.git.patch.common import get_patch_config -from bloom.commands.git.patch.common import set_patch_config - -from bloom.packages import get_package_data -from bloom.util import code from bloom.util import execute_command -from bloom.util import maybe_continue - -try: - import rosdistro -except ImportError as err: - debug(traceback.format_exc()) - error("rosdistro was not detected, please install it.", exit=True) - -try: - import em -except ImportError: - debug(traceback.format_exc()) - error("empy was not detected, please install it.", exit=True) # Drop the first log prefix for this command enable_drop_first_log_prefix(True) -TEMPLATE_EXTENSION = '.em' - - -def __place_template_folder(group, src, dst, gbp=False): - template_files = pkg_resources.resource_listdir(group, src) - # For each template, place - for template_file in template_files: - template_path = os.path.join(src, template_file) - template_dst = os.path.join(dst, template_file) - if pkg_resources.resource_isdir(group, template_path): - debug("Recursing on folder '{0}'".format(template_path)) - __place_template_folder(group, template_path, template_dst, gbp) - else: - try: - debug("Placing template '{0}'".format(template_path)) - template = pkg_resources.resource_string(group, template_path) - template_abs_path = pkg_resources.resource_filename(group, template_path) - except IOError as err: - error("Failed to load template " - "'{0}': {1}".format(template_file, str(err)), exit=True) - if not os.path.exists(dst): - os.makedirs(dst) - if os.path.exists(template_dst): - debug("Removing existing file '{0}'".format(template_dst)) - os.remove(template_dst) - with open(template_dst, 'w') as f: - if not isinstance(template, str): - template = template.decode('utf-8') - f.write(template) - shutil.copystat(template_abs_path, template_dst) - - -def place_template_files(path, build_type, gbp=False): - info(fmt("@!@{bf}==>@| Placing templates files in the 'rpm' folder.")) - rpm_path = os.path.join(path, 'rpm') - # Create/Clean the rpm folder - if not os.path.exists(rpm_path): - os.makedirs(rpm_path) - # Place template files - group = 'bloom.generators.rpm' - templates = os.path.join('templates', build_type) - __place_template_folder(group, templates, rpm_path, gbp) - - -def summarize_dependency_mapping(data, deps, build_deps, resolved_deps): - if len(deps) == 0 and len(build_deps) == 0: - return - info("Package '" + data['Package'] + "' has dependencies:") - header = " " + ansi('boldoff') + ansi('ulon') + \ - "rosdep key => " + data['Distribution'] + \ - " key" + ansi('reset') - template = " " + ansi('cyanf') + "{0:<20} " + ansi('purplef') + \ - "=> " + ansi('cyanf') + "{1}" + ansi('reset') - if len(deps) != 0: - info(ansi('purplef') + "Run Dependencies:" + - ansi('reset')) - info(header) - for key in [d.name for d in deps]: - info(template.format(key, resolved_deps[key])) - if len(build_deps) != 0: - info(ansi('purplef') + - "Build and Build Tool Dependencies:" + ansi('reset')) - info(header) - for key in [d.name for d in build_deps]: - info(template.format(key, resolved_deps[key])) - def format_depends(depends, resolved_deps): versions = { @@ -193,258 +76,7 @@ def format_depends(depends, resolved_deps): return formatted -def missing_dep_resolver(key, peer_packages): - if key in peer_packages: - return [sanitize_package_name(key)] - return default_fallback_resolver(key, peer_packages) - - -def generate_substitutions_from_package( - package, - os_name, - os_version, - ros_distro, - installation_prefix='/usr', - rpm_inc=0, - peer_packages=None, - releaser_history=None, - fallback_resolver=None -): - peer_packages = peer_packages or [] - data = {} - # Name, Version, Description - data['Name'] = package.name - data['Version'] = package.version - data['Description'] = rpmify_string(package.description) - # License - if not package.licenses or not package.licenses[0]: - error("No license set for package '{0}', aborting.".format(package.name), exit=True) - data['License'] = package.licenses[0] - # Websites - websites = [str(url) for url in package.urls if url.type == 'website'] - data['Homepage'] = websites[0] if websites else '' - if data['Homepage'] == '': - warning("No homepage set") - # RPM Increment Number - data['RPMInc'] = rpm_inc - # Package name - data['Package'] = sanitize_package_name(package.name) - # Installation prefix - data['InstallationPrefix'] = installation_prefix - # Resolve dependencies - package.evaluate_conditions(package_conditional_context(ros_distro)) - depends = [ - dep for dep in (package.run_depends + package.buildtool_export_depends) - if dep.evaluated_condition] - build_depends = [ - dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) - if dep.evaluated_condition] - unresolved_keys = [ - dep for dep in (depends + build_depends + package.replaces + package.conflicts) - if dep.evaluated_condition] - # The installer key is not considered here, but it is checked when the keys are checked before this - resolved_deps = resolve_dependencies(unresolved_keys, os_name, - os_version, ros_distro, - peer_packages + [d.name for d in package.replaces + package.conflicts], - fallback_resolver) - data['Depends'] = sorted( - set(format_depends(depends, resolved_deps)) - ) - data['BuildDepends'] = sorted( - set(format_depends(build_depends, resolved_deps)) - ) - data['Replaces'] = sorted( - set(format_depends(package.replaces, resolved_deps)) - ) - data['Conflicts'] = sorted( - set(format_depends(package.conflicts, resolved_deps)) - ) - - # Build-type specific substitutions. - build_type = package.get_build_type() - if build_type == 'catkin': - pass - elif build_type == 'cmake': - pass - elif build_type == 'ament_cmake': - error( - "Build type '{}' is not supported by this version of bloom.". - format(build_type), exit=True) - elif build_type == 'ament_python': - error( - "Build type '{}' is not supported by this version of bloom.". - format(build_type), exit=True) - # Don't set the install-scripts flag if it's already set in setup.cfg. - package_path = os.path.abspath(os.path.dirname(package.filename)) - setup_cfg_path = os.path.join(package_path, 'setup.cfg') - data['pass_install_scripts'] = True - if os.path.isfile(setup_cfg_path): - setup_cfg = SafeConfigParser() - setup_cfg.read([setup_cfg_path]) - if ( - setup_cfg.has_option('install', 'install-scripts') or - setup_cfg.has_option('install', 'install_scripts') - ): - data['pass_install_scripts'] = False - else: - error( - "Build type '{}' is not supported by this version of bloom.". - format(build_type), exit=True) - - # Set the distribution - data['Distribution'] = os_version - # Use the time stamp to set the date strings - stamp = datetime.datetime.now(tz.tzlocal()) - data['Date'] = stamp.strftime('%a %b %d %Y') - # Maintainers - maintainers = [] - for m in package.maintainers: - maintainers.append(str(m)) - data['Maintainer'] = maintainers[0] - data['Maintainers'] = ', '.join(maintainers) - # Changelog - if releaser_history: - sorted_releaser_history = sorted(releaser_history, - key=lambda k: LooseVersion(k), reverse=True) - sorted_releaser_history = sorted(sorted_releaser_history, - key=lambda k: strptime(releaser_history.get(k)[0], '%a %b %d %Y'), - reverse=True) - changelogs = [(v, releaser_history[v]) for v in sorted_releaser_history] - else: - # Ensure at least a minimal changelog - changelogs = [] - if package.version + '-' + str(rpm_inc) not in [x[0] for x in changelogs]: - changelogs.insert(0, ( - package.version + '-' + str(rpm_inc), ( - data['Date'], - package.maintainers[0].name, - package.maintainers[0].email - ) - )) - exported_tags = [e.tagname for e in package.exports] - data['NoArch'] = 'metapackage' in exported_tags or 'architecture_independent' in exported_tags - data['changelogs'] = changelogs - # Summarize dependencies - summarize_dependency_mapping(data, depends, build_depends, resolved_deps) - - def convertToUnicode(obj): - if sys.version_info.major == 2: - if isinstance(obj, str): - return unicode(obj.decode('utf8')) - elif isinstance(obj, unicode): - return obj - else: - if isinstance(obj, bytes): - return str(obj.decode('utf8')) - elif isinstance(obj, str): - return obj - if isinstance(obj, list): - for i, val in enumerate(obj): - obj[i] = convertToUnicode(val) - return obj - elif isinstance(obj, type(None)): - return None - elif isinstance(obj, tuple): - obj_tmp = list(obj) - for i, val in enumerate(obj_tmp): - obj_tmp[i] = convertToUnicode(obj_tmp[i]) - return tuple(obj_tmp) - elif isinstance(obj, int): - return obj - elif isinstance(obj, int): - return obj - raise RuntimeError('need to deal with type %s' % (str(type(obj)))) - - for item in data.items(): - data[item[0]] = convertToUnicode(item[1]) - - return data - - -def __process_template_folder(path, subs): - items = os.listdir(path) - processed_items = [] - for item in list(items): - item = os.path.abspath(os.path.join(path, item)) - if os.path.basename(item) in ['.', '..', '.git', '.svn']: - continue - if os.path.isdir(item): - sub_items = __process_template_folder(item, subs) - processed_items.extend([os.path.join(item, s) for s in sub_items]) - if not item.endswith(TEMPLATE_EXTENSION): - continue - with open(item, 'r') as f: - template = f.read() - # Remove extension - template_path = item[:-len(TEMPLATE_EXTENSION)] - # Expand template - info("Expanding '{0}' -> '{1}'".format( - os.path.relpath(item), - os.path.relpath(template_path))) - result = em.expand(template, **subs) - # Write the result - with io.open(template_path, 'w', encoding='utf-8') as f: - if sys.version_info.major == 2: - result = result.decode('utf-8') - f.write(result) - # Copy the permissions - shutil.copymode(item, template_path) - processed_items.append(item) - return processed_items - - -def process_template_files(path, subs): - info(fmt("@!@{bf}==>@| In place processing templates in 'rpm' folder.")) - rpm_dir = os.path.join(path, 'rpm') - if not os.path.exists(rpm_dir): - sys.exit("No rpm directory found at '{0}', cannot process templates." - .format(rpm_dir)) - return __process_template_folder(rpm_dir, subs) - - -def match_branches_with_prefix(prefix, get_branches, prune=False): - debug("match_branches_with_prefix(" + str(prefix) + ", " + - str(get_branches()) + ")") - branches = [] - # Match branches - existing_branches = get_branches() - for branch in existing_branches: - if branch.startswith('remotes/origin/'): - branch = branch.split('/', 2)[-1] - if branch.startswith(prefix): - branches.append(branch) - branches = list(set(branches)) - if prune: - # Prune listed branches by packages in latest upstream - with inbranch('upstream'): - pkg_names, version, pkgs_dict = get_package_data('upstream') - for branch in branches: - if branch.split(prefix)[-1].strip('/') not in pkg_names: - branches.remove(branch) - return branches - - -def get_package_from_branch(branch): - with inbranch(branch): - try: - package_data = get_package_data(branch) - except SystemExit: - return None - if type(package_data) not in [list, tuple]: - # It is a ret code - RpmGenerator.exit(package_data) - names, version, packages = package_data - if type(names) is list and len(names) > 1: - RpmGenerator.exit( - "RPM generator does not support generating " - "from branches with multiple packages in them, use " - "the release generator first to split packages into " - "individual branches.") - if type(packages) is dict: - return list(packages.values())[0] - - -def rpmify_string(value): +def format_description(value): markup_remover = re.compile(r'<.*?>') value = markup_remover.sub('', value) value = re.sub('\s+', ' ', value) @@ -453,373 +85,101 @@ def rpmify_string(value): return value -def sanitize_package_name(name): - return name.replace('_', '-') - - -class RpmGenerator(BloomGenerator): +class RpmGenerator(PackageManagerGenerator): title = 'rpm' + package_manager = 'rpm' description = "Generates RPMs from the catkin meta data" - has_run_rosdep = False - default_install_prefix = '/usr' - rosdistro = os.environ.get('ROS_DISTRO', 'indigo') def prepare_arguments(self, parser): # Add command line arguments for this generator add = parser.add_argument - add('-i', '--rpm-inc', help="RPM increment number", default='0') - add('-p', '--prefix', required=True, - help="branch prefix to match, and from which create RPMs" - " hint: if you want to match 'release/foo' use 'release'") - add('-a', '--match-all', default=False, action="store_true", - help="match all branches with the given prefix, " - "even if not in current upstream") - add('--distros', nargs='+', required=False, default=[], - help='A list of RPM (fedora) distros to generate for') - add('--install-prefix', default=None, - help="overrides the default installation prefix (/usr)") add('--os-name', default='fedora', help="overrides os_name, set to 'fedora' by default") + return PackageManagerGenerator.prepare_arguments(self, parser) def handle_arguments(self, args): - self.interactive = args.interactive - self.rpm_inc = args.rpm_inc self.os_name = args.os_name - self.distros = args.distros - if self.distros in [None, []]: - index = rosdistro.get_index(rosdistro.get_index_url()) - distribution_file = rosdistro.get_distribution_file(index, self.rosdistro) - if self.os_name not in distribution_file.release_platforms: - warning("No platforms defined for os '{0}' in release file for the '{1}' distro." - "\nNot performing RPM generation." - .format(self.os_name, self.rosdistro)) - sys.exit(0) - self.distros = distribution_file.release_platforms[self.os_name] - self.install_prefix = args.install_prefix - if args.install_prefix is None: - self.install_prefix = self.default_install_prefix - self.prefix = args.prefix - self.branches = match_branches_with_prefix(self.prefix, get_branches, prune=not args.match_all) - if len(self.branches) == 0: - error( - "No packages found, check your --prefix or --src arguments.", - exit=True - ) - self.packages = {} - self.tag_names = {} - self.names = [] - self.branch_args = [] - self.rpm_branches = [] - for branch in self.branches: - package = get_package_from_branch(branch) - if package is None: - # This is an ignored package - continue - self.packages[package.name] = package - self.names.append(package.name) - args = self.generate_branching_arguments(package, branch) - # First branch is rpm/[/] - self.rpm_branches.append(args[0][0]) - self.branch_args.extend(args) - - def summarize(self): - info("Generating source RPMs for the packages: " + str(self.names)) - info("RPM Incremental Version: " + str(self.rpm_inc)) - info("RPM Distributions: " + str(self.distros)) - - def get_branching_arguments(self): - return self.branch_args - - def update_rosdep(self): - update_rosdep() - self.has_run_rosdep = True - - def _check_all_keys_are_valid(self, peer_packages, rosdistro): - keys_to_resolve = [] - key_to_packages_which_depends_on = collections.defaultdict(list) - keys_to_ignore = set() - for package in self.packages.values(): - package.evaluate_conditions(package_conditional_context(rosdistro)) - depends = [ - dep for dep in (package.run_depends + package.buildtool_export_depends) - if dep.evaluated_condition] - build_depends = [ - dep for dep in (package.build_depends + package.buildtool_depends + package.test_depends) - if dep.evaluated_condition] - unresolved_keys = [ - dep for dep in (depends + build_depends + package.replaces + package.conflicts) - if dep.evaluated_condition] - keys_to_ignore = { - dep for dep in keys_to_ignore.union(package.replaces + package.conflicts) - if dep.evaluated_condition} - keys = [d.name for d in unresolved_keys] - keys_to_resolve.extend(keys) - for key in keys: - key_to_packages_which_depends_on[key].append(package.name) - - os_name = self.os_name - rosdistro = self.rosdistro - all_keys_valid = True - for key in sorted(set(keys_to_resolve)): - for os_version in self.distros: - try: - extended_peer_packages = peer_packages + [d.name for d in keys_to_ignore] - rule, installer_key, default_installer_key = \ - resolve_rosdep_key(key, os_name, os_version, rosdistro, extended_peer_packages, - retry=False) - if rule is None: - continue - if installer_key != default_installer_key: - error("Key '{0}' resolved to '{1}' with installer '{2}', " - "which does not match the default installer '{3}'." - .format(key, rule, installer_key, default_installer_key)) - BloomGenerator.exit( - "The RPM generator does not support dependencies " - "which are installed with the '{0}' installer." - .format(installer_key), - returncode=code.GENERATOR_INVALID_INSTALLER_KEY) - except (GeneratorError, RuntimeError) as e: - print(fmt("Failed to resolve @{cf}@!{key}@| on @{bf}{os_name}@|:@{cf}@!{os_version}@| with: {e}") - .format(**locals())) - print(fmt("@{cf}@!{0}@| is depended on by these packages: ").format(key) + - str(list(set(key_to_packages_which_depends_on[key])))) - print(fmt("@{kf}@!<== @{rf}@!Failed@|")) - all_keys_valid = False - return all_keys_valid + ret = PackageManagerGenerator.handle_arguments(self, args) + return ret def pre_modify(self): - info("\nPre-verifying RPM dependency keys...") - # Run rosdep update is needed - if not self.has_run_rosdep: - self.update_rosdep() - - peer_packages = [p.name for p in self.packages.values()] - - while not self._check_all_keys_are_valid(peer_packages, self.rosdistro): - error("Some of the dependencies for packages in this repository could not be resolved by rosdep.") - error("You can try to address the issues which appear above and try again if you wish, " - "or continue without releasing into RPM-based distributions (e.g. Fedora 24).") - try: - if not maybe_continue(msg="Would you like to try again?"): - error("User aborted after rosdep keys were not resolved.") - sys.exit(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO) - except (KeyboardInterrupt, EOFError): - error("\nUser quit.", exit=True) - update_rosdep() - invalidate_view_cache() - - info("All keys are " + ansi('greenf') + "OK" + ansi('reset') + "\n") + error_msg = ''.join([ + "Some of the dependencies for packages in this repository could not be resolved by rosdep.\n", + "You can try to address the issues which appear above and try again if you wish, ", + "or continue without releasing into RPM-based distributions (e.g. Fedora 24)." + ]) + PackageManagerGenerator._pre_modify(self, error_msg) for package in self.packages.values(): if not package.licenses or not package.licenses[0]: error("No license set for package '{0}', aborting.".format(package.name), exit=True) - def pre_branch(self, destination, source): - if destination in self.rpm_branches: - return - # Run rosdep update is needed - if not self.has_run_rosdep: - self.update_rosdep() - # Determine the current package being generated - name = destination.split('/')[-1] - distro = destination.split('/')[-2] - # Retrieve the package - package = self.packages[name] - # Report on this package - self.summarize_package(package, distro) - - def pre_rebase(self, destination): - # Get the stored configs is any - patches_branch = 'patches/' + destination - config = self.load_original_config(patches_branch) - if config is not None: - curr_config = get_patch_config(patches_branch) - if curr_config['parent'] == config['parent']: - set_patch_config(patches_branch, config) - - def post_rebase(self, destination): - name = destination.split('/')[-1] - # Retrieve the package - package = self.packages[name] - # Handle differently if this is an rpm vs distro branch - if destination in self.rpm_branches: - info("Placing RPM template files into '{0}' branch." - .format(destination)) - # Then this is an rpm branch - # Place the raw template files - self.place_template_files(package.get_build_type()) - else: - # This is a distro specific rpm branch - # Determine the current package being generated - distro = destination.split('/')[-2] - # Create RPMs for each distro - with inbranch(destination): - data = self.generate_rpm(package, distro) - # Create the tag name for later - self.tag_names[destination] = self.generate_tag_name(data) - # Update the patch configs - patches_branch = 'patches/' + destination - config = get_patch_config(patches_branch) - # Store it - self.store_original_config(config, patches_branch) - # Modify the base so import/export patch works - current_branch = get_current_branch() - if current_branch is None: - error("Could not determine current branch.", exit=True) - config['base'] = get_commit_hash(current_branch) - # Set it - set_patch_config(patches_branch, config) - - def post_patch(self, destination, color='bluef'): - if destination in self.rpm_branches: - return - # Tag after patches have been applied - with inbranch(destination): - # Tag - tag_name = self.tag_names[destination] - if tag_exists(tag_name): - if self.interactive: - warning("Tag exists: " + tag_name) - warning("Do you wish to overwrite it?") - if not maybe_continue('y'): - error("Answered no to continue, aborting.", exit=True) - else: - warning("Overwriting tag: " + tag_name) - else: - info("Creating tag: " + tag_name) - execute_command('git tag -f ' + tag_name) - # Report of success - name = destination.split('/')[-1] - package = self.packages[name] - distro = destination.split('/')[-2] - info(ansi(color) + "####" + ansi('reset'), use_prefix=False) - info( - ansi(color) + "#### " + ansi('greenf') + "Successfully" + - ansi(color) + " generated '" + ansi('boldon') + distro + - ansi('boldoff') + "' RPM for package" - " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + - " at version '" + ansi('boldon') + package.version + - "-" + str(self.rpm_inc) + ansi('boldoff') + "'" + - ansi('reset'), - use_prefix=False - ) - info(ansi(color) + "####\n" + ansi('reset'), use_prefix=False) - - def store_original_config(self, config, patches_branch): - with inbranch(patches_branch): - with open('rpm.store', 'w+') as f: - f.write(json.dumps(config)) - execute_command('git add rpm.store') - if has_changes(): - execute_command('git commit -m "Store original patch config"') - - def load_original_config(self, patches_branch): - config_store = show(patches_branch, 'rpm.store') - if config_store is None: - return config_store - return json.loads(config_store) - - def place_template_files(self, build_type, rpm_dir='rpm'): - # Create/Clean the rpm folder - if os.path.exists(rpm_dir): - if self.interactive: - warning("rpm directory exists: " + rpm_dir) - warning("Do you wish to overwrite it?") - if not maybe_continue('y'): - error("Answered no to continue, aborting.", exit=True) - else: - warning("Overwriting rpm directory: " + rpm_dir) - execute_command('git rm -rf ' + rpm_dir) - execute_command('git commit -m "Clearing previous rpm folder"') - if os.path.exists(rpm_dir): - shutil.rmtree(rpm_dir) - # Use generic place template files command - place_template_files('.', build_type, gbp=True) - # Commit results - execute_command('git add ' + rpm_dir) - execute_command('git commit -m "Placing rpm template files"') - - def get_releaser_history(self): - # Assumes that this is called in the target branch - patches_branch = 'patches/' + get_current_branch() - raw = show(patches_branch, 'releaser_history.json') - return None if raw is None else json.loads(raw) - - def set_releaser_history(self, history): - # Assumes that this is called in the target branch - patches_branch = 'patches/' + get_current_branch() - debug("Writing release history to '{0}' branch".format(patches_branch)) - with inbranch(patches_branch): - with open('releaser_history.json', 'w') as f: - f.write(json.dumps(history)) - execute_command('git add releaser_history.json') - if has_changes(): - execute_command('git commit -m "Store releaser history"') - - def get_subs(self, package, rpm_distro, releaser_history=None): - return generate_substitutions_from_package( - package, - self.os_name, - rpm_distro, - self.rosdistro, - self.install_prefix, - self.rpm_inc, - [p.name for p in self.packages.values()], - releaser_history=releaser_history, - fallback_resolver=missing_dep_resolver - ) - - def generate_rpm(self, package, rpm_distro, rpm_dir='rpm'): - info("Generating RPM for {0}...".format(rpm_distro)) + def generate_package(self, package, os_version): + info("Generating {0} for {1}...".format(self.package_manager, os_version)) # Try to retrieve the releaser_history releaser_history = self.get_releaser_history() # Generate substitution values - subs = self.get_subs(package, rpm_distro, releaser_history) + subs = self.get_subs(package, os_version, format_description, format_depends, releaser_history) # Use subs to create and store releaser history self.set_releaser_history(dict(subs['changelogs'])) # Template files - template_files = process_template_files('.', subs) + template_files = process_template_files('.', subs, self.package_manager) # Remove any residual template files execute_command('git rm -rf ' + ' '.join("'{}'".format(t) for t in template_files)) # Add changes to the rpm folder - execute_command('git add ' + rpm_dir) + execute_command('git add ' + self.package_manager) # Commit changes - execute_command('git commit -m "Generated RPM files for ' + - rpm_distro + '"') + execute_command('git commit -m "Generated {0} files for {1}"' + .format(self.package_manager, os_version)) # Rename the template spec file - execute_command('git mv ' + rpm_dir + '/template.spec ' + rpm_dir + '/' + subs['Package'] + '.spec') + execute_command('git mv {0}/template.spec {1}/{2}.spec' + .format(self.package_manager, self.package_manager, subs['Package'])) # Commit changes - execute_command('git commit -m "Renamed RPM spec file for ' + - rpm_distro + '"') + execute_command('git commit -m "Renamed {0} spec files for {1}"' + .format(self.package_manager, os_version)) # Return the subs for other use return subs - def generate_tag_name(self, data): - tag_name = '{Package}-{Version}-{RPMInc}_{Distribution}' - tag_name = 'rpm/' + tag_name.format(**data) - return tag_name + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + # Use the time stamp to set the date strings + stamp = datetime.datetime.now(tz.tzlocal()) + subs['Date'] = stamp.strftime('%a %b %d %Y') + # Maintainers + maintainers = [] + for m in package.maintainers: + maintainers.append(str(m)) + # Changelog + if releaser_history: + sorted_releaser_history = sorted(releaser_history, + key=lambda k: LooseVersion(k), reverse=True) + sorted_releaser_history = sorted(sorted_releaser_history, + key=lambda k: strptime(releaser_history.get(k)[0], '%a %b %d %Y'), + reverse=True) + changelogs = [(v, releaser_history[v]) for v in sorted_releaser_history] + else: + # Ensure at least a minimal changelog + changelogs = [] + if package.version + '-' + str(subs['Inc']) not in [x[0] for x in changelogs]: + changelogs.insert(0, ( + package.version + '-' + str(subs['Inc']), ( + subs['Date'], + package.maintainers[0].name, + package.maintainers[0].email + ) + )) + exported_tags = [e.tagname for e in package.exports] + subs['NoArch'] = 'metapackage' in exported_tags or 'architecture_independent' in exported_tags + subs['changelogs'] = changelogs + + # License + if not package.licenses or not package.licenses[0]: + error("No license set for package '{0}', aborting.".format(package.name), exit=True) + subs['License'] = package.licenses[0] - def generate_branching_arguments(self, package, branch): - n = package.name - # rpm branch - rpm_branch = 'rpm/' + n - # Branch first to the rpm branch - args = [[rpm_branch, branch, False]] - # Then for each RPM distro, branch from the base rpm branch - args.extend([ - ['rpm/' + d + '/' + n, rpm_branch, False] for d in self.distros - ]) - return args + return subs - def summarize_package(self, package, distro, color='bluef'): - info(ansi(color) + "\n####" + ansi('reset'), use_prefix=False) - info( - ansi(color) + "#### Generating '" + ansi('boldon') + distro + - ansi('boldoff') + "' RPM for package" - " '" + ansi('boldon') + package.name + ansi('boldoff') + "'" + - " at version '" + ansi('boldon') + package.version + - "-" + str(self.rpm_inc) + ansi('boldoff') + "'" + - ansi('reset'), - use_prefix=False - ) - info(ansi(color) + "####" + ansi('reset'), use_prefix=False) + def generate_tag_name(self, subs): + tag_name = '{Package}-{Version}-{Inc}_{Distribution}' + tag_name = self.package_manager + '/' + tag_name.format(**subs) + return tag_name diff --git a/bloom/generators/rpm/templates/catkin/template.spec.em b/bloom/generators/rpm/templates/catkin/template.spec.em index a65a2a81..6a71ad36 100644 --- a/bloom/generators/rpm/templates/catkin/template.spec.em +++ b/bloom/generators/rpm/templates/catkin/template.spec.em @@ -1,6 +1,6 @@ Name: @(Package) Version: @(Version) -Release: @(RPMInc)%{?dist} +Release: @(Inc)%{?dist} Summary: ROS @(Name) package Group: Development/Libraries diff --git a/bloom/generators/rpm/templates/cmake/template.spec.em b/bloom/generators/rpm/templates/cmake/template.spec.em index a65a2a81..6a71ad36 100644 --- a/bloom/generators/rpm/templates/cmake/template.spec.em +++ b/bloom/generators/rpm/templates/cmake/template.spec.em @@ -1,6 +1,6 @@ Name: @(Package) Version: @(Version) -Release: @(RPMInc)%{?dist} +Release: @(Inc)%{?dist} Summary: ROS @(Name) package Group: Development/Libraries From 452fece4f2fe8e4e5969c5b9b31d39affe366d79 Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Sun, 16 Jun 2019 18:55:02 +0800 Subject: [PATCH 11/23] add set_default_distros test --- bloom/generators/common.py | 4 +- .../test_generators/test_common/__init__.py | 0 .../test_common/test_generator.py | 42 +++++++++++++++++++ .../test_pkg/CHANGELOG.rst | 7 ++++ .../test_generator_data/test_pkg/package.xml | 18 ++++++++ 5 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 test/unit_tests/test_generators/test_common/__init__.py create mode 100644 test/unit_tests/test_generators/test_common/test_generator.py create mode 100644 test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/CHANGELOG.rst create mode 100644 test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/package.xml diff --git a/bloom/generators/common.py b/bloom/generators/common.py index 97ddb447..70751114 100644 --- a/bloom/generators/common.py +++ b/bloom/generators/common.py @@ -730,7 +730,7 @@ def get_package_from_branch(self, branch): if type(packages) is dict: return list(packages.values())[0] - def get_default_distros(self): + def set_default_distros(self): index = rosdistro.get_index(rosdistro.get_index_url()) distribution_file = rosdistro.get_distribution_file(index, self.rosdistro) if self.os_name not in distribution_file.release_platforms: @@ -748,7 +748,7 @@ def handle_arguments(self, args): self.inc = args.inc self.distros = args.distros if self.distros in [None, []]: - self.get_default_distros() + self.set_default_distros() self.install_prefix = args.install_prefix if args.install_prefix is None: self.install_prefix = self.default_install_prefix diff --git a/test/unit_tests/test_generators/test_common/__init__.py b/test/unit_tests/test_generators/test_common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/unit_tests/test_generators/test_common/test_generator.py b/test/unit_tests/test_generators/test_common/test_generator.py new file mode 100644 index 00000000..d9f6f4a2 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator.py @@ -0,0 +1,42 @@ +import os + +from ....utils.common import AssertRaisesContext +from ....utils.common import redirected_stdio + +from bloom.generators.debian.generator import PackageManagerGenerator + +from catkin_pkg.packages import find_packages + +GENERATE_DATA_PATH = 'test_generator_data' +test_data_dir = os.path.join(os.path.dirname(__file__), GENERATE_DATA_PATH) + + +def get_package(pkg_name): + packages = dict([(pkg.name, pkg) for path, pkg in find_packages(test_data_dir).items()]) + return packages[pkg_name] + + +def get_generator(): + gen = PackageManagerGenerator() + gen.package_manager = 'debian' + return gen + + +def test_set_default_distros(): + gen = get_generator() + + gen.rosdistro = 'dashing' + gen.os_name = 'ubuntu' + gen.set_default_distros() + assert gen.distros == ['bionic'] + + gen.distros = None + gen.os_name = "debian" + gen.os_not_required = True + with AssertRaisesContext(SystemExit, ""): + with redirected_stdio(): + gen.set_default_distros() + gen.os_not_required = False + with AssertRaisesContext(SystemExit, "No platforms defined"): + with redirected_stdio(): + gen.set_default_distros() diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/CHANGELOG.rst b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/CHANGELOG.rst new file mode 100644 index 00000000..731c4bb6 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/CHANGELOG.rst @@ -0,0 +1,7 @@ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Changelog for package test_pkg +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +0.1.0 (2013-10-09) +------------------- +* Initial test package diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/package.xml b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/package.xml new file mode 100644 index 00000000..c11af778 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/package.xml @@ -0,0 +1,18 @@ + + + test_pkg + 0.1.0 + The test_pkg package + http://wiki.ros.org/test_common_pkg + + nobody1 + nobody2 + + TODO + + roscpp + rospy + + catkin + + From 263275f422cc0937703ab10b734179942663491d Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Sun, 16 Jun 2019 19:09:29 +0800 Subject: [PATCH 12/23] test place template files logic added --- bloom/generators/common.py | 12 +- .../test_common/test_generator.py | 130 ++++++++++++++++++ .../bad_dependency_pkg/CHANGELOG.rst | 7 + .../bad_dependency_pkg/package.xml | 19 +++ .../test_pkg/debian/gbp.conf.em | 8 ++ .../test_pkg/debian/source/local_options.em | 2 + .../test_pkg/debian/test_pkg.service | 2 + 7 files changed, 175 insertions(+), 5 deletions(-) create mode 100644 test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/CHANGELOG.rst create mode 100644 test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/package.xml create mode 100644 test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/gbp.conf.em create mode 100644 test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/source/local_options.em create mode 100644 test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/test_pkg.service diff --git a/bloom/generators/common.py b/bloom/generators/common.py index 70751114..d8174abf 100644 --- a/bloom/generators/common.py +++ b/bloom/generators/common.py @@ -360,7 +360,7 @@ def generate_substitutions_from_package( return data -def __place_template_folder(group, src, dst, gbp=False): +def __place_template_folder(group, src, dst, gbp=False, overwrite=False): template_files = pkg_resources.resource_listdir(group, src) # For each template, place for template_file in template_files: @@ -382,7 +382,7 @@ def __place_template_folder(group, src, dst, gbp=False): "'{0}': {1}".format(template_file, str(err)), exit=True) if not os.path.exists(dst): os.makedirs(dst) - if os.path.exists(template_dst): + if os.path.exists(template_dst) and not overwrite: debug("Not overwriting existing file '{0}'".format(template_dst)) else: with io.open(template_dst, 'w', encoding='utf-8') as f: @@ -422,7 +422,7 @@ def convertToUnicode(obj): raise RuntimeError('need to deal with type %s' % (str(type(obj)))) -def place_template_files(path, build_type, package_manager, gbp=False): +def place_template_files(path, build_type, package_manager, gbp=False, overwrite=False): info(fmt("@!@{bf}==>@| Placing templates files in the '" + package_manager + "' folder.")) dir_path = os.path.join(path, package_manager) if not os.path.exists(dir_path): @@ -430,7 +430,7 @@ def place_template_files(path, build_type, package_manager, gbp=False): # Place template files group = 'bloom.generators.' + package_manager templates = os.path.join('templates', build_type) - __place_template_folder(group, templates, dir_path, gbp) + __place_template_folder(group, templates, dir_path, gbp, overwrite=overwrite) def summarize_dependency_mapping(data, deps, build_deps, resolved_deps): @@ -998,6 +998,7 @@ def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): def place_template_files(self, build_type, dir_path=None): # Create/Clean the package system folder + overwrite = False if dir_path is None: dir_path = os.path.join(".", self.package_manager) if os.path.exists(dir_path): @@ -1006,6 +1007,7 @@ def place_template_files(self, build_type, dir_path=None): warning("Do you wish to overwrite it?") if not maybe_continue('y'): error("Answered no to continue, aborting.", exit=True) + overwrite = True elif 'BLOOM_CLEAR_TEMPLATE_ON_GENERATION' in os.environ: warning("Overwriting {0} directory: {1}".format(self.package_manager, dir_path)) execute_command('git rm -rf ' + dir_path) @@ -1016,7 +1018,7 @@ def place_template_files(self, build_type, dir_path=None): else: warning("Not overwriting {0} directory.".format(self.package_manager)) # Use generic place template files command - place_template_files('.', build_type, self.package_manager, gbp=True) + place_template_files('.', build_type, self.package_manager, gbp=True, overwrite=overwrite) # Commit results execute_command('git add ' + dir_path) _, has_files, _ = execute_command('git diff --cached --name-only', return_io=True) diff --git a/test/unit_tests/test_generators/test_common/test_generator.py b/test/unit_tests/test_generators/test_common/test_generator.py index d9f6f4a2..1ccfdc95 100644 --- a/test/unit_tests/test_generators/test_common/test_generator.py +++ b/test/unit_tests/test_generators/test_common/test_generator.py @@ -1,10 +1,19 @@ import os +from collections import namedtuple +from distutils.dir_util import copy_tree + from ....utils.common import AssertRaisesContext +from ....utils.common import bloom_answer +from ....utils.common import change_directory from ....utils.common import redirected_stdio +from ....utils.common import temporary_directory +from ....utils.common import user from bloom.generators.debian.generator import PackageManagerGenerator +from bloom.util import code + from catkin_pkg.packages import find_packages GENERATE_DATA_PATH = 'test_generator_data' @@ -40,3 +49,124 @@ def test_set_default_distros(): with AssertRaisesContext(SystemExit, "No platforms defined"): with redirected_stdio(): gen.set_default_distros() + + +def listdir(dirname): + # list files in one direcotory, including its subdirectory's file + files = [] + for f in os.listdir(dirname): + if os.path.isfile(os.path.join(dirname, f)): + files.append(os.path.join(dirname, f)) + else: + for sub_f in os.listdir(os.path.join(dirname, f)): + files.append(os.path.join(dirname, f, sub_f)) + return files + + +def test_place_template_files(): + pkg_name = 'test_pkg' + pkg = get_package(pkg_name) + gen = get_generator() + gen.interactive = False + build_type = pkg.get_build_type() + + # Test normal place template files + with redirected_stdio(): + with temporary_directory(): + user('git init .') + gen.place_template_files(build_type) + placed_files = listdir('debian') + template_file_list = [ + 'debian/source/format.em', + 'debian/source/options.em', + 'debian/changelog.em', + 'debian/compat.em', + 'debian/control.em', + 'debian/copyright.em', + 'debian/gbp.conf.em', + 'debian/rules.em', + ] + for f in template_file_list: + assert f in placed_files, "{0} not placed".format(f) + + # Test if package system directory exists + test_dir_exist_func_list = [ + dir_exist_with_interactive, + dir_exist_with_clean, + dir_exist_default, + ] + TestPlaceTemplateFileStruct = namedtuple( + 'TestPlaceTemplateFileStruct', + 'generator build_type old_dir, directory,' + 'original_template_files_dict,' + 'original_normal_files' + ) + dir_target = os.path.join(os.path.dirname(__file__), GENERATE_DATA_PATH, pkg_name) + with change_directory(dir_target): + old_dir = os.getcwd() + original_files = listdir('debian') + original_template_files_dict = dict([(f, open(f).read()) + for f in original_files if f.endswith('.em')]) + original_normal_files = [f for f in original_files if not f.endswith('.em')] + for f in test_dir_exist_func_list: + with temporary_directory() as directory: + user('git init .') + copy_tree(old_dir, directory) + user('git add .') + user('git commit --allow-empty -m "Initial commit"') + data = TestPlaceTemplateFileStruct(gen, build_type, old_dir,directory, + original_template_files_dict, + original_normal_files) + f(data) + + +def dir_exist_with_interactive(data): + gen = data.generator + gen.interactive = True + + with AssertRaisesContext(SystemExit, "Answered no to continue"): + with redirected_stdio(): + with bloom_answer(['n']): + gen.place_template_files(data.build_type) + + with redirected_stdio(): + with bloom_answer(['y']): + gen.place_template_files(data.build_type) + placed_files_dict = dict([(f, open(f).read()) for f in listdir('debian')]) + # overwrite the template files should not remove the origianl debian files + for f in data.original_normal_files: + assert f in placed_files_dict.keys() + for f, content in data.original_template_files_dict.items(): + assert f in placed_files_dict.keys() + # Your gbp.conf.em will be changed if you answer yes to overwrite + if f == 'gbp.conf.em': + assert content != placed_files_dict[f] + + gen.interactive = False + + +def dir_exist_with_clean(data): + gen = data.generator + CLEAR_TEMPLATE_ENV_PARAMETER = 'BLOOM_CLEAR_TEMPLATE_ON_GENERATION' + os.environ.setdefault(CLEAR_TEMPLATE_ENV_PARAMETER, "1") + + with redirected_stdio(): + gen.place_template_files(data.build_type) + placed_files = listdir('debian') + for f in data.original_normal_files: + assert f not in placed_files + + os.environ.pop(CLEAR_TEMPLATE_ENV_PARAMETER) + + +def dir_exist_default(data): + gen = data.generator + with redirected_stdio(): + gen.place_template_files(data.build_type) + placed_files_dict = dict([(f, open(f).read()) for f in listdir('debian')]) + # default doesn't influence your file under debian directory + for f, content in data.original_template_files_dict.items(): + assert f in placed_files_dict.keys() + assert content == placed_files_dict[f] + for f in data.original_normal_files: + assert f in placed_files_dict.keys() diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/CHANGELOG.rst b/test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/CHANGELOG.rst new file mode 100644 index 00000000..f8eaf716 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/CHANGELOG.rst @@ -0,0 +1,7 @@ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Changelog for package bad_denpendency_pkg +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +0.1.0 (2013-10-09) +------------------- +* Initial test package diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/package.xml b/test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/package.xml new file mode 100644 index 00000000..6d6d5715 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/bad_dependency_pkg/package.xml @@ -0,0 +1,19 @@ + + + bad_dependency_pkg + 0.1.0 + The bad_dependency_pkg package + http://wiki.ros.org/test_common_pkg + + nobody1 + nobody2 + + TODO + + roscpp + rospy + bad_dependency + + catkin + + diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/gbp.conf.em b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/gbp.conf.em new file mode 100644 index 00000000..20bcdd92 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/gbp.conf.em @@ -0,0 +1,8 @@ +[DEFAULT] +# This section is for global settings. Affects all commands. +# Options set here have the lowest priority. +key = value + +[git-buildpackage] +upstream-tag=@(release_tag) +upstream-tree=tag diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/source/local_options.em b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/source/local_options.em new file mode 100644 index 00000000..f7812e91 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/source/local_options.em @@ -0,0 +1,2 @@ +unapply-patches +abort-on-upstream-changes \ No newline at end of file diff --git a/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/test_pkg.service b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/test_pkg.service new file mode 100644 index 00000000..3a4e12f0 --- /dev/null +++ b/test/unit_tests/test_generators/test_common/test_generator_data/test_pkg/debian/test_pkg.service @@ -0,0 +1,2 @@ +[Unit] +Description=Test pkg for bloom From d2ef24544b9056d05356c89112bee6d5aa6eb5c3 Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Sun, 16 Jun 2019 19:33:45 +0800 Subject: [PATCH 13/23] check_all_key_are_valid function test added Also fix typo here(unvalid => invalid) --- bloom/generators/common.py | 4 ++-- bloom/generators/debian/generator.py | 2 +- bloom/generators/rpm/generator.py | 2 +- .../test_common/test_generator.py | 16 ++++++++++++++++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/bloom/generators/common.py b/bloom/generators/common.py index d8174abf..a4f95ae1 100644 --- a/bloom/generators/common.py +++ b/bloom/generators/common.py @@ -841,7 +841,7 @@ def _check_all_keys_are_valid(self, peer_packages, rosdistro): all_keys_valid = False return all_keys_valid - def _pre_modify(self, key_unvalid_error_msg): + def check_all_keys_are_valid(self, key_invalid_error_msg): info("\nPre-verifying {0} dependency keys...".format(self.package_manager)) # Run rosdep update is needed if not self.has_run_rosdep: @@ -850,7 +850,7 @@ def _pre_modify(self, key_unvalid_error_msg): peer_packages = [p.name for p in self.packages.values()] while not self._check_all_keys_are_valid(peer_packages, self.rosdistro): - error(key_unvalid_error_msg) + error(key_invalid_error_msg) try: if not maybe_continue(msg="Would you like to try again?"): error("User aborted after rosdep keys were not resolved.") diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index 38d4897d..f3f466e0 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -179,7 +179,7 @@ def pre_modify(self): "Some of the dependencies for packages in this repository could not be resolved by rosdep.\n", "You can try to address the issues which appear above and try again if you wish." ]) - PackageManagerGenerator._pre_modify(self, error_msg) + PackageManagerGenerator.check_all_keys_are_valid(self, error_msg) def generate_package(self, package, os_version): info("Generating {0} for {1}...".format(self.package_manager, os_version)) diff --git a/bloom/generators/rpm/generator.py b/bloom/generators/rpm/generator.py index 83c80d0b..924a193e 100644 --- a/bloom/generators/rpm/generator.py +++ b/bloom/generators/rpm/generator.py @@ -108,7 +108,7 @@ def pre_modify(self): "You can try to address the issues which appear above and try again if you wish, ", "or continue without releasing into RPM-based distributions (e.g. Fedora 24)." ]) - PackageManagerGenerator._pre_modify(self, error_msg) + PackageManagerGenerator.check_all_keys_are_valid(self, error_msg) for package in self.packages.values(): if not package.licenses or not package.licenses[0]: diff --git a/test/unit_tests/test_generators/test_common/test_generator.py b/test/unit_tests/test_generators/test_common/test_generator.py index 1ccfdc95..f1b09c15 100644 --- a/test/unit_tests/test_generators/test_common/test_generator.py +++ b/test/unit_tests/test_generators/test_common/test_generator.py @@ -170,3 +170,19 @@ def dir_exist_default(data): assert content == placed_files_dict[f] for f in data.original_normal_files: assert f in placed_files_dict.keys() + + +def test_bad_dependency(): + bad_pkg_name = 'bad_dependency_pkg' + bad_pkg = get_package(bad_pkg_name) + pkg_bad_dict = {bad_pkg_name: bad_pkg} + + gen = get_generator() + gen.packages = pkg_bad_dict + gen.rosdistro = 'kinetic' + gen.os_name = 'ubuntu' + gen.distros = ['xenial'] + with bloom_answer(['n']): + with AssertRaisesContext(SystemExit, str(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO)): + with redirected_stdio(): + gen.check_all_keys_are_valid("") From d1aa11c791d96151bf3d36e0c977c821838489b0 Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Sun, 16 Jun 2019 19:35:11 +0800 Subject: [PATCH 14/23] test common part of substitutions --- bloom/generators/common.py | 7 +-- bloom/generators/debian/generator.py | 6 +-- bloom/generators/rpm/generator.py | 4 +- .../test_common/test_generator.py | 48 +++++++++++++++++++ 4 files changed, 55 insertions(+), 10 deletions(-) diff --git a/bloom/generators/common.py b/bloom/generators/common.py index a4f95ae1..fbac4cab 100644 --- a/bloom/generators/common.py +++ b/bloom/generators/common.py @@ -285,7 +285,7 @@ def generate_substitutions_from_package( warning("No homepage set, defaulting to ''") data['Homepage'] = homepage # Increment Number - data['Inc'] = '' if native else '-{0}'.format(inc) + data['Inc'] = '' if native else '{0}'.format(inc) # Package name data['Package'] = sanitize_package_name(package.name) # Installation prefix @@ -1025,7 +1025,7 @@ def place_template_files(self, build_type, dir_path=None): if has_files: execute_command('git commit -m "Placing {0} template files"'.format(self.package_manager)) - def get_subs(self, package, os_version, format_description, format_depends, releaser_history=None): + def get_subs(self, package, os_version, format_description, format_depends): # This is the common part for generate templacte substitute, then successor of # the generator will add its specic content via define its get_subs_hook function subs = generate_substitutions_from_package( @@ -1040,7 +1040,8 @@ def get_subs(self, package, os_version, format_description, format_depends, rele [p.name for p in self.packages.values()], fallback_resolver=self.missing_dep_resolver ) - subs['release_tag'] = 'release/{0}/{1}-{2}'.format(subs['Name'], subs['Version'], self.inc) + # Try to retrieve the releaser_history + releaser_history = self.get_releaser_history() subs = self.get_subs_hook(subs, package, self.rosdistro, releaser_history=releaser_history) for item in subs.items(): subs[item[0]] = convertToUnicode(item[1]) diff --git a/bloom/generators/debian/generator.py b/bloom/generators/debian/generator.py index f3f466e0..fadc027c 100644 --- a/bloom/generators/debian/generator.py +++ b/bloom/generators/debian/generator.py @@ -183,10 +183,8 @@ def pre_modify(self): def generate_package(self, package, os_version): info("Generating {0} for {1}...".format(self.package_manager, os_version)) - # Try to retrieve the releaser_history - releaser_history = self.get_releaser_history() # Generate substitution values - subs = self.get_subs(package, os_version, format_description, format_depends, releaser_history) + subs = self.get_subs(package, os_version, format_description, format_depends) # Use subs to create and store releaser history releaser_history = [(v, (n, e)) for v, _, _, n, e in subs['changelogs']] self.set_releaser_history(dict(releaser_history)) @@ -279,6 +277,6 @@ def get_subs_hook(subs, package, rosdistro, releaser_history=None): return subs def generate_tag_name(self, subs): - tag_name = '{Package}_{Version}{Inc}_{Distribution}' + tag_name = '{Package}_{Version}-{Inc}_{Distribution}' tag_name = self.package_manager + '/' + tag_name.format(**subs) return tag_name diff --git a/bloom/generators/rpm/generator.py b/bloom/generators/rpm/generator.py index 924a193e..29dca7d3 100644 --- a/bloom/generators/rpm/generator.py +++ b/bloom/generators/rpm/generator.py @@ -116,10 +116,8 @@ def pre_modify(self): def generate_package(self, package, os_version): info("Generating {0} for {1}...".format(self.package_manager, os_version)) - # Try to retrieve the releaser_history - releaser_history = self.get_releaser_history() # Generate substitution values - subs = self.get_subs(package, os_version, format_description, format_depends, releaser_history) + subs = self.get_subs(package, os_version, format_description, format_depends) # Use subs to create and store releaser history self.set_releaser_history(dict(subs['changelogs'])) # Template files diff --git a/test/unit_tests/test_generators/test_common/test_generator.py b/test/unit_tests/test_generators/test_common/test_generator.py index f1b09c15..5d866de3 100644 --- a/test/unit_tests/test_generators/test_common/test_generator.py +++ b/test/unit_tests/test_generators/test_common/test_generator.py @@ -186,3 +186,51 @@ def test_bad_dependency(): with AssertRaisesContext(SystemExit, str(code.GENERATOR_NO_ROSDEP_KEY_FOR_DISTRO)): with redirected_stdio(): gen.check_all_keys_are_valid("") + + +def format_depends(depends, resolved_deps): + formatted = [] + for d in depends: + formatted.append("{0}".format(d)) + return formatted + + +def format_description(value): + return value + + +class OneGenerator(PackageManagerGenerator): + package_manager = "test" + @staticmethod + def get_subs_hook(subs, package, rosdistro, releaser_history=None): + subs['Name'] = 'test_pkg improved' + subs['specific_part'] = 'none' + return subs + + +def test_get_substitute(): + pkg_name = 'test_pkg' + pkg = get_package(pkg_name) + pkg_dict = {pkg_name: pkg} + + gen = OneGenerator() + gen.packages = pkg_dict + gen.os_name = "ubuntu" + gen.os_version = "xenial" + gen.rosdistro = "kinetic" + gen.install_prefix = gen.default_install_prefix + gen.inc = "1" + + with redirected_stdio(): + subs = gen.get_subs(pkg, gen.os_version, format_description, format_depends) + + assert 'test_pkg improved' == subs['Name'] + assert '0.1.0' == subs['Version'] + assert 'The test_pkg package' == subs['Description'] + assert '1' == subs['Inc'] + assert 'test-pkg' == subs['Package'] + assert ['roscpp', 'rospy'] == subs['Depends'] + assert gen.os_version == subs['Distribution'] + assert "nobody1 " == subs['Maintainer'] + assert "nobody1 , nobody2 " == subs['Maintainers'] + assert 'none' == subs['specific_part'] From 8c23028a6f8adfe78d067985d6126844f2f6aaa2 Mon Sep 17 00:00:00 2001 From: lennonwoo Date: Wed, 3 Jul 2019 19:56:12 +0800 Subject: [PATCH 15/23] add missed rosdistro in super's get_subs_hooks This commit add the missed rosdistro when call super's get_subs_hook in rosdebian or rosrpm. The release_history should be passed as **kwargs format since it have default value already. And I also combine two line in get_release_tag method since it's not exceed 120 lines. --- bloom/generators/rosdebian.py | 6 ++---- bloom/generators/rosrpm.py | 5 ++--- .../test_generators/test_common/test_generator.py | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/bloom/generators/rosdebian.py b/bloom/generators/rosdebian.py index 2df405db..af03b161 100644 --- a/bloom/generators/rosdebian.py +++ b/bloom/generators/rosdebian.py @@ -37,7 +37,6 @@ from bloom.generators.common import generate_substitutions_from_package from bloom.generators.common import sanitize_package_name - from bloom.generators.debian import DebianGenerator from bloom.generators.debian import format_description from bloom.generators.debian import format_depends @@ -81,7 +80,7 @@ def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): @staticmethod def get_subs_hook(subs, package, rosdistro, releaser_history=None): - subs = DebianGenerator.get_subs_hook(subs, package, releaser_history) + subs = DebianGenerator.get_subs_hook(subs, package, rosdistro, releaser_history=releaser_history) subs['Package'] = rosify_package_name(subs['Package'], rosdistro) # ROS 2 specific bloom extensions. @@ -127,8 +126,7 @@ def generate_branching_arguments(self, package, branch): return args def get_release_tag(self, data): - return 'release/{0}/{1}/{2}-{3}'\ - .format(self.rosdistro, data['Name'], data['Version'], self.inc) + return 'release/{0}/{1}/{2}-{3}'.format(self.rosdistro, data['Name'], data['Version'], self.inc) def rosify_package_name(name, rosdistro): diff --git a/bloom/generators/rosrpm.py b/bloom/generators/rosrpm.py index f672ff4c..3dc3b4fc 100644 --- a/bloom/generators/rosrpm.py +++ b/bloom/generators/rosrpm.py @@ -79,7 +79,7 @@ def missing_dep_resolver(key, peer_packages, os_name, os_version, ros_distro): @staticmethod def get_subs_hook(subs, package, rosdistro, releaser_history=None): - subs = RpmGenerator.get_subs_hook(subs, package, releaser_history) + subs = RpmGenerator.get_subs_hook(subs, package, rosdistro, releaser_history=releaser_history) subs['Package'] = rosify_package_name(subs['Package'], rosdistro) return subs @@ -93,8 +93,7 @@ def generate_branching_arguments(self, package, branch): return args def get_release_tag(self, data): - return 'release/{0}/{1}/{2}-{3}'\ - .format(self.rosdistro, data['Name'], data['Version'], self.inc) + return 'release/{0}/{1}/{2}-{3}'.format(self.rosdistro, data['Name'], data['Version'], self.inc) def rosify_package_name(name, rosdistro): diff --git a/test/unit_tests/test_generators/test_common/test_generator.py b/test/unit_tests/test_generators/test_common/test_generator.py index 5d866de3..92fc3b65 100644 --- a/test/unit_tests/test_generators/test_common/test_generator.py +++ b/test/unit_tests/test_generators/test_common/test_generator.py @@ -10,7 +10,7 @@ from ....utils.common import temporary_directory from ....utils.common import user -from bloom.generators.debian.generator import PackageManagerGenerator +from bloom.generators.common import PackageManagerGenerator from bloom.util import code From 87123ef3a13fe909401256356d460914263fd7b9 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 22 Apr 2019 16:59:18 -0700 Subject: [PATCH 16/23] Drop 'Group' value from RPM spec templates Neither Fedora or RHEL 7+ use this value. If another distribution were to be supported for RPM generation in Bloom that can take advantage of this value, we should find a better way to add it instead of hard- coding it into the template. Signed-off-by: Scott K Logan --- bloom/generators/rpm/templates/catkin/template.spec.em | 1 - bloom/generators/rpm/templates/cmake/template.spec.em | 1 - 2 files changed, 2 deletions(-) diff --git a/bloom/generators/rpm/templates/catkin/template.spec.em b/bloom/generators/rpm/templates/catkin/template.spec.em index 6a71ad36..9634d611 100644 --- a/bloom/generators/rpm/templates/catkin/template.spec.em +++ b/bloom/generators/rpm/templates/catkin/template.spec.em @@ -3,7 +3,6 @@ Version: @(Version) Release: @(Inc)%{?dist} Summary: ROS @(Name) package -Group: Development/Libraries License: @(License) @[if Homepage and Homepage != '']URL: @(Homepage)@\n@[end if]Source0: %{name}-%{version}.tar.gz @[if NoArch]@\nBuildArch: noarch@\n@[end if] diff --git a/bloom/generators/rpm/templates/cmake/template.spec.em b/bloom/generators/rpm/templates/cmake/template.spec.em index 6a71ad36..9634d611 100644 --- a/bloom/generators/rpm/templates/cmake/template.spec.em +++ b/bloom/generators/rpm/templates/cmake/template.spec.em @@ -3,7 +3,6 @@ Version: @(Version) Release: @(Inc)%{?dist} Summary: ROS @(Name) package -Group: Development/Libraries License: @(License) @[if Homepage and Homepage != '']URL: @(Homepage)@\n@[end if]Source0: %{name}-%{version}.tar.gz @[if NoArch]@\nBuildArch: noarch@\n@[end if] From 5db12cc161c76fc46f63d049634668caeba0f7b5 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 22 Apr 2019 17:04:32 -0700 Subject: [PATCH 17/23] Use 'autosetup' macro in RPM spec files Supported in RPM 4.11 and newer: https://rpm.org/user_doc/autosetup.html Signed-off-by: Scott K Logan --- bloom/generators/rpm/templates/catkin/template.spec.em | 2 +- bloom/generators/rpm/templates/cmake/template.spec.em | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bloom/generators/rpm/templates/catkin/template.spec.em b/bloom/generators/rpm/templates/catkin/template.spec.em index 9634d611..750937c2 100644 --- a/bloom/generators/rpm/templates/catkin/template.spec.em +++ b/bloom/generators/rpm/templates/catkin/template.spec.em @@ -11,7 +11,7 @@ License: @(License) @(Description) %prep -%setup -q +%autosetup %build # In case we're installing to a non-standard location, look for a setup.sh diff --git a/bloom/generators/rpm/templates/cmake/template.spec.em b/bloom/generators/rpm/templates/cmake/template.spec.em index 9634d611..750937c2 100644 --- a/bloom/generators/rpm/templates/cmake/template.spec.em +++ b/bloom/generators/rpm/templates/cmake/template.spec.em @@ -11,7 +11,7 @@ License: @(License) @(Description) %prep -%setup -q +%autosetup %build # In case we're installing to a non-standard location, look for a setup.sh From ff203dae110aee8453a6a2b560662fca8470d093 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 22 Apr 2019 17:11:57 -0700 Subject: [PATCH 18/23] Use 'make_*' macros in RPM spec files ...rather than assuming that the make executable is 'make'. Signed-off-by: Scott K Logan --- bloom/generators/rpm/templates/catkin/template.spec.em | 5 ++--- bloom/generators/rpm/templates/cmake/template.spec.em | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bloom/generators/rpm/templates/catkin/template.spec.em b/bloom/generators/rpm/templates/catkin/template.spec.em index 750937c2..4a669928 100644 --- a/bloom/generators/rpm/templates/catkin/template.spec.em +++ b/bloom/generators/rpm/templates/catkin/template.spec.em @@ -31,15 +31,14 @@ mkdir -p obj-%{_target_platform} && cd obj-%{_target_platform} -DSETUPTOOLS_DEB_LAYOUT=OFF \ -DCATKIN_BUILD_BINARY_PACKAGE="1" \ -make %{?_smp_mflags} +%make_build %install # In case we're installing to a non-standard location, look for a setup.sh # in the install tree that was dropped by catkin, and source it. It will # set things like CMAKE_PREFIX_PATH, PKG_CONFIG_PATH, and PYTHONPATH. if [ -f "@(InstallationPrefix)/setup.sh" ]; then . "@(InstallationPrefix)/setup.sh"; fi -cd obj-%{_target_platform} -make %{?_smp_mflags} install DESTDIR=%{buildroot} +%make_install -C obj-%{_target_platform} %files @(InstallationPrefix) diff --git a/bloom/generators/rpm/templates/cmake/template.spec.em b/bloom/generators/rpm/templates/cmake/template.spec.em index 750937c2..4a669928 100644 --- a/bloom/generators/rpm/templates/cmake/template.spec.em +++ b/bloom/generators/rpm/templates/cmake/template.spec.em @@ -31,15 +31,14 @@ mkdir -p obj-%{_target_platform} && cd obj-%{_target_platform} -DSETUPTOOLS_DEB_LAYOUT=OFF \ -DCATKIN_BUILD_BINARY_PACKAGE="1" \ -make %{?_smp_mflags} +%make_build %install # In case we're installing to a non-standard location, look for a setup.sh # in the install tree that was dropped by catkin, and source it. It will # set things like CMAKE_PREFIX_PATH, PKG_CONFIG_PATH, and PYTHONPATH. if [ -f "@(InstallationPrefix)/setup.sh" ]; then . "@(InstallationPrefix)/setup.sh"; fi -cd obj-%{_target_platform} -make %{?_smp_mflags} install DESTDIR=%{buildroot} +%make_install -C obj-%{_target_platform} %files @(InstallationPrefix) From fd52e5c89e6e2c32085eb00fae0efb924114a173 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 22 Apr 2019 17:13:17 -0700 Subject: [PATCH 19/23] Use more standard 4-space indentation in RPM spec files Also move terminating '..' to the end to make it easier to patch cmake arguments and less likely that the make arguments accidentally get added to the end of the cmake arguments. Signed-off-by: Scott K Logan --- .../rpm/templates/catkin/template.spec.em | 23 ++++++++++--------- .../rpm/templates/cmake/template.spec.em | 23 ++++++++++--------- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/bloom/generators/rpm/templates/catkin/template.spec.em b/bloom/generators/rpm/templates/catkin/template.spec.em index 4a669928..6606475f 100644 --- a/bloom/generators/rpm/templates/catkin/template.spec.em +++ b/bloom/generators/rpm/templates/catkin/template.spec.em @@ -19,17 +19,18 @@ License: @(License) # set things like CMAKE_PREFIX_PATH, PKG_CONFIG_PATH, and PYTHONPATH. if [ -f "@(InstallationPrefix)/setup.sh" ]; then . "@(InstallationPrefix)/setup.sh"; fi mkdir -p obj-%{_target_platform} && cd obj-%{_target_platform} -%cmake .. \ - -UINCLUDE_INSTALL_DIR \ - -ULIB_INSTALL_DIR \ - -USYSCONF_INSTALL_DIR \ - -USHARE_INSTALL_PREFIX \ - -ULIB_SUFFIX \ - -DCMAKE_INSTALL_LIBDIR="lib" \ - -DCMAKE_INSTALL_PREFIX="@(InstallationPrefix)" \ - -DCMAKE_PREFIX_PATH="@(InstallationPrefix)" \ - -DSETUPTOOLS_DEB_LAYOUT=OFF \ - -DCATKIN_BUILD_BINARY_PACKAGE="1" \ +%cmake \ + -UINCLUDE_INSTALL_DIR \ + -ULIB_INSTALL_DIR \ + -USYSCONF_INSTALL_DIR \ + -USHARE_INSTALL_PREFIX \ + -ULIB_SUFFIX \ + -DCMAKE_INSTALL_LIBDIR="lib" \ + -DCMAKE_INSTALL_PREFIX="@(InstallationPrefix)" \ + -DCMAKE_PREFIX_PATH="@(InstallationPrefix)" \ + -DSETUPTOOLS_DEB_LAYOUT=OFF \ + -DCATKIN_BUILD_BINARY_PACKAGE="1" \ + .. %make_build diff --git a/bloom/generators/rpm/templates/cmake/template.spec.em b/bloom/generators/rpm/templates/cmake/template.spec.em index 4a669928..6606475f 100644 --- a/bloom/generators/rpm/templates/cmake/template.spec.em +++ b/bloom/generators/rpm/templates/cmake/template.spec.em @@ -19,17 +19,18 @@ License: @(License) # set things like CMAKE_PREFIX_PATH, PKG_CONFIG_PATH, and PYTHONPATH. if [ -f "@(InstallationPrefix)/setup.sh" ]; then . "@(InstallationPrefix)/setup.sh"; fi mkdir -p obj-%{_target_platform} && cd obj-%{_target_platform} -%cmake .. \ - -UINCLUDE_INSTALL_DIR \ - -ULIB_INSTALL_DIR \ - -USYSCONF_INSTALL_DIR \ - -USHARE_INSTALL_PREFIX \ - -ULIB_SUFFIX \ - -DCMAKE_INSTALL_LIBDIR="lib" \ - -DCMAKE_INSTALL_PREFIX="@(InstallationPrefix)" \ - -DCMAKE_PREFIX_PATH="@(InstallationPrefix)" \ - -DSETUPTOOLS_DEB_LAYOUT=OFF \ - -DCATKIN_BUILD_BINARY_PACKAGE="1" \ +%cmake \ + -UINCLUDE_INSTALL_DIR \ + -ULIB_INSTALL_DIR \ + -USYSCONF_INSTALL_DIR \ + -USHARE_INSTALL_PREFIX \ + -ULIB_SUFFIX \ + -DCMAKE_INSTALL_LIBDIR="lib" \ + -DCMAKE_INSTALL_PREFIX="@(InstallationPrefix)" \ + -DCMAKE_PREFIX_PATH="@(InstallationPrefix)" \ + -DSETUPTOOLS_DEB_LAYOUT=OFF \ + -DCATKIN_BUILD_BINARY_PACKAGE="1" \ + .. %make_build From 68583c43c255b0274e9a48e165b0eb2412444426 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 22 Apr 2019 17:17:23 -0700 Subject: [PATCH 20/23] Disable dependency generation in RPM spec files There are two classes of issues here: 1. ROS packages often provide libraries which are also provided by the operating system. If an operating system declares a dependency on that library, we don't want the package manager to install the ROS package instead of the system package. 2. Many ROS packages don't install libraries in a way that the 'provides' portion of dependency generation can detect, so when another ROS package takes a dependency on that library, the automatic dependency can't be met and the downstream package cannot be installed. More info on RPM dependency generation: https://rpm.org/user_doc/dependency_generators.html Signed-off-by: Scott K Logan --- bloom/generators/rpm/templates/catkin/template.spec.em | 3 +++ bloom/generators/rpm/templates/cmake/template.spec.em | 3 +++ 2 files changed, 6 insertions(+) diff --git a/bloom/generators/rpm/templates/catkin/template.spec.em b/bloom/generators/rpm/templates/catkin/template.spec.em index 6606475f..7bf39afc 100644 --- a/bloom/generators/rpm/templates/catkin/template.spec.em +++ b/bloom/generators/rpm/templates/catkin/template.spec.em @@ -1,3 +1,6 @@ +%global __provides_exclude_from ^@(InstallationPrefix)/.*$ +%global __requires_exclude_from ^@(InstallationPrefix)/.*$ + Name: @(Package) Version: @(Version) Release: @(Inc)%{?dist} diff --git a/bloom/generators/rpm/templates/cmake/template.spec.em b/bloom/generators/rpm/templates/cmake/template.spec.em index 6606475f..7bf39afc 100644 --- a/bloom/generators/rpm/templates/cmake/template.spec.em +++ b/bloom/generators/rpm/templates/cmake/template.spec.em @@ -1,3 +1,6 @@ +%global __provides_exclude_from ^@(InstallationPrefix)/.*$ +%global __requires_exclude_from ^@(InstallationPrefix)/.*$ + Name: @(Package) Version: @(Version) Release: @(Inc)%{?dist} From 24eb6d42ab88d22a1545536dcf01c799b89e8cc4 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 22 Apr 2019 17:28:07 -0700 Subject: [PATCH 21/23] Disable automagic byte compilation in RPM spec files The BRP Python bytecompiler will always use the sytem's default Python interpreter, which may not be the interpreter we're targeting. Safest option is to disable the automagic byte compilation altogether. Note that this doesn't mean that python files won't ever be compiled, it just means that the catch-all policy at the end of the process won't attempt to compile anything which hasn't already been compiled. Signed-off-by: Scott K Logan --- bloom/generators/rpm/templates/catkin/template.spec.em | 1 + bloom/generators/rpm/templates/cmake/template.spec.em | 1 + 2 files changed, 2 insertions(+) diff --git a/bloom/generators/rpm/templates/catkin/template.spec.em b/bloom/generators/rpm/templates/catkin/template.spec.em index 7bf39afc..78513af9 100644 --- a/bloom/generators/rpm/templates/catkin/template.spec.em +++ b/bloom/generators/rpm/templates/catkin/template.spec.em @@ -1,3 +1,4 @@ +%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g') %global __provides_exclude_from ^@(InstallationPrefix)/.*$ %global __requires_exclude_from ^@(InstallationPrefix)/.*$ diff --git a/bloom/generators/rpm/templates/cmake/template.spec.em b/bloom/generators/rpm/templates/cmake/template.spec.em index 7bf39afc..78513af9 100644 --- a/bloom/generators/rpm/templates/cmake/template.spec.em +++ b/bloom/generators/rpm/templates/cmake/template.spec.em @@ -1,3 +1,4 @@ +%global __os_install_post %(echo '%{__os_install_post}' | sed -e 's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g') %global __provides_exclude_from ^@(InstallationPrefix)/.*$ %global __requires_exclude_from ^@(InstallationPrefix)/.*$ From 6bb02489d3191430ae6ae94961b4dad13c4bf8b0 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Mon, 22 Apr 2019 17:33:52 -0700 Subject: [PATCH 22/23] Use 'cmake3' macro in RPM spec files This change shouldn't modify the behavior in Fedora, where all current releases define 'cmake3' to be the same as 'cmake'. In RHEL 7, where cmake 2 is the default, we need to use the 'cmake3' macro to use the supplamental 'cmake3' executable instead of the system default. All current ROS releases except Indigo have a *minimum* cmake requirement of 3. Signed-off-by: Scott K Logan --- bloom/generators/rpm/templates/catkin/template.spec.em | 2 +- bloom/generators/rpm/templates/cmake/template.spec.em | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bloom/generators/rpm/templates/catkin/template.spec.em b/bloom/generators/rpm/templates/catkin/template.spec.em index 78513af9..816f949f 100644 --- a/bloom/generators/rpm/templates/catkin/template.spec.em +++ b/bloom/generators/rpm/templates/catkin/template.spec.em @@ -23,7 +23,7 @@ License: @(License) # set things like CMAKE_PREFIX_PATH, PKG_CONFIG_PATH, and PYTHONPATH. if [ -f "@(InstallationPrefix)/setup.sh" ]; then . "@(InstallationPrefix)/setup.sh"; fi mkdir -p obj-%{_target_platform} && cd obj-%{_target_platform} -%cmake \ +%cmake3 \ -UINCLUDE_INSTALL_DIR \ -ULIB_INSTALL_DIR \ -USYSCONF_INSTALL_DIR \ diff --git a/bloom/generators/rpm/templates/cmake/template.spec.em b/bloom/generators/rpm/templates/cmake/template.spec.em index 78513af9..816f949f 100644 --- a/bloom/generators/rpm/templates/cmake/template.spec.em +++ b/bloom/generators/rpm/templates/cmake/template.spec.em @@ -23,7 +23,7 @@ License: @(License) # set things like CMAKE_PREFIX_PATH, PKG_CONFIG_PATH, and PYTHONPATH. if [ -f "@(InstallationPrefix)/setup.sh" ]; then . "@(InstallationPrefix)/setup.sh"; fi mkdir -p obj-%{_target_platform} && cd obj-%{_target_platform} -%cmake \ +%cmake3 \ -UINCLUDE_INSTALL_DIR \ -ULIB_INSTALL_DIR \ -USYSCONF_INSTALL_DIR \ From 95afe33fda5f857d926effc51240cfc63d2b0b08 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Thu, 18 Jul 2019 15:35:16 -0700 Subject: [PATCH 23/23] Always create archive of sources when building RPM (#540) The SCM plugin for mock will create an archive of the sources if either this marker file is present in the root of the project or the `write_tar` option is specified in mock's configuration. The way bloom works, we always want to archive the sources, but this isn't the default configuration of the SCM plugin. Writing this marker file means one less configuration is necessary when consuming the release repo to build the RPMs with mock. Signed-off-by: Scott K Logan --- bloom/generators/rpm/generator.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bloom/generators/rpm/generator.py b/bloom/generators/rpm/generator.py index 29dca7d3..cf00c1bc 100644 --- a/bloom/generators/rpm/generator.py +++ b/bloom/generators/rpm/generator.py @@ -124,8 +124,10 @@ def generate_package(self, package, os_version): template_files = process_template_files('.', subs, self.package_manager) # Remove any residual template files execute_command('git rm -rf ' + ' '.join("'{}'".format(t) for t in template_files)) - # Add changes to the rpm folder - execute_command('git add ' + self.package_manager) + # Add marker file to tell mock to archive the sources + open('.write_tar', 'a').close() + # Add marker file changes to the rpm folder + execute_command('git add .write_tar ' + self.package_manager) # Commit changes execute_command('git commit -m "Generated {0} files for {1}"' .format(self.package_manager, os_version))