From 102736a8124cdb790315ef05daae5f1e86102d7e Mon Sep 17 00:00:00 2001 From: "kilian.funk" Date: Thu, 19 Sep 2024 09:50:50 -0700 Subject: [PATCH 1/2] Fix tar repo type for locking --- repos/config/detail/BUILD.bazel | 1 + repos/config/detail/generate_ros2_config.py | 6 +++--- repos/config/detail/lock_repos.py | 23 +++++++++++---------- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/repos/config/detail/BUILD.bazel b/repos/config/detail/BUILD.bazel index 927b7ea..498ad90 100644 --- a/repos/config/detail/BUILD.bazel +++ b/repos/config/detail/BUILD.bazel @@ -16,6 +16,7 @@ load("@python_deps//:requirements.bzl", "requirement") load("@ros2_config//:repos_index_file.bzl", "REPOS_INDEX_FILE") load("@ros2_config//:repos_overlay_files.bzl", "REPOS_OVERLAY_FILES") load("@ros2_config//:repos_setup_file.bzl", "REPOS_SETUP_FILE") +load("@rules_python//python:defs.bzl", "py_binary") py_binary( name = "repos_lock.update", diff --git a/repos/config/detail/generate_ros2_config.py b/repos/config/detail/generate_ros2_config.py index 62c9011..89e5114 100755 --- a/repos/config/detail/generate_ros2_config.py +++ b/repos/config/detail/generate_ros2_config.py @@ -54,7 +54,7 @@ def build_load_command(repo, spec): def build_http_archive_load_command(repo, spec): build_files = "\n".join([f' "{k}": "{v}",' for k,v in spec['bazel'].items()]) - return f""" + return f"""\ _maybe( name = "{repo.replace('/','.')}", build_files = {{ @@ -69,7 +69,7 @@ def build_http_archive_load_command(repo, spec): def build_local_load_command(repo, spec): build_files = "\n".join([f' "{k}": "{v}",' for k,v in spec['bazel'].items()]) - return f""" + return f"""\ _maybe( name = "{repo.replace('/','.')}", build_files = {{ @@ -83,7 +83,7 @@ def build_local_load_command(repo, spec): def build_git_load_command(repo, spec): build_files = "\n".join([f' "{k}": "{v}",' for k,v in spec['bazel'].items()]) - return f""" + return f"""\ _maybe( name = "{repo.replace('/','.')}", branch = "{spec['version']}", diff --git a/repos/config/detail/lock_repos.py b/repos/config/detail/lock_repos.py index 1bd8d0a..2981024 100755 --- a/repos/config/detail/lock_repos.py +++ b/repos/config/detail/lock_repos.py @@ -46,12 +46,7 @@ def main(): for repo, spec in repos["repositories"].items(): if not spec["type"] in REPO_TYPES: raise ValueError(f"Repo type {spec['type']} not supported. Need one of {REPO_TYPES} instead.") - if args.tar: - # use tarballs - additional_attributes = fetch_archive_details(spec['url'], spec['version']) - else: - # default: use git repositories - additional_attributes = fetch_repo_details(spec['url'], spec['version']) + additional_attributes = fetch_dependency_details(use_tar = args.tar, **spec) add_attributes(repos["repositories"][repo], additional_attributes) print("{}: {}".format(repo, [*additional_attributes.values()])) @@ -61,23 +56,29 @@ def main(): with open(args.setup_bzl, mode='w', encoding='utf8') as setup_bzl: print_setup_file(yaml_files=[lock_file.name] + args.overlays, output_file=setup_bzl) + +def fetch_dependency_details(*, use_tar, type, **kwargs): + if type == "tar" or use_tar: + return fetch_archive_details(**kwargs) + return fetch_repo_details(**kwargs) + def add_attributes(dictionary, additional_attributes): for k,v in additional_attributes.items(): dictionary[k] = v -def fetch_archive_details(url, tag): +def fetch_archive_details(url, version, **kwargs): cwd = os.getcwd() with tempfile.TemporaryDirectory() as tempdir: url = url.rsplit(".git", 1)[0] project = url.split('/')[-1] - url += f"/archive/refs/tags/{tag}.tar.gz" + url += f"/archive/refs/tags/{version}.tar.gz" result = subprocess.run( ["curl", "-L", "-f", "-o", "archive.tar.gz", url], stdout=subprocess.PIPE, encoding='utf8' ) if result.returncode != 0: - raise ValueError(f"Error loading {url}, {tag}: " + (result.stderr or "")) + raise ValueError(f"Error loading {url}, {version}: " + (result.stderr or "")) archive_bytes = Path('archive.tar.gz').read_bytes() hash = hashlib.sha256(archive_bytes).hexdigest() @@ -101,12 +102,12 @@ def extract_archive_root_folder(path, origin): return result.stdout.split('\n')[0].split('/')[0] -def fetch_repo_details(url, branch): +def fetch_repo_details(url, version, **kwargs): cwd = os.getcwd() with tempfile.TemporaryDirectory() as tempdir: result = subprocess.run( ["git", "clone", url, "--no-checkout", tempdir, "--depth", "1", - "--branch", branch, "--bare", "-q"], + "--branch", version, "--bare", "-q"], capture_output = True, encoding='utf8' ) From 2f60c98e311dbec2f3f5cb4793f4a793bdc53101 Mon Sep 17 00:00:00 2001 From: "kilian.funk" Date: Thu, 19 Sep 2024 17:05:54 -0700 Subject: [PATCH 2/2] Remove temp yaml files --- repos/config/detail/generate_ros2_config.py | 53 ++++++++++----------- repos/config/detail/lock_repos.py | 34 ++++++------- 2 files changed, 41 insertions(+), 46 deletions(-) diff --git a/repos/config/detail/generate_ros2_config.py b/repos/config/detail/generate_ros2_config.py index 89e5114..184315b 100755 --- a/repos/config/detail/generate_ros2_config.py +++ b/repos/config/detail/generate_ros2_config.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import yaml @@ -23,9 +22,9 @@ # load("@bazel_tools//tools/build_defs/repo:utils.bzl", _maybe = "maybe") -load("@rules_ros//repos/config/detail:git_repository.bzl", _git_repository = "git_repository") +load("@rules_ros//repos/config/detail:git_repository.bzl", "git_repository") load("@rules_ros//repos/config/detail:http_archive.bzl", "http_archive") -load("@rules_ros//repos/config/detail:new_local_repository.bzl", _new_local_repository = "new_local_repository") +load("@rules_ros//repos/config/detail:new_local_repository.bzl", "new_local_repository") def setup(): pass @@ -40,26 +39,32 @@ def print_setup(repos, output_file): def build_load_command(repo, spec): - if spec.get('type') == "git": - return build_git_load_command(repo, spec) - if spec.get('type') == "http_archive": - return build_http_archive_load_command(repo, spec) - if spec.get('type') == "local": - return build_local_load_command(repo, spec) - else: + builder = { + "git": build_git_load_command, + "http_archive": build_http_archive_load_command, + "local": build_local_load_command, + } + if spec.get('type') not in builder.keys(): return f""" print("WARNING: Unknown repo type {spec.get('type')} for repo @{repo.replace('/', '.')}") """ + return builder[spec.get('type')](repo, spec) + + +def build_build_files_attr(build_files): + if not build_files: + return "" + content = '\n'.join(f" '{k}': '{v}'," for k,v in build_files.items()) + return f"""build_files = {{ +{content} + }},""" def build_http_archive_load_command(repo, spec): - build_files = "\n".join([f' "{k}": "{v}",' for k,v in spec['bazel'].items()]) return f"""\ _maybe( name = "{repo.replace('/','.')}", - build_files = {{ -{build_files} - }}, + {build_build_files_attr(spec['bazel'])} url = "{spec['url']}", sha256 = "{spec['hash']}", strip_prefix = "{spec['strip_prefix']}", @@ -68,31 +73,25 @@ def build_http_archive_load_command(repo, spec): """ def build_local_load_command(repo, spec): - build_files = "\n".join([f' "{k}": "{v}",' for k,v in spec['bazel'].items()]) return f"""\ _maybe( name = "{repo.replace('/','.')}", - build_files = {{ -{build_files} - }}, + {build_build_files_attr(spec['bazel'])} path = "{spec['path']}", sha256 = "{spec['hash']}", - repo_rule = _new_local_repository, + repo_rule = new_local_repository, ) """ def build_git_load_command(repo, spec): - build_files = "\n".join([f' "{k}": "{v}",' for k,v in spec['bazel'].items()]) return f"""\ _maybe( name = "{repo.replace('/','.')}", branch = "{spec['version']}", - build_files = {{ -{build_files} - }}, + {build_build_files_attr(spec['bazel'])} commit = "{spec['hash']}", remote = "{spec['url']}", - repo_rule = _git_repository, + repo_rule = git_repository, shallow_since = "{spec['shallow_since']}", ) """ @@ -106,11 +105,7 @@ def merge_dict(origin, to_add): origin[key]=value -def print_setup_file(yaml_files, output_file): - if len(yaml_files) < 1: - raise ValueError("At least one YAML file is required as input.") - - repos = {} +def print_setup_file(repos, yaml_files, output_file): for input_path in yaml_files: with (open(input_path,"r")) as repo_file: merge_dict(repos, yaml.safe_load(repo_file)["repositories"]) diff --git a/repos/config/detail/lock_repos.py b/repos/config/detail/lock_repos.py index 2981024..08eebc4 100755 --- a/repos/config/detail/lock_repos.py +++ b/repos/config/detail/lock_repos.py @@ -50,45 +50,45 @@ def main(): add_attributes(repos["repositories"][repo], additional_attributes) print("{}: {}".format(repo, [*additional_attributes.values()])) - with tempfile.NamedTemporaryFile(mode='w+t', encoding='utf8') as lock_file: - yaml.dump(repos, lock_file, default_flow_style=False, allow_unicode=True) - - with open(args.setup_bzl, mode='w', encoding='utf8') as setup_bzl: - print_setup_file(yaml_files=[lock_file.name] + args.overlays, output_file=setup_bzl) + with open(args.setup_bzl, mode='w', encoding='utf8') as setup_bzl: + print_setup_file(repos = repos["repositories"], yaml_files=args.overlays, output_file=setup_bzl) def fetch_dependency_details(*, use_tar, type, **kwargs): if type == "tar" or use_tar: - return fetch_archive_details(**kwargs) - return fetch_repo_details(**kwargs) + return fetch_http_details(type = type, **kwargs) + return fetch_git_details(type = type, **kwargs) def add_attributes(dictionary, additional_attributes): for k,v in additional_attributes.items(): dictionary[k] = v -def fetch_archive_details(url, version, **kwargs): - cwd = os.getcwd() +def fetch_http_details(*, type, version = None, url = None, **kwargs): + forward_type = "http_archive" + if "sha256" in kwargs: + return { "type": forward_type} with tempfile.TemporaryDirectory() as tempdir: - url = url.rsplit(".git", 1)[0] - project = url.split('/')[-1] - url += f"/archive/refs/tags/{version}.tar.gz" + archive_filename = Path(tempdir) / "archive.tar.gz" + if type == "git": + url = url.rsplit(".git", 1)[0] + url += f"/archive/refs/tags/{version}.tar.gz" result = subprocess.run( - ["curl", "-L", "-f", "-o", "archive.tar.gz", url], + ["curl", "-L", "-f", "-o", archive_filename, url], stdout=subprocess.PIPE, encoding='utf8' ) if result.returncode != 0: raise ValueError(f"Error loading {url}, {version}: " + (result.stderr or "")) - archive_bytes = Path('archive.tar.gz').read_bytes() + archive_bytes = archive_filename.read_bytes() hash = hashlib.sha256(archive_bytes).hexdigest() - strip_prefix = extract_archive_root_folder("archive.tar.gz", url) + strip_prefix = extract_archive_root_folder(archive_filename, url) return { "hash":hash, "url": url, "strip_prefix": strip_prefix, - "type": "http_archive", + "type": forward_type, } def extract_archive_root_folder(path, origin): @@ -102,7 +102,7 @@ def extract_archive_root_folder(path, origin): return result.stdout.split('\n')[0].split('/')[0] -def fetch_repo_details(url, version, **kwargs): +def fetch_git_details(url, version, **kwargs): cwd = os.getcwd() with tempfile.TemporaryDirectory() as tempdir: result = subprocess.run(