From fdd742a6c75ac0e2525790d08ec43846c6b8d8b2 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Thu, 18 Jul 2024 09:33:29 +0200 Subject: [PATCH 1/5] Fix test use of `sed` for macos This fixes the invocation of `sed` in the test to be compatible with macos' `sed` version. Previously this test would fail with ``` sed: 1: "./install-dir/bin/zkg": invalid command code ``` --- testing/tests/bundled-zkg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/tests/bundled-zkg b/testing/tests/bundled-zkg index 0809d883..0508ca44 100644 --- a/testing/tests/bundled-zkg +++ b/testing/tests/bundled-zkg @@ -9,7 +9,7 @@ set -eu mkdir -p install-dir/{bin,lib/python} cp -R $TEST_BASE/../zeekpkg ./install-dir/lib/python cp $TEST_BASE/../zkg ./install-dir/bin -sed -i "s,@PY_MOD_INSTALL_DIR@,$(pwd)/install-dir/lib/python," ./install-dir/bin/zkg +sed -i -e "s,@PY_MOD_INSTALL_DIR@,$(pwd)/install-dir/lib/python," ./install-dir/bin/zkg export PATH=$(pwd)/install-dir/bin:$PATH # Ensure we're using the right zkg From eed5c491212093201d4c1f758b0cd2857ae34d84 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Thu, 18 Jul 2024 09:59:21 +0200 Subject: [PATCH 2/5] Add installation workaround for tar files with macos metadata --- zeekpkg/manager.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index a78a99a3..9a8d14db 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -3240,7 +3240,15 @@ def _copy_package_dir(package, dirname, src, dst, scratch_dir): ld = os.listdir(tmp_dir) if len(ld) != 1: - return f"failed to copy package {dirname}: invalid tarfile" + # Apple `tar` might store HFS+ extended metadata in tar files. + # These metadata files have the names `._FOO` for each entry `FOO`. + # Since we expect a single top-level directory for the extracted + # plugin, ignore the metadata file if we see it. + ld.sort() + if len(ld) == 2 and ld[0] == f"._{ld[1]}": + ld = ld[1:] + else: + return f"failed to copy package {dirname}: invalid tarfile" src = os.path.join(tmp_dir, ld[0]) From e39aedd577ed63149c97f1e48a012adcb7946f56 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Thu, 18 Jul 2024 09:29:57 +0200 Subject: [PATCH 3/5] Use f-strings in more places --- doc/ext/sphinxarg/ext.py | 14 ++++++++------ doc/ext/sphinxarg/parser.py | 5 +++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/doc/ext/sphinxarg/ext.py b/doc/ext/sphinxarg/ext.py index e26ce391..dd6d5b05 100644 --- a/doc/ext/sphinxarg/ext.py +++ b/doc/ext/sphinxarg/ext.py @@ -42,7 +42,7 @@ def map_nested_definitions(nested_content): "@before", "@after", ): - raise Exception("Unknown classifier: %s" % classifier) + raise Exception(f"Unknown classifier: {classifier}") idx = subitem.first_child_matching_class(nodes.term) if idx is not None: ch = subitem[idx] @@ -70,8 +70,9 @@ def print_arg_list(data, nested_content): my_def.append( nodes.paragraph( text=( - "Possible choices: %s" - % ", ".join([str(c) for c in arg["choices"]]) + "Possible choices: {}".format( + ", ".join([str(c) for c in arg["choices"]]), + ) ), ), ) @@ -115,8 +116,9 @@ def print_opt_list(data, nested_content): my_def.append( nodes.paragraph( text=( - "Possible choices: %s" - % ", ".join([str(c) for c in opt["choices"]]) + "Possible choices: {}".format( + ", ".join([str(c) for c in opt["choices"]]), + ) ), ), ) @@ -169,7 +171,7 @@ def apply_definition(definitions, my_def, name): return my_def + definition.children if classifier == "@before": return definition.children + my_def - raise Exception("Unknown classifier: %s" % classifier) + raise Exception(f"Unknown classifier: {classifier}") return my_def diff --git a/doc/ext/sphinxarg/parser.py b/doc/ext/sphinxarg/parser.py index 85a94aa6..8b52a23f 100644 --- a/doc/ext/sphinxarg/parser.py +++ b/doc/ext/sphinxarg/parser.py @@ -16,8 +16,9 @@ def parser_navigate(parser_result, path, current_path=None): return parser_result if "children" not in parser_result: raise NavigationError( - "Current parser have no children elements. (path: %s)" - % " ".join(current_path), + "Current parser have no children elements. (path: {})".format( + " ".join(current_path), + ), ) next_hop = path.pop(0) for child in parser_result["children"]: From bd7d994c7b391356e797129c2903fba82b47a610 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Thu, 18 Jul 2024 09:27:30 +0200 Subject: [PATCH 4/5] Bump pre-commit hooks --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a6ab9133..aa3a70ee 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: - id: check-yaml - id: check-added-large-files - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.7 + rev: v0.5.2 hooks: - id: ruff args: [--fix] From db1446671301f1e4b672401fbc0116cb4a743a37 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Thu, 18 Jul 2024 09:27:12 +0200 Subject: [PATCH 5/5] Use documented Git package exception API --- zeekpkg/_util.py | 6 +++--- zeekpkg/manager.py | 32 ++++++++++++++++---------------- zeekpkg/source.py | 6 +++--- zeekpkg/template.py | 8 ++++---- zkg | 6 +++--- 5 files changed, 29 insertions(+), 29 deletions(-) diff --git a/zeekpkg/_util.py b/zeekpkg/_util.py index e7f96a44..2776a29e 100644 --- a/zeekpkg/_util.py +++ b/zeekpkg/_util.py @@ -132,7 +132,7 @@ def git_clone(git_url, dst_path, shallow=False): recursive=True, depth=1, ) - except git.exc.GitCommandError: + except git.GitCommandError: if not git_url.startswith(".") and not git_url.startswith("/"): # Not a local repo raise @@ -171,7 +171,7 @@ def git_checkout(clone, version): version (str): the branch, tag, or commit to checkout Raises: - git.exc.GitCommandError: if the git repo is invalid + git.GitCommandError: if the git repo is invalid """ clone.git.checkout(version) clone.git.submodule("sync", "--recursive") @@ -255,7 +255,7 @@ def git_pull(repo): clone (git.Repo): the git clone on which to operate Raises: - git.exc.GitCommandError: in case of git trouble + git.GitCommandError: in case of git trouble """ repo.git.pull() repo.git.submodule("sync", "--recursive") diff --git a/zeekpkg/manager.py b/zeekpkg/manager.py index 9a8d14db..93ef5ea8 100644 --- a/zeekpkg/manager.py +++ b/zeekpkg/manager.py @@ -580,7 +580,7 @@ def add_source(self, name, git_url): git_url=git_url, version=version, ) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: LOG.warning("failed to clone git repo: %s", error) return "failed to clone git repo" else: @@ -1068,7 +1068,7 @@ def _refresh_source(self, name, aggregate=False, push=False): try: source.clone.git.fetch("--recurse-submodules=yes") git_pull(source.clone) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: LOG.error("failed to pull source %s: %s", name, error) return self.SourceAggregationResults( f"failed to pull from remote source: {error}", @@ -1125,7 +1125,7 @@ def _refresh_source(self, name, aggregate=False, push=False): try: clone = git_clone(url, clonepath, shallow=True) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: LOG.warn( "failed to clone %s, skipping aggregation: %s", url, @@ -1143,7 +1143,7 @@ def _refresh_source(self, name, aggregate=False, push=False): try: git_checkout(clone, version) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: LOG.warn( 'failed to checkout branch/version "%s" of %s, ' "skipping aggregation: %s", @@ -1264,7 +1264,7 @@ def refresh_installed_packages(self): try: clone.git.fetch("--recurse-submodules=yes") - except git.exc.GitCommandError as error: + except git.GitCommandError as error: LOG.warn( "failed to fetch package %s: %s", ipkg.package.qualified_name(), @@ -1859,7 +1859,7 @@ def info(self, pkg_path, version="", prefer_installed=True): try: return self._info(package, status, version) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: LOG.info( 'getting info on "%s": invalid git repo path: %s', pkg_path, @@ -1892,7 +1892,7 @@ def info(self, pkg_path, version="", prefer_installed=True): try: return self._info(package, status, version) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: LOG.info('getting info on "%s": invalid git repo path: %s', pkg_path, error) reason = "git repository is either invalid or unreachable" return PackageInfo(package=package, invalid_reason=reason, status=status) @@ -1904,7 +1904,7 @@ def _info(self, package, status, version): A :class:`.package.PackageInfo` object. Raises: - git.exc.GitCommandError: when failing to clone the package repo + git.GitCommandError: when failing to clone the package repo """ clonepath = os.path.join(self.scratch_dir, package.name) clone = _clone_package(package, clonepath, version) @@ -1918,7 +1918,7 @@ def _info(self, package, status, version): try: git_checkout(clone, version) - except git.exc.GitCommandError: + except git.GitCommandError: reason = f'no such commit, branch, or version tag: "{version}"' return PackageInfo(package=package, status=status, invalid_reason=reason) @@ -2421,7 +2421,7 @@ def match_package_url_and_version(git_url, version): try: git_clone(git_url, clonepath, shallow=(not is_sha1(version))) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: return f"failed to clone {git_url}: {error}" # Record the built-in packages expected by this bundle (or simply @@ -2607,7 +2607,7 @@ def test(self, pkg_path, version="", test_dependencies=False): try: clone = _clone_package(info.package, clonepath, version) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: LOG.warning("failed to clone git repo: %s", error) return ( f"failed to clone {info.package.git_url}", @@ -2617,7 +2617,7 @@ def test(self, pkg_path, version="", test_dependencies=False): try: git_checkout(clone, version) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: LOG.warning("failed to checkout git repo version: %s", error) return ( f"failed to checkout {version} of {info.package.git_url}", @@ -2941,7 +2941,7 @@ def install(self, pkg_path, version=""): try: package = Package(git_url=pkg_path) return self._install(package, version) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: LOG.info('installing "%s": invalid git repo path: %s', pkg_path, error) LOG.info('installing "%s": matched no source package', pkg_path) @@ -2962,7 +2962,7 @@ def install(self, pkg_path, version=""): try: return self._install(matches[0], version) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: LOG.warning('installing "%s": source package git repo is invalid', pkg_path) return f'failed to clone package "{pkg_path}": {error}' @@ -3022,7 +3022,7 @@ def _install(self, package, version, use_existing_clone=False): string explaining why it failed. Raises: - git.exc.GitCommandError: if the git repo is invalid + git.GitCommandError: if the git repo is invalid IOError: if the package manifest file can't be written """ clonepath = os.path.join(self.package_clonedir, package.name) @@ -3297,7 +3297,7 @@ def _clone_package(package, clonepath, version): git.Repo: the cloned package Raises: - git.exc.GitCommandError: if the git repo is invalid + git.GitCommandError: if the git repo is invalid """ delete_path(clonepath) shallow = not is_sha1(version) diff --git a/zeekpkg/source.py b/zeekpkg/source.py index 0fa76bab..ff5a521d 100644 --- a/zeekpkg/source.py +++ b/zeekpkg/source.py @@ -42,7 +42,7 @@ def __init__(self, name, clone_path, git_url, version=None): """Create a package source. Raises: - git.exc.GitCommandError: if the git repo is invalid + git.GitCommandError: if the git repo is invalid OSError: if the git repo is invalid and can't be re-initialized """ git_url = os.path.expanduser(git_url) @@ -52,10 +52,10 @@ def __init__(self, name, clone_path, git_url, version=None): try: self.clone = git.Repo(clone_path) - except git.exc.NoSuchPathError: + except git.NoSuchPathError: LOG.debug('creating source clone of "%s" at %s', name, clone_path) self.clone = git_clone(git_url, clone_path, shallow=True) - except git.exc.InvalidGitRepositoryError: + except git.InvalidGitRepositoryError: LOG.debug('deleting invalid source clone of "%s" at %s', name, clone_path) shutil.rmtree(clone_path) self.clone = git_clone(git_url, clone_path, shallow=True) diff --git a/zeekpkg/template.py b/zeekpkg/template.py index 6e723de1..3087d64f 100644 --- a/zeekpkg/template.py +++ b/zeekpkg/template.py @@ -105,7 +105,7 @@ def load(config, template, version=None): repo = git.Repo(template) if not repo.is_dirty(): version = repo.head.ref.commit.hexsha[:8] - except git.exc.InvalidGitRepositoryError: + except git.InvalidGitRepositoryError: pass templatedir = template else: @@ -139,7 +139,7 @@ def load(config, template, version=None): repo = None if repo is None: repo = git_clone(template, templatedir) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: msg = f'failed to update template "{template}": {error}' LOG.error(msg) raise GitError(msg) from error @@ -154,7 +154,7 @@ def load(config, template, version=None): try: git_checkout(repo, version) - except git.exc.GitCommandError as error: + except git.GitCommandError as error: msg = f'failed to checkout branch/version "{version}" of template {template}: {error}' LOG.warn(msg) raise GitError(msg) from error @@ -168,7 +168,7 @@ def load(config, template, version=None): git_pull(repo) except TypeError: pass # Not on a branch, do nothing - except git.exc.GitCommandError as error: + except git.GitCommandError as error: msg = f'failed to update branch "{version}" of template {template}: {error}' LOG.warning(msg) raise GitError(msg) from error diff --git a/zkg b/zkg index b65f58e2..eb084494 100755 --- a/zkg +++ b/zkg @@ -309,7 +309,7 @@ def create_config(args, configfile): def active_git_branch(path): try: repo = git.Repo(path) - except git.exc.NoSuchPathError: + except git.NoSuchPathError: return None if not repo.working_tree_dir: @@ -341,7 +341,7 @@ def is_local_git_repo(git_url): # accessible, (2) it's not the root directory of a git repo. git.Repo(git_url) return True - except (git.exc.InvalidGitRepositoryError, git.exc.NoSuchPathError): + except (git.InvalidGitRepositoryError, git.NoSuchPathError): return False @@ -350,7 +350,7 @@ def is_local_git_repo_dirty(git_url): return False try: repo = git.Repo(git_url) - except git.exc.NoSuchPathError: + except git.NoSuchPathError: return False return repo.is_dirty(untracked_files=True)