From 354bf97fe91c0286d0e761d2101f0afcbd19cd39 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 18 Dec 2024 14:13:01 -0500 Subject: [PATCH 1/4] install compression tools as core deps --- bbot/core/helpers/depsinstaller/installer.py | 93 ++++++++++++++------ 1 file changed, 66 insertions(+), 27 deletions(-) diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 9938034639..1537a1b270 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -20,11 +20,42 @@ class DepsInstaller: + CORE_DEPS = { + # core BBOT dependencies in the format of binary: package_name + # each one will only be installed if the binary is not found + "unzip": "unzip", + "zipinfo": "unzip", + "curl": "curl", + "git": "git", + "make": "make", + "gcc": "gcc", + "bash": "bash", + "which": "which", + "unrar": "unrar", + "tar": "tar", + "7z": [ + { + "name": "Install 7zip (Debian)", + "package": {"name": ["p7zip-full"], "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] == 'Debian'", + }, + { + "name": "Install 7zip (Non-Debian)", + "package": {"name": ["p7zip"], "state": "present"}, + "become": True, + "when": "ansible_facts['os_family'] != 'Debian'", + }, + ], + } + def __init__(self, parent_helper): self.parent_helper = parent_helper self.preset = self.parent_helper.preset self.core = self.preset.core + self.os_platform = os_platform() + # respect BBOT's http timeout self.web_config = self.parent_helper.config.get("web", {}) http_timeout = self.web_config.get("http_timeout", 30) @@ -202,28 +233,32 @@ def apt_install(self, packages): """ Install packages with the OS's default package manager (apt, pacman, dnf, etc.) """ - packages_str = ",".join(packages) + args, kwargs = self._make_apt_ansible_args(packages) + success, err = self.ansible_run(module="package", args=args, **kwargs) + if success: + log.info(f'Successfully installed OS packages "{",".join(sorted(packages))}"') + else: + log.warning( + f"Failed to install OS packages ({err}). Recommend installing the following packages manually:" + ) + for p in packages: + log.warning(f" - {p}") + return success + + def _make_apt_ansible_args(self, packages): + packages_str = ",".join(sorted(packages)) log.info(f"Installing the following OS packages: {packages_str}") args = {"name": packages_str, "state": "present"} # , "update_cache": True, "cache_valid_time": 86400} kwargs = {} # don't sudo brew - if os_platform() != "darwin": + if self.os_platform != "darwin": kwargs = { "ansible_args": { "ansible_become": True, "ansible_become_method": "sudo", } } - success, err = self.ansible_run(module="package", args=args, **kwargs) - if success: - log.info(f'Successfully installed OS packages "{packages_str}"') - else: - log.warning( - f"Failed to install OS packages ({err}). Recommend installing the following packages manually:" - ) - for p in packages: - log.warning(f" - {p}") - return success + return args, kwargs def shell(self, module, commands): tasks = [] @@ -269,7 +304,7 @@ def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None): for task in tasks: if "package" in task: # special case for macos - if os_platform() == "darwin": + if self.os_platform == "darwin": # don't sudo brew task["become"] = False # brew doesn't support update_cache @@ -347,26 +382,30 @@ def ensure_root(self, message=""): def install_core_deps(self): to_install = set() + to_install_friendly = set() + playbook = [] self._install_sudo_askpass() # ensure tldextract data is cached self.parent_helper.tldextract("evilcorp.co.uk") - # command: package_name - core_deps = { - "unzip": "unzip", - "zipinfo": "unzip", - "curl": "curl", - "git": "git", - "make": "make", - "gcc": "gcc", - "bash": "bash", - "which": "which", - } - for command, package_name in core_deps.items(): + for command, package_name_or_playbook in self.CORE_DEPS.items(): if not self.parent_helper.which(command): - to_install.add(package_name) + to_install_friendly.add(command) + if isinstance(package_name_or_playbook, str): + to_install.add(package_name_or_playbook) + else: + playbook.extend(package_name_or_playbook) if to_install: + playbook.append( + { + "name": "Install Core BBOT Dependencies", + "package": {"name": list(to_install), "state": "present"}, + "become": True, + } + ) + if playbook: + self.log.info(f"Installing core BBOT dependencies: {",".join(sorted(to_install_friendly))}") self.ensure_root() - self.apt_install(list(to_install)) + self.ansible_run(tasks=playbook) def _setup_sudo_cache(self): if not self._sudo_cache_setup: From f1c487ed46a9ac7986d999d121a5f944aae9a373 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 18 Dec 2024 15:25:49 -0500 Subject: [PATCH 2/4] fix tests --- bbot/core/helpers/depsinstaller/installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 1537a1b270..00d91f34d0 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -403,7 +403,7 @@ def install_core_deps(self): } ) if playbook: - self.log.info(f"Installing core BBOT dependencies: {",".join(sorted(to_install_friendly))}") + log.info(f"Installing core BBOT dependencies: {','.join(sorted(to_install_friendly))}") self.ensure_root() self.ansible_run(tasks=playbook) From 681941139de93f13f76ce6a4711c832b1ee85e04 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 18 Dec 2024 16:52:09 -0500 Subject: [PATCH 3/4] silence ansible --- bbot/core/helpers/depsinstaller/installer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 00d91f34d0..3bc2e4553d 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -327,8 +327,8 @@ def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None): }, module=module, module_args=module_args, - quiet=not self.ansible_debug, - verbosity=(3 if self.ansible_debug else 0), + quiet=True, + verbosity=0, cancel_callback=lambda: None, ) @@ -338,7 +338,7 @@ def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None): err = "" for e in res.events: if self.ansible_debug and not success: - log.debug(json.dumps(e, indent=4)) + log.debug(json.dumps(e, indent=2)) if e["event"] == "runner_on_failed": err = e["event_data"]["res"]["msg"] break From 48bf17b68e71cb4fe4a01d0a9a72be50634a3535 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 19 Dec 2024 16:11:32 -0500 Subject: [PATCH 4/4] unrar-free --- bbot/core/helpers/depsinstaller/installer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py index 3bc2e4553d..48d2f970fa 100644 --- a/bbot/core/helpers/depsinstaller/installer.py +++ b/bbot/core/helpers/depsinstaller/installer.py @@ -31,8 +31,9 @@ class DepsInstaller: "gcc": "gcc", "bash": "bash", "which": "which", - "unrar": "unrar", + "unrar": "unrar-free", "tar": "tar", + # debian why are you like this "7z": [ { "name": "Install 7zip (Debian)",