Merge pull request #2096 from blacklanternsecurity/compression-core-deps

Install compression tools as core dependencies

bbot/core/helpers/depsinstaller/installer.py

@@ -20,11 +20,43 @@
 
 
 class DepsInstaller:
+    CORE_DEPS = {
+        # core BBOT dependencies in the format of binary: package_name
+        # each one will only be installed if the binary is not found
+        "unzip": "unzip",
+        "zipinfo": "unzip",
+        "curl": "curl",
+        "git": "git",
+        "make": "make",
+        "gcc": "gcc",
+        "bash": "bash",
+        "which": "which",
+        "unrar": "unrar-free",
+        "tar": "tar",
+        # debian why are you like this
+        "7z": [
+            {
+                "name": "Install 7zip (Debian)",
+                "package": {"name": ["p7zip-full"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] == 'Debian'",
+            },
+            {
+                "name": "Install 7zip (Non-Debian)",
+                "package": {"name": ["p7zip"], "state": "present"},
+                "become": True,
+                "when": "ansible_facts['os_family'] != 'Debian'",
+            },
+        ],
+    }
+
     def __init__(self, parent_helper):
         self.parent_helper = parent_helper
         self.preset = self.parent_helper.preset
         self.core = self.preset.core
 
+        self.os_platform = os_platform()
+
         # respect BBOT's http timeout
         self.web_config = self.parent_helper.config.get("web", {})
         http_timeout = self.web_config.get("http_timeout", 30)
@@ -202,28 +234,32 @@ def apt_install(self, packages):
         """
         Install packages with the OS's default package manager (apt, pacman, dnf, etc.)
         """
-        packages_str = ",".join(packages)
+        args, kwargs = self._make_apt_ansible_args(packages)
+        success, err = self.ansible_run(module="package", args=args, **kwargs)
+        if success:
+            log.info(f'Successfully installed OS packages "{",".join(sorted(packages))}"')
+        else:
+            log.warning(
+                f"Failed to install OS packages ({err}). Recommend installing the following packages manually:"
+            )
+            for p in packages:
+                log.warning(f" - {p}")
+        return success
+
+    def _make_apt_ansible_args(self, packages):
+        packages_str = ",".join(sorted(packages))
         log.info(f"Installing the following OS packages: {packages_str}")
         args = {"name": packages_str, "state": "present"}  # , "update_cache": True, "cache_valid_time": 86400}
         kwargs = {}
         # don't sudo brew
-        if os_platform() != "darwin":
+        if self.os_platform != "darwin":
             kwargs = {
                 "ansible_args": {
                     "ansible_become": True,
                     "ansible_become_method": "sudo",
                 }
             }
-        success, err = self.ansible_run(module="package", args=args, **kwargs)
-        if success:
-            log.info(f'Successfully installed OS packages "{packages_str}"')
-        else:
-            log.warning(
-                f"Failed to install OS packages ({err}). Recommend installing the following packages manually:"
-            )
-            for p in packages:
-                log.warning(f" - {p}")
-        return success
+        return args, kwargs
 
     def shell(self, module, commands):
         tasks = []
@@ -269,7 +305,7 @@ def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):
             for task in tasks:
                 if "package" in task:
                     # special case for macos
-                    if os_platform() == "darwin":
+                    if self.os_platform == "darwin":
                         # don't sudo brew
                         task["become"] = False
                         # brew doesn't support update_cache
@@ -292,8 +328,8 @@ def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):
             },
             module=module,
             module_args=module_args,
-            quiet=not self.ansible_debug,
-            verbosity=(3 if self.ansible_debug else 0),
+            quiet=True,
+            verbosity=0,
             cancel_callback=lambda: None,
         )
 
@@ -303,7 +339,7 @@ def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):
         err = ""
         for e in res.events:
             if self.ansible_debug and not success:
-                log.debug(json.dumps(e, indent=4))
+                log.debug(json.dumps(e, indent=2))
             if e["event"] == "runner_on_failed":
                 err = e["event_data"]["res"]["msg"]
                 break
@@ -347,26 +383,30 @@ def ensure_root(self, message=""):
 
     def install_core_deps(self):
         to_install = set()
+        to_install_friendly = set()
+        playbook = []
         self._install_sudo_askpass()
         # ensure tldextract data is cached
         self.parent_helper.tldextract("evilcorp.co.uk")
-        # command: package_name
-        core_deps = {
-            "unzip": "unzip",
-            "zipinfo": "unzip",
-            "curl": "curl",
-            "git": "git",
-            "make": "make",
-            "gcc": "gcc",
-            "bash": "bash",
-            "which": "which",
-        }
-        for command, package_name in core_deps.items():
+        for command, package_name_or_playbook in self.CORE_DEPS.items():
             if not self.parent_helper.which(command):
-                to_install.add(package_name)
+                to_install_friendly.add(command)
+                if isinstance(package_name_or_playbook, str):
+                    to_install.add(package_name_or_playbook)
+                else:
+                    playbook.extend(package_name_or_playbook)
         if to_install:
+            playbook.append(
+                {
+                    "name": "Install Core BBOT Dependencies",
+                    "package": {"name": list(to_install), "state": "present"},
+                    "become": True,
+                }
+            )
+        if playbook:
+            log.info(f"Installing core BBOT dependencies: {','.join(sorted(to_install_friendly))}")
             self.ensure_root()
-            self.apt_install(list(to_install))
+            self.ansible_run(tasks=playbook)
 
     def _setup_sudo_cache(self):
         if not self._sudo_cache_setup: