Shared Ansible Dependencies

bbot/core/helpers/depsinstaller/installer.py

@@ -148,6 +148,20 @@ async def install_module(self, module):
         if deps_pip:
             success &= await self.pip_install(deps_pip, constraints=deps_pip_constraints)
 
+        # shared/common
+        deps_common = preloaded["deps"]["common"]
+        if deps_common:
+            for dep_common in deps_common:
+                if self.setup_status.get(dep_common, False) == True:
+                    log.critical(
+                        f'Skipping installation of dependency "{dep_common}" for module "{module}" since it is already installed'
+                    )
+                    continue
+                ansible_tasks = self.preset.module_loader._shared_deps[dep_common]
+                result = self.tasks(module, ansible_tasks)
+                self.setup_status[dep_common] = result
+                success &= result
+
         return success
 
     async def pip_install(self, packages, constraints=None):

bbot/core/modules.py

@@ -13,9 +13,11 @@
 from contextlib import suppress
 
 from bbot.core import CORE
+from bbot.errors import BBOTError
 from bbot.logger import log_to_stderr
 
 from .flags import flag_descriptions
+from .shared_deps import SHARED_DEPS
 from .helpers.misc import list_files, sha1, search_dict_by_key, search_format_dict, make_table, os_platform, mkdir
 
 
@@ -43,6 +45,8 @@ class ModuleLoader:
     def __init__(self):
         self.core = CORE
 
+        self._shared_deps = dict(SHARED_DEPS)
+
         self.__preloaded = {}
         self._modules = {}
         self._configs = {}
@@ -250,6 +254,7 @@ def configs(self, type=None):
 
     def find_and_replace(self, **kwargs):
         self.__preloaded = search_format_dict(self.__preloaded, **kwargs)
+        self._shared_deps = search_format_dict(self._shared_deps, **kwargs)
 
     def check_type(self, module, type):
         return self._preloaded[module]["type"] == type
@@ -312,6 +317,7 @@ def preload_module(self, module_file):
         deps_pip_constraints = []
         deps_shell = []
         deps_apt = []
+        deps_common = []
         ansible_tasks = []
         python_code = open(module_file).read()
         # take a hash of the code so we can keep track of when it changes
@@ -330,10 +336,10 @@ def preload_module(self, module_file):
                         if any([target.id == "options" for target in class_attr.targets]):
                             config.update(ast.literal_eval(class_attr.value))
                         # module options
-                        if any([target.id == "options_desc" for target in class_attr.targets]):
+                        elif any([target.id == "options_desc" for target in class_attr.targets]):
                             options_desc.update(ast.literal_eval(class_attr.value))
                         # module metadata
-                        if any([target.id == "meta" for target in class_attr.targets]):
+                        elif any([target.id == "meta" for target in class_attr.targets]):
                             meta = ast.literal_eval(class_attr.value)
 
                     # class attributes that are lists
@@ -344,27 +350,27 @@ def preload_module(self, module_file):
                                 if type(flag.value) == str:
                                     flags.add(flag.value)
                         # watched events
-                        if any([target.id == "watched_events" for target in class_attr.targets]):
+                        elif any([target.id == "watched_events" for target in class_attr.targets]):
                             for event_type in class_attr.value.elts:
                                 if type(event_type.value) == str:
                                     watched_events.add(event_type.value)
                         # produced events
-                        if any([target.id == "produced_events" for target in class_attr.targets]):
+                        elif any([target.id == "produced_events" for target in class_attr.targets]):
                             for event_type in class_attr.value.elts:
                                 if type(event_type.value) == str:
                                     produced_events.add(event_type.value)
 
                         # bbot module dependencies
-                        if any([target.id == "deps_modules" for target in class_attr.targets]):
+                        elif any([target.id == "deps_modules" for target in class_attr.targets]):
                             for dep_module in class_attr.value.elts:
                                 if type(dep_module.value) == str:
                                     deps_modules.add(dep_module.value)
                         # python dependencies
-                        if any([target.id == "deps_pip" for target in class_attr.targets]):
+                        elif any([target.id == "deps_pip" for target in class_attr.targets]):
                             for dep_pip in class_attr.value.elts:
                                 if type(dep_pip.value) == str:
                                     deps_pip.append(dep_pip.value)
-                        if any([target.id == "deps_pip_constraints" for target in class_attr.targets]):
+                        elif any([target.id == "deps_pip_constraints" for target in class_attr.targets]):
                             for dep_pip in class_attr.value.elts:
                                 if type(dep_pip.value) == str:
                                     deps_pip_constraints.append(dep_pip.value)
@@ -380,6 +386,11 @@ def preload_module(self, module_file):
                         # ansible playbook
                         elif any([target.id == "deps_ansible" for target in class_attr.targets]):
                             ansible_tasks = ast.literal_eval(class_attr.value)
+                        # shared/common module dependencies
+                        elif any([target.id == "deps_common" for target in class_attr.targets]):
+                            for dep_common in class_attr.value.elts:
+                                if type(dep_common.value) == str:
+                                    deps_common.append(dep_common.value)
 
         for task in ansible_tasks:
             if not "become" in task:
@@ -403,13 +414,24 @@ def preload_module(self, module_file):
                 "shell": deps_shell,
                 "apt": deps_apt,
                 "ansible": ansible_tasks,
+                "common": deps_common,
             },
             "sudo": len(deps_apt) > 0,
         }
-        if any(x == True for x in search_dict_by_key("become", ansible_tasks)) or any(
-            x == True for x in search_dict_by_key("ansible_become", ansible_tasks)
-        ):
-            preloaded_data["sudo"] = True
+        ansible_task_list = list(ansible_tasks)
+        for dep_common in deps_common:
+            try:
+                ansible_task_list.extend(self._shared_deps[dep_common])
+            except KeyError:
+                common_choices = ",".join(self._shared_deps)
+                raise BBOTError(
+                    f'Error while preloading module "{module_file}": No shared dependency named "{dep_common}" (choices: {common_choices})'
+                )
+        for ansible_task in ansible_task_list:
+            if any(x == True for x in search_dict_by_key("become", ansible_task)) or any(
+                x == True for x in search_dict_by_key("ansible_become", ansible_tasks)
+            ):
+                preloaded_data["sudo"] = True
         return preloaded_data
 
     def load_modules(self, module_names):

bbot/core/shared_deps.py

@@ -0,0 +1,119 @@
+DEP_FFUF = [
+    {
+        "name": "Download ffuf",
+        "unarchive": {
+            "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_DEPS_FFUF_VERSION}/ffuf_#{BBOT_DEPS_FFUF_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.tar.gz",
+            "include": "ffuf",
+            "dest": "#{BBOT_TOOLS}",
+            "remote_src": True,
+        },
+    }
+]
+
+DEP_DOCKER = [
+    {
+        "name": "Check if Docker is already installed",
+        "command": "docker --version",
+        "register": "docker_installed",
+        "ignore_errors": True,
+    },
+    {
+        "name": "Install Docker (Non-Debian)",
+        "package": {"name": "docker", "state": "present"},
+        "become": True,
+        "when": "ansible_facts['os_family'] != 'Debian' and docker_installed.rc != 0",
+    },
+    {
+        "name": "Install Docker (Debian)",
+        "package": {
+            "name": "docker.io",
+            "state": "present",
+        },
+        "become": True,
+        "when": "ansible_facts['os_family'] == 'Debian' and docker_installed.rc != 0",
+    },
+]
+
+DEP_MASSDNS = [
+    {
+        "name": "install dev tools",
+        "package": {"name": ["gcc", "git", "make"], "state": "present"},
+        "become": True,
+        "ignore_errors": True,
+    },
+    {
+        "name": "Download massdns source code",
+        "git": {
+            "repo": "https://github.com/blechschmidt/massdns.git",
+            "dest": "#{BBOT_TEMP}/massdns",
+            "single_branch": True,
+            "version": "master",
+        },
+    },
+    {
+        "name": "Build massdns (Linux)",
+        "command": {"chdir": "#{BBOT_TEMP}/massdns", "cmd": "make", "creates": "#{BBOT_TEMP}/massdns/bin/massdns"},
+        "when": "ansible_facts['system'] == 'Linux'",
+    },
+    {
+        "name": "Build massdns (non-Linux)",
+        "command": {
+            "chdir": "#{BBOT_TEMP}/massdns",
+            "cmd": "make nolinux",
+            "creates": "#{BBOT_TEMP}/massdns/bin/massdns",
+        },
+        "when": "ansible_facts['system'] != 'Linux'",
+    },
+    {
+        "name": "Install massdns",
+        "copy": {"src": "#{BBOT_TEMP}/massdns/bin/massdns", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"},
+    },
+]
+
+DEP_CHROMIUM = [
+    {
+        "name": "Install Chromium (Non-Debian)",
+        "package": {"name": "chromium", "state": "present"},
+        "become": True,
+        "when": "ansible_facts['os_family'] != 'Debian'",
+        "ignore_errors": True,
+    },
+    {
+        "name": "Install Chromium dependencies (Debian)",
+        "package": {
+            "name": "libasound2,libatk-bridge2.0-0,libatk1.0-0,libcairo2,libcups2,libdrm2,libgbm1,libnss3,libpango-1.0-0,libxcomposite1,libxdamage1,libxfixes3,libxkbcommon0,libxrandr2",
+            "state": "present",
+        },
+        "become": True,
+        "when": "ansible_facts['os_family'] == 'Debian'",
+        "ignore_errors": True,
+    },
+    {
+        "name": "Get latest Chromium version (Debian)",
+        "uri": {
+            "url": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2FLAST_CHANGE?alt=media",
+            "return_content": True,
+        },
+        "register": "chromium_version",
+        "when": "ansible_facts['os_family'] == 'Debian'",
+        "ignore_errors": True,
+    },
+    {
+        "name": "Download Chromium (Debian)",
+        "unarchive": {
+            "src": "https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F{{ chromium_version.content }}%2Fchrome-linux.zip?alt=media",
+            "remote_src": True,
+            "dest": "#{BBOT_TOOLS}",
+            "creates": "#{BBOT_TOOLS}/chrome-linux",
+        },
+        "when": "ansible_facts['os_family'] == 'Debian'",
+        "ignore_errors": True,
+    },
+]
+
+# shared module dependencies -- ffuf, massdns, chromium, etc.
+SHARED_DEPS = {}
+for var, val in list(locals().items()):
+    if var.startswith("DEP_") and isinstance(val, list):
+        var = var.split("_", 1)[-1].lower()
+        SHARED_DEPS[var] = val

bbot/defaults.yml

@@ -19,6 +19,11 @@ http_proxy:
 # Web user-agent
 user_agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36 Edg/119.0.2151.97
 
+# Tool dependencies
+deps:
+  ffuf:
+    version: "2.1.0"
+
 ### WEB SPIDER ###
 
 # Set the maximum number of HTTP links that can be followed in a row (0 == no spidering allowed)

bbot/modules/deadly/dastardly.py

@@ -9,29 +9,7 @@ class dastardly(BaseModule):
     meta = {"description": "Lightweight web application security scanner"}
 
     deps_pip = ["lxml~=4.9.2"]
-    deps_ansible = [
-        {
-            "name": "Check if Docker is already installed",
-            "command": "docker --version",
-            "register": "docker_installed",
-            "ignore_errors": True,
-        },
-        {
-            "name": "Install Docker (Non-Debian)",
-            "package": {"name": "docker", "state": "present"},
-            "become": True,
-            "when": "ansible_facts['os_family'] != 'Debian' and docker_installed.rc != 0",
-        },
-        {
-            "name": "Install Docker (Debian)",
-            "package": {
-                "name": "docker.io",
-                "state": "present",
-            },
-            "become": True,
-            "when": "ansible_facts['os_family'] == 'Debian' and docker_installed.rc != 0",
-        },
-    ]
+    deps_common = ["docker"]
     per_hostport_only = True
 
     async def setup(self):

bbot/modules/deadly/ffuf.py

@@ -17,29 +17,17 @@ class ffuf(BaseModule):
         "wordlist": "https://raw.githubusercontent.com/danielmiessler/SecLists/master/Discovery/Web-Content/raft-small-directories.txt",
         "lines": 5000,
         "max_depth": 0,
-        "version": "2.0.0",
         "extensions": "",
     }
 
     options_desc = {
         "wordlist": "Specify wordlist to use when finding directories",
         "lines": "take only the first N lines from the wordlist when finding directories",
         "max_depth": "the maximum directory depth to attempt to solve",
-        "version": "ffuf version",
         "extensions": "Optionally include a list of extensions to extend the keyword with (comma separated)",
     }
 
-    deps_ansible = [
-        {
-            "name": "Download ffuf",
-            "unarchive": {
-                "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_MODULES_FFUF_VERSION}/ffuf_#{BBOT_MODULES_FFUF_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.tar.gz",
-                "include": "ffuf",
-                "dest": "#{BBOT_TOOLS}",
-                "remote_src": True,
-            },
-        }
-    ]
+    deps_common = ["ffuf"]
 
     banned_characters = [" "]
 

bbot/modules/deadly/vhost.py

@@ -22,17 +22,7 @@ class vhost(ffuf):
         "lines": "take only the first N lines from the wordlist when finding directories",
     }
 
-    deps_ansible = [
-        {
-            "name": "Download ffuf",
-            "unarchive": {
-                "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_MODULES_FFUF_VERSION}/ffuf_#{BBOT_MODULES_FFUF_VERSION}_#{BBOT_OS}_#{BBOT_CPU_ARCH}.tar.gz",
-                "include": "ffuf",
-                "dest": "#{BBOT_TOOLS}",
-                "remote_src": True,
-            },
-        }
-    ]
+    deps_common = ["ffuf"]
 
     in_scope_only = True
 

bbot/modules/ffuf_shortnames.py

@@ -59,17 +59,7 @@ class ffuf_shortnames(ffuf):
         "find_delimiters": "Attempt to detect common delimiters and make additional ffuf runs against them",
     }
 
-    deps_ansible = [
-        {
-            "name": "Download ffuf",
-            "unarchive": {
-                "src": "https://github.com/ffuf/ffuf/releases/download/v#{BBOT_MODULES_FFUF_VERSION}/ffuf_#{BBOT_MODULES_FFUF_VERSION}_#{BBOT_OS_PLATFORM}_#{BBOT_CPU_ARCH}.tar.gz",
-                "include": "ffuf",
-                "dest": "#{BBOT_TOOLS}",
-                "remote_src": True,
-            },
-        }
-    ]
+    deps_common = ["ffuf"]
 
     in_scope_only = True