Merge pull request #798 from blacklanternsecurity/batch-size-override

Allow Overriding Module batch_size and max_event_handlers

bbot/core/configurator/__init__.py

@@ -1,4 +1,5 @@
 import os
+import re
 import sys
 from pathlib import Path
 from omegaconf import OmegaConf
@@ -41,20 +42,28 @@
 sentinel = object()
 
 
+exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|max_event_handlers)$")
+
+
 def check_cli_args():
-    for c in args.cli_config:
-        if not is_file(c):
-            c = c.split("=")[0].strip()
-            v = OmegaConf.select(default_config, c, default=sentinel)
-            if v is sentinel:
+    conf = [a for a in args.cli_config if not is_file(a)]
+    all_options = None
+    for c in conf:
+        c = c.split("=")[0].strip()
+        v = OmegaConf.select(default_config, c, default=sentinel)
+        # if option isn't in the default config
+        if v is sentinel:
+            if exclude_from_validation.match(c):
+                continue
+            if all_options is None:
                 from ...modules import module_loader
 
                 modules_options = set()
                 for module_options in module_loader.modules_options().values():
                     modules_options.update(set(o[0] for o in module_options))
                 global_options = set(default_config.keys()) - {"modules", "output_modules"}
                 all_options = global_options.union(modules_options)
-                match_and_exit(c, all_options, msg="module option")
+            match_and_exit(c, all_options, msg="module option")
 
 
 def ensure_config_files():

bbot/modules/badsecrets.py

@@ -10,7 +10,6 @@ class badsecrets(BaseModule):
     produced_events = ["FINDING", "VULNERABILITY"]
     flags = ["active", "safe", "web-basic", "web-thorough"]
     meta = {"description": "Library for detecting known or weak secrets across many web frameworks"}
-    max_event_handlers = 2
     deps_pip = ["badsecrets~=0.4"]
 
     @property

bbot/modules/base.py

@@ -94,8 +94,8 @@ class BaseModule:
     target_only = False
     in_scope_only = False
 
-    max_event_handlers = 1
-    batch_size = 1
+    _max_event_handlers = 1
+    _batch_size = 1
     batch_wait = 10
     failed_request_abort_threshold = 5
 
@@ -288,6 +288,22 @@ async def ping(self):
         """
         return
 
+    @property
+    def batch_size(self):
+        batch_size = self.config.get("batch_size", None)
+        # only allow overriding the batch size if its default value is greater than 1
+        # this prevents modules from being accidentally neutered by an incorect batch_size setting
+        if batch_size is None or self._batch_size == 1:
+            batch_size = self._batch_size
+        return batch_size
+
+    @property
+    def max_event_handlers(self):
+        max_event_handlers = self.config.get("max_event_handlers", None)
+        if max_event_handlers is None:
+            max_event_handlers = self._max_event_handlers
+        return max_event_handlers
+
     @property
     def auth_secret(self):
         """Indicates if the module is properly configured for authentication.
@@ -484,12 +500,8 @@ def num_incoming_events(self):
             ret = self.incoming_event_queue.qsize()
         return ret
 
-    @property
-    def _max_event_handlers(self):
-        return self.max_event_handlers
-
     def start(self):
-        self._tasks = [asyncio.create_task(self._worker()) for _ in range(self._max_event_handlers)]
+        self._tasks = [asyncio.create_task(self._worker()) for _ in range(self.max_event_handlers)]
 
     async def _setup(self):
         """

bbot/modules/deadly/nuclei.py

@@ -10,8 +10,6 @@ class nuclei(BaseModule):
     flags = ["active", "aggressive"]
     meta = {"description": "Fast and customisable vulnerability scanner"}
 
-    batch_size = 25
-
     options = {
         "version": "2.9.15",
         "tags": "",
@@ -49,6 +47,7 @@ class nuclei(BaseModule):
     ]
     deps_pip = ["pyyaml~=6.0"]
     in_scope_only = True
+    _batch_size = 25
 
     async def setup(self):
         # attempt to update nuclei templates

bbot/modules/dnscommonsrv.py

@@ -94,7 +94,7 @@ class dnscommonsrv(BaseModule):
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
     meta = {"description": "Check for common SRV records"}
-    max_event_handlers = 5
+    _max_event_handlers = 5
 
     async def filter_event(self, event):
         # skip SRV wildcards

bbot/modules/dnszonetransfer.py

@@ -11,7 +11,7 @@ class dnszonetransfer(BaseModule):
     meta = {"description": "Attempt DNS zone transfers"}
     options = {"timeout": 10}
     options_desc = {"timeout": "Max seconds to wait before timing out"}
-    max_event_handlers = 5
+    _max_event_handlers = 5
     suppress_dupes = False
 
     async def setup(self):

bbot/modules/fingerprintx.py

@@ -10,8 +10,8 @@ class fingerprintx(BaseModule):
     meta = {"description": "Fingerprint exposed services like RDP, SSH, MySQL, etc."}
     options = {"version": "1.1.4"}
     options_desc = {"version": "fingerprintx version"}
-    batch_size = 10
-    max_event_handlers = 2
+    _batch_size = 10
+    _max_event_handlers = 2
     _priority = 2
 
     deps_ansible = [

bbot/modules/gowitness.py

@@ -11,7 +11,6 @@ class gowitness(BaseModule):
     produced_events = ["WEBSCREENSHOT", "URL", "URL_UNVERIFIED", "TECHNOLOGY"]
     flags = ["active", "safe", "web-screenshots"]
     meta = {"description": "Take screenshots of webpages"}
-    batch_size = 100
     options = {
         "version": "2.4.2",
         "threads": 4,
@@ -76,6 +75,7 @@ class gowitness(BaseModule):
             },
         },
     ]
+    _batch_size = 100
     # visit up to and including the scan's configured search distance plus one
     # this is one hop further than the default
     scope_distance_modifier = 1

bbot/modules/httpx.py

@@ -10,7 +10,6 @@ class httpx(BaseModule):
     flags = ["active", "safe", "web-basic", "web-thorough", "social-enum", "subdomain-enum", "cloud-enum"]
     meta = {"description": "Visit webpages. Many other modules rely on httpx"}
 
-    batch_size = 500
     options = {"threads": 50, "in_scope_only": True, "version": "1.2.5", "max_response_size": 5242880}
     options_desc = {
         "threads": "Number of httpx threads to use",
@@ -31,6 +30,7 @@ class httpx(BaseModule):
     ]
 
     scope_distance_modifier = 1
+    _batch_size = 500
     _priority = 2
 
     async def setup(self):

bbot/modules/iis_shortnames.py

@@ -21,7 +21,7 @@ class iis_shortnames(BaseModule):
     }
     in_scope_only = True
 
-    max_event_handlers = 8
+    _max_event_handlers = 8
 
     async def detect(self, target):
         technique = None

bbot/modules/nmap.py

@@ -19,7 +19,7 @@ class nmap(BaseModule):
         "timing": "-T<0-5>: Set timing template (higher is faster)",
         "skip_host_discovery": "skip host discovery (-Pn)",
     }
-    max_event_handlers = 2
+    _max_event_handlers = 2
     batch_size = 256
     _priority = 2
 

bbot/modules/nsec.py

@@ -6,7 +6,7 @@ class NSEC(BaseModule):
     produced_events = ["DNS_NAME"]
     flags = ["subdomain-enum", "passive", "safe"]
     meta = {"description": "Enumerate subdomains by NSEC-walking"}
-    max_event_handlers = 5
+    _max_event_handlers = 5
 
     async def filter_event(self, event):
         if "ns-record" in event.tags:

bbot/modules/oauth.py

@@ -13,7 +13,7 @@ class OAUTH(BaseModule):
 
     in_scope_only = False
     scope_distance_modifier = 1
-    max_event_handlers = 2
+    _max_event_handlers = 2
 
     async def setup(self):
         self.processed = set()

bbot/modules/output/neo4j.py

@@ -16,7 +16,7 @@ class neo4j(BaseOutputModule):
         "password": "Neo4j password",
     }
     deps_pip = ["git+https://github.com/blacklanternsecurity/py2neo"]
-    batch_size = 50
+    _batch_size = 50
 
     async def setup(self):
         try:

bbot/modules/output/teams.py

@@ -10,7 +10,7 @@ class Teams(Discord):
         "event_types": "Types of events to send",
         "min_severity": "Only allow VULNERABILITY events of this severity or highter",
     }
-    max_event_handlers = 5
+    _max_event_handlers = 5
     good_status_code = 200
     content_key = "text"
 

bbot/modules/paramminer_cookies.py

@@ -25,7 +25,7 @@ class paramminer_cookies(paramminer_headers):
     options_desc = {"wordlist": "Define the wordlist to be used to derive cookies"}
     scanned_hosts = []
     boring_words = set()
-    max_event_handlers = 12
+    _max_event_handlers = 12
     in_scope_only = True
     compare_mode = "cookie"
     default_wordlist = "paramminer_parameters.txt"

bbot/modules/paramminer_headers.py

@@ -69,7 +69,7 @@ class paramminer_headers(BaseModule):
         "zx-request-id",
         "zx-timer",
     }
-    max_event_handlers = 12
+    _max_event_handlers = 12
     in_scope_only = True
     compare_mode = "header"
     default_wordlist = "paramminer_headers.txt"

bbot/modules/sslcert.py

@@ -18,7 +18,7 @@ class sslcert(BaseModule):
     options_desc = {"timeout": "Socket connect timeout in seconds", "skip_non_ssl": "Don't try common non-SSL ports"}
     deps_apt = ["openssl"]
     deps_pip = ["pyOpenSSL~=23.1.1"]
-    max_event_handlers = 25
+    _max_event_handlers = 25
     scope_distance_modifier = 1
     _priority = 2
 

bbot/modules/subdomain_hijack.py

@@ -16,7 +16,7 @@ class subdomain_hijack(BaseModule):
     }
     options_desc = {"fingerprints": "URL or path to fingerprints.json"}
     scope_distance_modifier = 3
-    max_event_handlers = 5
+    _max_event_handlers = 5
 
     async def setup(self):
         fingerprints_url = self.config.get("fingerprints")

bbot/modules/telerik.py

@@ -157,7 +157,7 @@ class telerik(BaseModule):
         },
     ]
 
-    max_event_handlers = 5
+    _max_event_handlers = 5
 
     async def setup(self):
         self.timeout = self.scan.config.get("httpx_timeout", 5)

bbot/modules/wappalyzer.py

@@ -20,7 +20,7 @@ class wappalyzer(BaseModule):
     deps_pip = ["python-Wappalyzer~=0.3.1"]
     # accept all events regardless of scope distance
     scope_distance_modifier = None
-    max_event_handlers = 5
+    _max_event_handlers = 5
 
     async def setup(self):
         self.wappalyzer = await self.scan.run_in_executor(Wappalyzer.latest)