diff --git a/bbot/core/configurator/__init__.py b/bbot/core/configurator/__init__.py index 58526317b..3d3a1fe60 100644 --- a/bbot/core/configurator/__init__.py +++ b/bbot/core/configurator/__init__.py @@ -1,4 +1,5 @@ import os +import re import sys from pathlib import Path from omegaconf import OmegaConf @@ -41,12 +42,20 @@ sentinel = object() +exclude_from_validation = re.compile(r".*modules\.[a-z0-9_]+\.(?:batch_size|max_event_handlers)$") + + def check_cli_args(): - for c in args.cli_config: - if not is_file(c): - c = c.split("=")[0].strip() - v = OmegaConf.select(default_config, c, default=sentinel) - if v is sentinel: + conf = [a for a in args.cli_config if not is_file(a)] + all_options = None + for c in conf: + c = c.split("=")[0].strip() + v = OmegaConf.select(default_config, c, default=sentinel) + # if option isn't in the default config + if v is sentinel: + if exclude_from_validation.match(c): + continue + if all_options is None: from ...modules import module_loader modules_options = set() @@ -54,7 +63,7 @@ def check_cli_args(): modules_options.update(set(o[0] for o in module_options)) global_options = set(default_config.keys()) - {"modules", "output_modules"} all_options = global_options.union(modules_options) - match_and_exit(c, all_options, msg="module option") + match_and_exit(c, all_options, msg="module option") def ensure_config_files(): diff --git a/bbot/modules/badsecrets.py b/bbot/modules/badsecrets.py index 3ca03c2bf..cc52feecb 100644 --- a/bbot/modules/badsecrets.py +++ b/bbot/modules/badsecrets.py @@ -10,7 +10,6 @@ class badsecrets(BaseModule): produced_events = ["FINDING", "VULNERABILITY"] flags = ["active", "safe", "web-basic", "web-thorough"] meta = {"description": "Library for detecting known or weak secrets across many web frameworks"} - max_event_handlers = 2 deps_pip = ["badsecrets~=0.4"] @property diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 9101c800a..125b9c14b 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -94,8 +94,8 @@ class BaseModule: target_only = False in_scope_only = False - max_event_handlers = 1 - batch_size = 1 + _max_event_handlers = 1 + _batch_size = 1 batch_wait = 10 failed_request_abort_threshold = 5 @@ -288,6 +288,22 @@ async def ping(self): """ return + @property + def batch_size(self): + batch_size = self.config.get("batch_size", None) + # only allow overriding the batch size if its default value is greater than 1 + # this prevents modules from being accidentally neutered by an incorect batch_size setting + if batch_size is None or self._batch_size == 1: + batch_size = self._batch_size + return batch_size + + @property + def max_event_handlers(self): + max_event_handlers = self.config.get("max_event_handlers", None) + if max_event_handlers is None: + max_event_handlers = self._max_event_handlers + return max_event_handlers + @property def auth_secret(self): """Indicates if the module is properly configured for authentication. @@ -484,12 +500,8 @@ def num_incoming_events(self): ret = self.incoming_event_queue.qsize() return ret - @property - def _max_event_handlers(self): - return self.max_event_handlers - def start(self): - self._tasks = [asyncio.create_task(self._worker()) for _ in range(self._max_event_handlers)] + self._tasks = [asyncio.create_task(self._worker()) for _ in range(self.max_event_handlers)] async def _setup(self): """ diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index 8f2feff5c..40d4614ea 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -10,8 +10,6 @@ class nuclei(BaseModule): flags = ["active", "aggressive"] meta = {"description": "Fast and customisable vulnerability scanner"} - batch_size = 25 - options = { "version": "2.9.15", "tags": "", @@ -49,6 +47,7 @@ class nuclei(BaseModule): ] deps_pip = ["pyyaml~=6.0"] in_scope_only = True + _batch_size = 25 async def setup(self): # attempt to update nuclei templates diff --git a/bbot/modules/dnscommonsrv.py b/bbot/modules/dnscommonsrv.py index 6c80212d7..538f51621 100644 --- a/bbot/modules/dnscommonsrv.py +++ b/bbot/modules/dnscommonsrv.py @@ -94,7 +94,7 @@ class dnscommonsrv(BaseModule): produced_events = ["DNS_NAME"] flags = ["subdomain-enum", "passive", "safe"] meta = {"description": "Check for common SRV records"} - max_event_handlers = 5 + _max_event_handlers = 5 async def filter_event(self, event): # skip SRV wildcards diff --git a/bbot/modules/dnszonetransfer.py b/bbot/modules/dnszonetransfer.py index bb9819fbb..d950514b5 100644 --- a/bbot/modules/dnszonetransfer.py +++ b/bbot/modules/dnszonetransfer.py @@ -11,7 +11,7 @@ class dnszonetransfer(BaseModule): meta = {"description": "Attempt DNS zone transfers"} options = {"timeout": 10} options_desc = {"timeout": "Max seconds to wait before timing out"} - max_event_handlers = 5 + _max_event_handlers = 5 suppress_dupes = False async def setup(self): diff --git a/bbot/modules/fingerprintx.py b/bbot/modules/fingerprintx.py index 3bc36da66..be5695541 100644 --- a/bbot/modules/fingerprintx.py +++ b/bbot/modules/fingerprintx.py @@ -10,8 +10,8 @@ class fingerprintx(BaseModule): meta = {"description": "Fingerprint exposed services like RDP, SSH, MySQL, etc."} options = {"version": "1.1.4"} options_desc = {"version": "fingerprintx version"} - batch_size = 10 - max_event_handlers = 2 + _batch_size = 10 + _max_event_handlers = 2 _priority = 2 deps_ansible = [ diff --git a/bbot/modules/gowitness.py b/bbot/modules/gowitness.py index f19c5ed49..5f4c4a5e8 100644 --- a/bbot/modules/gowitness.py +++ b/bbot/modules/gowitness.py @@ -11,7 +11,6 @@ class gowitness(BaseModule): produced_events = ["WEBSCREENSHOT", "URL", "URL_UNVERIFIED", "TECHNOLOGY"] flags = ["active", "safe", "web-screenshots"] meta = {"description": "Take screenshots of webpages"} - batch_size = 100 options = { "version": "2.4.2", "threads": 4, @@ -76,6 +75,7 @@ class gowitness(BaseModule): }, }, ] + _batch_size = 100 # visit up to and including the scan's configured search distance plus one # this is one hop further than the default scope_distance_modifier = 1 diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index ef77668db..ff53e9972 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -10,7 +10,6 @@ class httpx(BaseModule): flags = ["active", "safe", "web-basic", "web-thorough", "social-enum", "subdomain-enum", "cloud-enum"] meta = {"description": "Visit webpages. Many other modules rely on httpx"} - batch_size = 500 options = {"threads": 50, "in_scope_only": True, "version": "1.2.5", "max_response_size": 5242880} options_desc = { "threads": "Number of httpx threads to use", @@ -31,6 +30,7 @@ class httpx(BaseModule): ] scope_distance_modifier = 1 + _batch_size = 500 _priority = 2 async def setup(self): diff --git a/bbot/modules/iis_shortnames.py b/bbot/modules/iis_shortnames.py index 9e3566ac8..f9914e316 100644 --- a/bbot/modules/iis_shortnames.py +++ b/bbot/modules/iis_shortnames.py @@ -21,7 +21,7 @@ class iis_shortnames(BaseModule): } in_scope_only = True - max_event_handlers = 8 + _max_event_handlers = 8 async def detect(self, target): technique = None diff --git a/bbot/modules/nmap.py b/bbot/modules/nmap.py index e2900ab1b..5e485e5ac 100644 --- a/bbot/modules/nmap.py +++ b/bbot/modules/nmap.py @@ -19,7 +19,7 @@ class nmap(BaseModule): "timing": "-T<0-5>: Set timing template (higher is faster)", "skip_host_discovery": "skip host discovery (-Pn)", } - max_event_handlers = 2 + _max_event_handlers = 2 batch_size = 256 _priority = 2 diff --git a/bbot/modules/nsec.py b/bbot/modules/nsec.py index 2ab254da7..cafed732e 100644 --- a/bbot/modules/nsec.py +++ b/bbot/modules/nsec.py @@ -6,7 +6,7 @@ class NSEC(BaseModule): produced_events = ["DNS_NAME"] flags = ["subdomain-enum", "passive", "safe"] meta = {"description": "Enumerate subdomains by NSEC-walking"} - max_event_handlers = 5 + _max_event_handlers = 5 async def filter_event(self, event): if "ns-record" in event.tags: diff --git a/bbot/modules/oauth.py b/bbot/modules/oauth.py index 0bf6457c2..f4d592511 100644 --- a/bbot/modules/oauth.py +++ b/bbot/modules/oauth.py @@ -13,7 +13,7 @@ class OAUTH(BaseModule): in_scope_only = False scope_distance_modifier = 1 - max_event_handlers = 2 + _max_event_handlers = 2 async def setup(self): self.processed = set() diff --git a/bbot/modules/output/neo4j.py b/bbot/modules/output/neo4j.py index 18bda8bad..2fc05fc9c 100644 --- a/bbot/modules/output/neo4j.py +++ b/bbot/modules/output/neo4j.py @@ -16,7 +16,7 @@ class neo4j(BaseOutputModule): "password": "Neo4j password", } deps_pip = ["git+https://github.com/blacklanternsecurity/py2neo"] - batch_size = 50 + _batch_size = 50 async def setup(self): try: diff --git a/bbot/modules/output/teams.py b/bbot/modules/output/teams.py index 0953fef5f..f858d897d 100644 --- a/bbot/modules/output/teams.py +++ b/bbot/modules/output/teams.py @@ -10,7 +10,7 @@ class Teams(Discord): "event_types": "Types of events to send", "min_severity": "Only allow VULNERABILITY events of this severity or highter", } - max_event_handlers = 5 + _max_event_handlers = 5 good_status_code = 200 content_key = "text" diff --git a/bbot/modules/paramminer_cookies.py b/bbot/modules/paramminer_cookies.py index 12b9ac32b..251931f58 100644 --- a/bbot/modules/paramminer_cookies.py +++ b/bbot/modules/paramminer_cookies.py @@ -25,7 +25,7 @@ class paramminer_cookies(paramminer_headers): options_desc = {"wordlist": "Define the wordlist to be used to derive cookies"} scanned_hosts = [] boring_words = set() - max_event_handlers = 12 + _max_event_handlers = 12 in_scope_only = True compare_mode = "cookie" default_wordlist = "paramminer_parameters.txt" diff --git a/bbot/modules/paramminer_headers.py b/bbot/modules/paramminer_headers.py index 8632ddf0d..65880d9c8 100644 --- a/bbot/modules/paramminer_headers.py +++ b/bbot/modules/paramminer_headers.py @@ -69,7 +69,7 @@ class paramminer_headers(BaseModule): "zx-request-id", "zx-timer", } - max_event_handlers = 12 + _max_event_handlers = 12 in_scope_only = True compare_mode = "header" default_wordlist = "paramminer_headers.txt" diff --git a/bbot/modules/sslcert.py b/bbot/modules/sslcert.py index de598dd73..01dc9844b 100644 --- a/bbot/modules/sslcert.py +++ b/bbot/modules/sslcert.py @@ -18,7 +18,7 @@ class sslcert(BaseModule): options_desc = {"timeout": "Socket connect timeout in seconds", "skip_non_ssl": "Don't try common non-SSL ports"} deps_apt = ["openssl"] deps_pip = ["pyOpenSSL~=23.1.1"] - max_event_handlers = 25 + _max_event_handlers = 25 scope_distance_modifier = 1 _priority = 2 diff --git a/bbot/modules/subdomain_hijack.py b/bbot/modules/subdomain_hijack.py index 201e8b4a3..a25b1d64b 100644 --- a/bbot/modules/subdomain_hijack.py +++ b/bbot/modules/subdomain_hijack.py @@ -16,7 +16,7 @@ class subdomain_hijack(BaseModule): } options_desc = {"fingerprints": "URL or path to fingerprints.json"} scope_distance_modifier = 3 - max_event_handlers = 5 + _max_event_handlers = 5 async def setup(self): fingerprints_url = self.config.get("fingerprints") diff --git a/bbot/modules/telerik.py b/bbot/modules/telerik.py index f65ad9df2..e012b6af6 100644 --- a/bbot/modules/telerik.py +++ b/bbot/modules/telerik.py @@ -157,7 +157,7 @@ class telerik(BaseModule): }, ] - max_event_handlers = 5 + _max_event_handlers = 5 async def setup(self): self.timeout = self.scan.config.get("httpx_timeout", 5) diff --git a/bbot/modules/wappalyzer.py b/bbot/modules/wappalyzer.py index a372d1791..7bc10b448 100644 --- a/bbot/modules/wappalyzer.py +++ b/bbot/modules/wappalyzer.py @@ -20,7 +20,7 @@ class wappalyzer(BaseModule): deps_pip = ["python-Wappalyzer~=0.3.1"] # accept all events regardless of scope distance scope_distance_modifier = None - max_event_handlers = 5 + _max_event_handlers = 5 async def setup(self): self.wappalyzer = await self.scan.run_in_executor(Wappalyzer.latest)