diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 601ccaeac..96a7ab607 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -663,19 +663,6 @@ def _event_precheck(self, event): # exclude certain URLs (e.g. javascript): if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags: return False, "its extension was listed in url_extension_httpx_only" - # if event is an IP address that was speculated from a CIDR - source_is_range = getattr(event.source, "type", "") == "IP_RANGE" - if ( - source_is_range - and event.type == "IP_ADDRESS" - and str(event.module) == "speculate" - and self.name != "speculate" - ): - # and the current module listens for both ranges and CIDRs - if all([x in self.watched_events for x in ("IP_RANGE", "IP_ADDRESS")]): - # then skip the event. - # this helps avoid double-portscanning both an individual IP and its parent CIDR. - return False, "module consumes IP ranges directly" return True, "precheck succeeded" diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 0a506393b..db6ede3f3 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -122,10 +122,9 @@ async def handle_event(self, event): async def filter_event(self, event): # don't accept IP_RANGE --> IP_ADDRESS events from self - if str(event.module) == "speculate": - if not (event.type == "IP_ADDRESS" and str(getattr(event.source, "type")) == "IP_RANGE"): - return False + if event.module == self and event.type == "IP_ADDRESS" and str(getattr(event.source, "type")) == "IP_RANGE": + return False, "cannot accept IP_RANGE->IP_ADDRESS speculate from self" # don't accept errored DNS_NAMEs if any(t in event.tags for t in ("unresolved", "a-error", "aaaa-error")): - return False + return False, "there were errors resolving this hostname" return True diff --git a/bbot/modules/masscan.py b/bbot/modules/masscan.py index eb455d8fc..183a92292 100644 --- a/bbot/modules/masscan.py +++ b/bbot/modules/masscan.py @@ -2,18 +2,18 @@ import subprocess from contextlib import suppress -from bbot.modules.base import BaseModule +from bbot.modules.templates.portscanner import portscanner -class masscan(BaseModule): +class masscan(portscanner): flags = ["active", "portscan", "aggressive"] - watched_events = ["SCAN"] + watched_events = ["IP_ADDRESS", "IP_RANGE"] produced_events = ["OPEN_TCP_PORT"] meta = {"description": "Port scan IP subnets with masscan"} - # 600 packets/s ~= entire private IP space in 8 hours options = { "top_ports": 100, "ports": "", + # 600 packets/s ~= entire private IP space in 8 hours "rate": 600, "wait": 10, "ping_first": False, @@ -58,7 +58,7 @@ class masscan(BaseModule): "copy": {"src": "#{BBOT_TEMP}/masscan/bin/masscan", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"}, }, ] - _qsize = 100 + batch_size = 1000000 async def setup(self): self.top_ports = self.config.get("top_ports", 100) @@ -67,21 +67,31 @@ async def setup(self): self.wait = self.config.get("wait", 10) self.ping_first = self.config.get("ping_first", False) self.ping_only = self.config.get("ping_only", False) + self.use_cache = self.config.get("use_cache", False) self.alive_hosts = dict() + + exclude, invalid_exclude = self._build_targets(self.scan.blacklist) + self.exclude = ",".join(exclude) + _, invalid_targets = self._build_targets(self.scan.target) + if invalid_exclude > 0: + self.warning( + f"Masscan can only accept IP addresses or IP ranges for blacklist ({invalid_exclude:,} blacklisted were hostnames)" + ) + if invalid_targets > 0: + self.warning( + f"Masscan can only accept IP addresses or IP ranges as target ({invalid_targets:,} targets were hostnames)" + ) + # make a quick dry run to validate ports etc. - self._target_findkey = "9.8.7.6" if not self.helpers.in_tests: try: - dry_run_command = self._build_masscan_command(self._target_findkey, dry_run=True) + dry_run_command = self._build_masscan_command(dry_run=True) dry_run_result = await self.helpers.run(dry_run_command) - self.masscan_config = dry_run_result.stdout - self.masscan_config = "\n".join(l for l in self.masscan_config.splitlines() if "nocapture" not in l) except subprocess.CalledProcessError as e: self.warning(f"Error in masscan: {e.stderr}") return False self.run_time = self.helpers.make_date() - self.use_cache = self.config.get("use_cache", False) self.ping_cache = self.scan.home / f"masscan_ping.txt" self.syn_cache = self.scan.home / f"masscan_syn.txt" if self.use_cache: @@ -101,23 +111,14 @@ async def setup(self): self.helpers.depsinstaller.ensure_root(message="Masscan requires root privileges") self.ping_cache_fd = None self.syn_cache_fd = None - return True - async def handle_event(self, event): + return await super().setup() + + async def handle_batch(self, *events): if self.use_cache: self.emit_from_cache() else: - exclude, invalid_exclude = self._build_targets(self.scan.blacklist) - targets, invalid_targets = self._build_targets(self.scan.whitelist) - if invalid_exclude > 0: - self.warning( - f"Masscan can only accept IP addresses or IP ranges for blacklist ({invalid_exclude:,} blacklisted were hostnames)" - ) - if invalid_targets > 0: - self.warning( - f"Masscan can only accept IP addresses or IP ranges as target ({invalid_targets:,} targets were hostnames)" - ) - + targets = [str(e.data) for e in events] if not targets: self.warning("No targets specified") return @@ -126,7 +127,7 @@ async def handle_event(self, event): if self.ping_first or self.ping_only: self.verbose("Starting masscan (ping scan)") - await self.masscan(targets, result_callback=self.append_alive_host, exclude=exclude, ping=True) + await self.masscan(targets, result_callback=self.append_alive_host, ping=True) targets = ",".join(str(h) for h in self.alive_hosts) if not targets: self.warning("No hosts responded to pings") @@ -135,39 +136,28 @@ async def handle_event(self, event): # TCP SYN scan if not self.ping_only: self.verbose("Starting masscan (TCP SYN scan)") - await self.masscan(targets, result_callback=self.emit_open_tcp_port, exclude=exclude) + await self.masscan(targets, result_callback=self.emit_open_tcp_port) else: self.verbose("Only ping sweep was requested, skipping TCP SYN scan") # save memory self.alive_hosts.clear() - async def masscan(self, targets, result_callback, exclude=None, ping=False): - # config file - masscan_config = self.masscan_config.replace(self._target_findkey, targets) - self.debug("Masscan config:") - for line in masscan_config.splitlines(): - self.debug(line) - config_file = self.helpers.tempfile(masscan_config) - # output file - # process_output = functools.partial(self.process_output, result_callback=result_callback) - # json_output_file = self.helpers.tempfile_tail(process_output) - # command - command = self._build_masscan_command(config=config_file, exclude=exclude, ping=ping) - # execute + async def masscan(self, targets, result_callback, ping=False): + target_file = self.helpers.tempfile(targets, pipe=False) + command = self._build_masscan_command(target_file, ping=ping) stats_file = self.helpers.tempfile_tail(callback=self.verbose) try: with open(stats_file, "w") as stats_fh: async for line in self.helpers.run_live(command, sudo=True, stderr=stats_fh): self.process_output(line, result_callback=result_callback) finally: - stats_file.unlink() + for file in (stats_file, target_file): + file.unlink() - def _build_masscan_command(self, targets=None, config=None, exclude=None, dry_run=False, ping=False): + def _build_masscan_command(self, target_file=None, dry_run=False, ping=False): command = ("masscan", "--rate", self.rate, "--wait", self.wait, "--open-only", "-oJ", "-") - if targets is not None: - command += (targets,) - if config is not None: - command += ("-c", config) + if target_file is not None: + command += ("-iL", str(target_file)) if ping: command += ("--ping",) elif not dry_run: @@ -175,8 +165,8 @@ def _build_masscan_command(self, targets=None, config=None, exclude=None, dry_ru command += ("-p", self.ports) else: command += ("--top-ports", str(self.top_ports)) - if exclude is not None: - command += ("--exclude", exclude) + if self.exclude: + command += ("--exclude", self.exclude) if dry_run: command += ("--echo",) return command @@ -258,7 +248,7 @@ def get_source_event(self, host): source_event = self.scan.root_event return source_event - def _build_targets(self, target): + def _build_targets(self, target, delimiter=","): invalid_targets = 0 targets = [] for t in target: @@ -267,7 +257,7 @@ def _build_targets(self, target): invalid_targets += 1 else: targets.append(t) - return ",".join(str(t) for t in targets), invalid_targets + return [str(t) for t in targets], invalid_targets async def cleanup(self): if self.ping_first: diff --git a/bbot/modules/nmap.py b/bbot/modules/nmap.py index 64cfde0eb..05b495fbf 100644 --- a/bbot/modules/nmap.py +++ b/bbot/modules/nmap.py @@ -1,8 +1,8 @@ from lxml import etree -from bbot.modules.base import BaseModule +from bbot.modules.templates.portscanner import portscanner -class nmap(BaseModule): +class nmap(portscanner): watched_events = ["IP_ADDRESS", "DNS_NAME", "IP_RANGE"] produced_events = ["OPEN_TCP_PORT"] flags = ["active", "portscan", "aggressive", "web-thorough"] @@ -32,16 +32,7 @@ async def setup(self): self.timing = self.config.get("timing", "T4") self.top_ports = self.config.get("top_ports", 100) self.skip_host_discovery = self.config.get("skip_host_discovery", True) - self.ip_ranges = [e.host for e in self.scan.target.events if e.type == "IP_RANGE"] - return True - - async def filter_event(self, event): - # skip IP_ADDRESSes if they are included in any of our target IP_RANGEs - if event.type == "IP_ADDRESS": - for net in self.helpers.ip_network_parents(event.data, include_self=True): - if net in self.ip_ranges: - return False, f"Skipping {event.host} because it is already included in {net}" - return True + return await super().setup() async def handle_batch(self, *events): target = self.helpers.make_target(*events) diff --git a/bbot/modules/output/base.py b/bbot/modules/output/base.py index 99a03b0cf..9d5f2c3ee 100644 --- a/bbot/modules/output/base.py +++ b/bbot/modules/output/base.py @@ -39,19 +39,6 @@ def _event_precheck(self, event): if event._internal: return False, "_internal is True" - # if event is an IP address that was speculated from a CIDR - source_is_range = getattr(event.source, "type", "") == "IP_RANGE" - if ( - source_is_range - and event.type == "IP_ADDRESS" - and str(event.module) == "speculate" - and self.name != "speculate" - ): - # and the current module listens for both ranges and CIDRs - if all([x in self.watched_events for x in ("IP_RANGE", "IP_ADDRESS")]): - # then skip the event. - # this helps avoid double-portscanning both an individual IP and its parent CIDR. - return False, "module consumes IP ranges directly" return True, "precheck succeeded" def is_incoming_duplicate(self, event, add=False):