Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Portscanner Rework #886

Merged
merged 16 commits into from
Dec 23, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion bbot/core/helpers/misc.py
Original file line number Diff line number Diff line change
Expand Up @@ -2479,7 +2479,7 @@ def parse_port_string(port_string):
>>> parse_port_string("invalid")
ValueError: Invalid port or port range: invalid
"""
elements = port_string.split(",")
elements = str(port_string).split(",")
ports = []

for element in elements:
Expand Down
26 changes: 7 additions & 19 deletions bbot/modules/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -663,19 +663,6 @@ def _event_precheck(self, event):
# exclude certain URLs (e.g. javascript):
if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
return False, "its extension was listed in url_extension_httpx_only"
# if event is an IP address that was speculated from a CIDR
source_is_range = getattr(event.source, "type", "") == "IP_RANGE"
if (
source_is_range
and event.type == "IP_ADDRESS"
and str(event.module) == "speculate"
and self.name != "speculate"
):
# and the current module listens for both ranges and CIDRs
if all([x in self.watched_events for x in ("IP_RANGE", "IP_ADDRESS")]):
# then skip the event.
# this helps avoid double-portscanning both an individual IP and its parent CIDR.
return False, "module consumes IP ranges directly"

return True, "precheck succeeded"

Expand Down Expand Up @@ -718,11 +705,12 @@ async def __event_postcheck(self, event):
if self._is_graph_important(event):
return True, "event is critical to the graph"

# don't send out-of-scope targets to active modules
# don't send out-of-scope targets to active modules (excluding portscanners, because they can handle it)
# this only takes effect if your target and whitelist are different
# TODO: the logic here seems incomplete, it could probably use some work.
if "active" in self.flags and "target" in event.tags and event not in self.scan.whitelist:
return False, "it is not in whitelist and module has active flag"
if "active" in self.flags and "portscan" not in self.flags:
if "target" in event.tags and event not in self.scan.whitelist:
return False, "it is not in whitelist and module has active flag"

# check scope distance
filter_result, reason = self._scope_distance_check(event)
Expand Down Expand Up @@ -811,10 +799,10 @@ async def queue_event(self, event, precheck=True):
acceptable, reason = self._event_precheck(event)
if not acceptable:
if reason and reason != "its type is not in watched_events":
self.debug(f"Not accepting {event} because {reason}")
self.debug(f"Not queueing {event} because {reason}")
return
else:
self.debug(f"Accepting {event} because {reason}")
self.debug(f"Queueing {event} because {reason}")
try:
self.incoming_event_queue.put_nowait(event)
async with self._event_received:
Expand Down Expand Up @@ -914,7 +902,7 @@ def _outgoing_dedup_hash(self, event):
"""
Determines the criteria for what is considered to be a duplicate event if `suppress_dupes` is True.
"""
return hash(event)
return hash((event, self.name))

def get_per_host_hash(self, event):
"""
Expand Down
27 changes: 14 additions & 13 deletions bbot/modules/internal/speculate.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,18 +33,18 @@ class speculate(BaseInternalModule):
_priority = 4

async def setup(self):
self.open_port_consumers = any(["OPEN_TCP_PORT" in m.watched_events for m in self.scan.modules.values()])
scan_modules = [m for m in self.scan.modules.values() if m._type == "scan"]
self.open_port_consumers = any(["OPEN_TCP_PORT" in m.watched_events for m in scan_modules])
self.portscanner_enabled = any(["portscan" in m.flags for m in self.scan.modules.values()])
self.emit_open_ports = self.open_port_consumers and not self.portscanner_enabled
self.range_to_ip = True
self.dns_resolution = self.scan.config.get("dns_resolution", True)

port_string = self.config.get("ports", "80,443")

try:
self.ports = self.helpers.parse_port_string(str(port_string))
except ValueError as e:
self.warning(f"Error parsing ports: {e}")
return False
return False, f"Error parsing ports: {e}"

if not self.portscanner_enabled:
self.info(f"No portscanner enabled. Assuming open ports: {', '.join(str(x) for x in self.ports)}")
Expand Down Expand Up @@ -76,10 +76,15 @@ async def handle_event(self, event):
self.emit_event(parent, "DNS_NAME", source=event, internal=True)

# generate open ports
emit_open_ports = self.open_port_consumers and not self.portscanner_enabled

# we speculate on distance-1 stuff too, because distance-1 open ports are needed by certain modules like sslcert
event_in_scope_distance = event.scope_distance <= (self.scan.scope_search_distance + 1)
speculate_open_ports = self.emit_open_ports and event_in_scope_distance

# from URLs
if event.type == "URL" or (event.type == "URL_UNVERIFIED" and emit_open_ports):
if event.host and event.port not in self.ports:
if event.type == "URL" or (event.type == "URL_UNVERIFIED" and self.open_port_consumers):
# only speculate port from a URL if it wouldn't be speculated naturally from the host
if event.host and (event.port not in self.ports or not speculate_open_ports):
self.emit_event(
self.helpers.make_netloc(event.host, event.port),
"OPEN_TCP_PORT",
Expand All @@ -100,7 +105,7 @@ async def handle_event(self, event):
self.emit_event(url_event)

# from hosts
if emit_open_ports:
if speculate_open_ports:
# don't act on unresolved DNS_NAMEs
usable_dns = False
if event.type == "DNS_NAME":
Expand All @@ -121,11 +126,7 @@ async def handle_event(self, event):
self.helpers.cloud.speculate(event)

async def filter_event(self, event):
# don't accept IP_RANGE --> IP_ADDRESS events from self
if str(event.module) == "speculate":
if not (event.type == "IP_ADDRESS" and str(getattr(event.source, "type")) == "IP_RANGE"):
return False
# don't accept errored DNS_NAMEs
if any(t in event.tags for t in ("unresolved", "a-error", "aaaa-error")):
return False
return False, "there were errors resolving this hostname"
return True
121 changes: 50 additions & 71 deletions bbot/modules/masscan.py
Original file line number Diff line number Diff line change
@@ -1,27 +1,26 @@
import json
import subprocess
from contextlib import suppress

from bbot.modules.base import BaseModule
from bbot.modules.templates.portscanner import portscanner


class masscan(BaseModule):
class masscan(portscanner):
flags = ["active", "portscan", "aggressive"]
watched_events = ["SCAN"]
watched_events = ["IP_ADDRESS", "IP_RANGE"]
produced_events = ["OPEN_TCP_PORT"]
meta = {"description": "Port scan IP subnets with masscan"}
# 600 packets/s ~= entire private IP space in 8 hours
meta = {"description": "Port scan with masscan. By default, scans top 100 ports."}
options = {
"top_ports": 100,
"ports": "",
# ping scan at 600 packets/s ~= entire private IP space in 8 hours
"rate": 600,
"wait": 10,
"wait": 5,
"ping_first": False,
"ping_only": False,
"use_cache": False,
}
options_desc = {
"top_ports": "Top ports to scan (default 100)",
"top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
"ports": "Ports to scan",
"rate": "Rate in packets per second",
"wait": "Seconds to wait for replies after scan is complete",
Expand Down Expand Up @@ -58,30 +57,30 @@ class masscan(BaseModule):
"copy": {"src": "#{BBOT_TEMP}/masscan/bin/masscan", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"},
},
]
_qsize = 100
batch_size = 1000000

async def setup(self):
self.top_ports = self.config.get("top_ports", 100)
self.ports = self.config.get("ports", "80,443")
self.rate = self.config.get("rate", 600)
self.wait = self.config.get("wait", 10)
self.ping_first = self.config.get("ping_first", False)
self.ping_only = self.config.get("ping_only", False)
self.alive_hosts = dict()
# make a quick dry run to validate ports etc.
self._target_findkey = "9.8.7.6"
if not self.helpers.in_tests:
self.use_cache = self.config.get("use_cache", False)
self.ports = self.config.get("ports", "")
if self.ports:
try:
dry_run_command = self._build_masscan_command(self._target_findkey, dry_run=True)
dry_run_result = await self.helpers.run(dry_run_command)
self.masscan_config = dry_run_result.stdout
self.masscan_config = "\n".join(l for l in self.masscan_config.splitlines() if "nocapture" not in l)
except subprocess.CalledProcessError as e:
self.warning(f"Error in masscan: {e.stderr}")
return False
self.helpers.parse_port_string(self.ports)
except ValueError as e:
return False, f"Error parsing ports: {e}"
self.alive_hosts = dict()

_, invalid_targets = self._build_targets(self.scan.target)
if invalid_targets > 0:
self.warning(
f"Masscan can only accept IP addresses or IP ranges as target ({invalid_targets:,} targets were hostnames)"
)

self.run_time = self.helpers.make_date()
self.use_cache = self.config.get("use_cache", False)
self.ping_cache = self.scan.home / f"masscan_ping.txt"
self.syn_cache = self.scan.home / f"masscan_syn.txt"
if self.use_cache:
Expand All @@ -101,23 +100,14 @@ async def setup(self):
self.helpers.depsinstaller.ensure_root(message="Masscan requires root privileges")
self.ping_cache_fd = None
self.syn_cache_fd = None
return True

async def handle_event(self, event):
return await super().setup()

async def handle_batch(self, *events):
if self.use_cache:
self.emit_from_cache()
else:
exclude, invalid_exclude = self._build_targets(self.scan.blacklist)
targets, invalid_targets = self._build_targets(self.scan.whitelist)
if invalid_exclude > 0:
self.warning(
f"Masscan can only accept IP addresses or IP ranges for blacklist ({invalid_exclude:,} blacklisted were hostnames)"
)
if invalid_targets > 0:
self.warning(
f"Masscan can only accept IP addresses or IP ranges as target ({invalid_targets:,} targets were hostnames)"
)

targets = [str(e.data) for e in events]
if not targets:
self.warning("No targets specified")
return
Expand All @@ -126,7 +116,7 @@ async def handle_event(self, event):
if self.ping_first or self.ping_only:
self.verbose("Starting masscan (ping scan)")

await self.masscan(targets, result_callback=self.append_alive_host, exclude=exclude, ping=True)
await self.masscan(targets, result_callback=self.append_alive_host, ping=True)
targets = ",".join(str(h) for h in self.alive_hosts)
if not targets:
self.warning("No hosts responded to pings")
Expand All @@ -135,48 +125,46 @@ async def handle_event(self, event):
# TCP SYN scan
if not self.ping_only:
self.verbose("Starting masscan (TCP SYN scan)")
await self.masscan(targets, result_callback=self.emit_open_tcp_port, exclude=exclude)
await self.masscan(targets, result_callback=self.emit_open_tcp_port)
else:
self.verbose("Only ping sweep was requested, skipping TCP SYN scan")
# save memory
self.alive_hosts.clear()

async def masscan(self, targets, result_callback, exclude=None, ping=False):
# config file
masscan_config = self.masscan_config.replace(self._target_findkey, targets)
self.debug("Masscan config:")
for line in masscan_config.splitlines():
self.debug(line)
config_file = self.helpers.tempfile(masscan_config)
# output file
# process_output = functools.partial(self.process_output, result_callback=result_callback)
# json_output_file = self.helpers.tempfile_tail(process_output)
# command
command = self._build_masscan_command(config=config_file, exclude=exclude, ping=ping)
# execute
async def masscan(self, targets, result_callback, ping=False):
target_file = self.helpers.tempfile(targets, pipe=False)
command = self._build_masscan_command(target_file, ping=ping)
stats_file = self.helpers.tempfile_tail(callback=self.verbose)
try:
with open(stats_file, "w") as stats_fh:
async for line in self.helpers.run_live(command, sudo=True, stderr=stats_fh):
self.process_output(line, result_callback=result_callback)
finally:
stats_file.unlink()
for file in (stats_file, target_file):
file.unlink()

def _build_masscan_command(self, targets=None, config=None, exclude=None, dry_run=False, ping=False):
command = ("masscan", "--rate", self.rate, "--wait", self.wait, "--open-only", "-oJ", "-")
if targets is not None:
command += (targets,)
if config is not None:
command += ("-c", config)
def _build_masscan_command(self, target_file=None, dry_run=False, ping=False):
command = (
"masscan",
"--excludefile",
str(self.exclude_file),
"--rate",
self.rate,
"--wait",
self.wait,
"--open-only",
"-oJ",
"-",
)
if target_file is not None:
command += ("-iL", str(target_file))
if ping:
command += ("--ping",)
elif not dry_run:
else:
if self.ports:
command += ("-p", self.ports)
else:
command += ("--top-ports", str(self.top_ports))
if exclude is not None:
command += ("--exclude", exclude)
if dry_run:
command += ("--echo",)
return command
Expand Down Expand Up @@ -258,23 +246,14 @@ def get_source_event(self, host):
source_event = self.scan.root_event
return source_event

def _build_targets(self, target):
invalid_targets = 0
targets = []
for t in target:
t = self.helpers.make_ip_type(t.data)
if isinstance(t, str):
invalid_targets += 1
else:
targets.append(t)
return ",".join(str(t) for t in targets), invalid_targets

async def cleanup(self):
if self.ping_first:
with suppress(Exception):
self.ping_cache_fd.close()
with suppress(Exception):
self.syn_cache_fd.close()
with suppress(Exception):
self.exclude_file.unlink()

def _write_ping_result(self, host):
if self.ping_cache_fd is None:
Expand Down
18 changes: 10 additions & 8 deletions bbot/modules/nmap.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,21 @@
from lxml import etree
from bbot.modules.base import BaseModule
from bbot.modules.templates.portscanner import portscanner


class nmap(BaseModule):
watched_events = ["IP_ADDRESS", "DNS_NAME"]
class nmap(portscanner):
watched_events = ["IP_ADDRESS", "DNS_NAME", "IP_RANGE"]
produced_events = ["OPEN_TCP_PORT"]
flags = ["active", "portscan", "aggressive", "web-thorough"]
meta = {"description": "Execute port scans with nmap"}
meta = {"description": "Port scan with nmap. By default, scans top 100 ports."}
options = {
"ports": "",
"top_ports": 100,
"ports": "",
"timing": "T4",
"skip_host_discovery": True,
}
options_desc = {
"ports": "ports to scan",
"top_ports": "top ports to scan",
"top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
"ports": "Ports to scan",
"timing": "-T<0-5>: Set timing template (higher is faster)",
"skip_host_discovery": "skip host discovery (-Pn)",
}
Expand All @@ -32,7 +32,7 @@ async def setup(self):
self.timing = self.config.get("timing", "T4")
self.top_ports = self.config.get("top_ports", 100)
self.skip_host_discovery = self.config.get("skip_host_discovery", True)
return True
return await super().setup()

async def handle_batch(self, *events):
target = self.helpers.make_target(*events)
Expand Down Expand Up @@ -65,6 +65,8 @@ def construct_command(self, targets):
temp_filename = self.helpers.temp_filename(extension="xml")
command = [
"nmap",
"--excludefile",
str(self.exclude_file),
"-n",
"--resolve-all",
f"-{self.timing}",
Expand Down
Loading