Merge pull request #886 from blacklanternsecurity/nmap-accept-ip-range

Portscanner Rework

bbot/core/helpers/misc.py

@@ -2483,7 +2483,7 @@ def parse_port_string(port_string):
         >>> parse_port_string("invalid")
         ValueError: Invalid port or port range: invalid
     """
-    elements = port_string.split(",")
+    elements = str(port_string).split(",")
     ports = []
 
     for element in elements:

bbot/modules/base.py

@@ -663,19 +663,6 @@ def _event_precheck(self, event):
         # exclude certain URLs (e.g. javascript):
         if event.type.startswith("URL") and self.name != "httpx" and "httpx-only" in event.tags:
             return False, "its extension was listed in url_extension_httpx_only"
-        # if event is an IP address that was speculated from a CIDR
-        source_is_range = getattr(event.source, "type", "") == "IP_RANGE"
-        if (
-            source_is_range
-            and event.type == "IP_ADDRESS"
-            and str(event.module) == "speculate"
-            and self.name != "speculate"
-        ):
-            # and the current module listens for both ranges and CIDRs
-            if all([x in self.watched_events for x in ("IP_RANGE", "IP_ADDRESS")]):
-                # then skip the event.
-                # this helps avoid double-portscanning both an individual IP and its parent CIDR.
-                return False, "module consumes IP ranges directly"
 
         return True, "precheck succeeded"
 
@@ -718,11 +705,12 @@ async def __event_postcheck(self, event):
         if self._is_graph_important(event):
             return True, "event is critical to the graph"
 
-        # don't send out-of-scope targets to active modules
+        # don't send out-of-scope targets to active modules (excluding portscanners, because they can handle it)
         # this only takes effect if your target and whitelist are different
         # TODO: the logic here seems incomplete, it could probably use some work.
-        if "active" in self.flags and "target" in event.tags and event not in self.scan.whitelist:
-            return False, "it is not in whitelist and module has active flag"
+        if "active" in self.flags and "portscan" not in self.flags:
+            if "target" in event.tags and event not in self.scan.whitelist:
+                return False, "it is not in whitelist and module has active flag"
 
         # check scope distance
         filter_result, reason = self._scope_distance_check(event)
@@ -811,10 +799,10 @@ async def queue_event(self, event, precheck=True):
                 acceptable, reason = self._event_precheck(event)
             if not acceptable:
                 if reason and reason != "its type is not in watched_events":
-                    self.debug(f"Not accepting {event} because {reason}")
+                    self.debug(f"Not queueing {event} because {reason}")
                 return
             else:
-                self.debug(f"Accepting {event} because {reason}")
+                self.debug(f"Queueing {event} because {reason}")
             try:
                 self.incoming_event_queue.put_nowait(event)
                 async with self._event_received:
@@ -914,7 +902,7 @@ def _outgoing_dedup_hash(self, event):
         """
         Determines the criteria for what is considered to be a duplicate event if `suppress_dupes` is True.
         """
-        return hash(event)
+        return hash((event, self.name))
 
     def get_per_host_hash(self, event):
         """

bbot/modules/internal/speculate.py

@@ -35,19 +35,19 @@ class speculate(BaseInternalModule):
     _priority = 4
 
     async def setup(self):
-        self.open_port_consumers = any(["OPEN_TCP_PORT" in m.watched_events for m in self.scan.modules.values()])
+        scan_modules = [m for m in self.scan.modules.values() if m._type == "scan"]
+        self.open_port_consumers = any(["OPEN_TCP_PORT" in m.watched_events for m in scan_modules])
         self.portscanner_enabled = any(["portscan" in m.flags for m in self.scan.modules.values()])
+        self.emit_open_ports = self.open_port_consumers and not self.portscanner_enabled
         self.range_to_ip = True
         self.dns_resolution = self.scan.config.get("dns_resolution", True)
         self.org_stubs_seen = set()
 
         port_string = self.config.get("ports", "80,443")
-
         try:
             self.ports = self.helpers.parse_port_string(str(port_string))
         except ValueError as e:
-            self.warning(f"Error parsing ports: {e}")
-            return False
+            return False, f"Error parsing ports: {e}"
 
         if not self.portscanner_enabled:
             self.info(f"No portscanner enabled. Assuming open ports: {', '.join(str(x) for x in self.ports)}")
@@ -79,10 +79,15 @@ async def handle_event(self, event):
                 self.emit_event(parent, "DNS_NAME", source=event, internal=True)
 
         # generate open ports
-        emit_open_ports = self.open_port_consumers and not self.portscanner_enabled
+
+        # we speculate on distance-1 stuff too, because distance-1 open ports are needed by certain modules like sslcert
+        event_in_scope_distance = event.scope_distance <= (self.scan.scope_search_distance + 1)
+        speculate_open_ports = self.emit_open_ports and event_in_scope_distance
+
         # from URLs
-        if event.type == "URL" or (event.type == "URL_UNVERIFIED" and emit_open_ports):
-            if event.host and event.port not in self.ports:
+        if event.type == "URL" or (event.type == "URL_UNVERIFIED" and self.open_port_consumers):
+            # only speculate port from a URL if it wouldn't be speculated naturally from the host
+            if event.host and (event.port not in self.ports or not speculate_open_ports):
                 self.emit_event(
                     self.helpers.make_netloc(event.host, event.port),
                     "OPEN_TCP_PORT",
@@ -103,7 +108,7 @@ async def handle_event(self, event):
                     self.emit_event(url_event)
 
         # from hosts
-        if emit_open_ports:
+        if speculate_open_ports:
             # don't act on unresolved DNS_NAMEs
             usable_dns = False
             if event.type == "DNS_NAME":
@@ -149,11 +154,7 @@ async def handle_event(self, event):
                 self.emit_event(stub_event)
 
     async def filter_event(self, event):
-        # don't accept IP_RANGE --> IP_ADDRESS events from self
-        if str(event.module) == "speculate":
-            if not (event.type == "IP_ADDRESS" and str(getattr(event.source, "type")) == "IP_RANGE"):
-                return False
         # don't accept errored DNS_NAMEs
         if any(t in event.tags for t in ("unresolved", "a-error", "aaaa-error")):
-            return False
+            return False, "there were errors resolving this hostname"
         return True

bbot/modules/masscan.py

@@ -1,27 +1,26 @@
 import json
-import subprocess
 from contextlib import suppress
 
-from bbot.modules.base import BaseModule
+from bbot.modules.templates.portscanner import portscanner
 
 
-class masscan(BaseModule):
+class masscan(portscanner):
     flags = ["active", "portscan", "aggressive"]
-    watched_events = ["SCAN"]
+    watched_events = ["IP_ADDRESS", "IP_RANGE"]
     produced_events = ["OPEN_TCP_PORT"]
-    meta = {"description": "Port scan IP subnets with masscan"}
-    # 600 packets/s ~= entire private IP space in 8 hours
+    meta = {"description": "Port scan with masscan. By default, scans top 100 ports."}
     options = {
         "top_ports": 100,
         "ports": "",
+        # ping scan at 600 packets/s ~= entire private IP space in 8 hours
         "rate": 600,
-        "wait": 10,
+        "wait": 5,
         "ping_first": False,
         "ping_only": False,
         "use_cache": False,
     }
     options_desc = {
-        "top_ports": "Top ports to scan (default 100)",
+        "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
         "ports": "Ports to scan",
         "rate": "Rate in packets per second",
         "wait": "Seconds to wait for replies after scan is complete",
@@ -58,30 +57,30 @@ class masscan(BaseModule):
             "copy": {"src": "#{BBOT_TEMP}/masscan/bin/masscan", "dest": "#{BBOT_TOOLS}/", "mode": "u+x,g+x,o+x"},
         },
     ]
-    _qsize = 100
+    batch_size = 1000000
 
     async def setup(self):
         self.top_ports = self.config.get("top_ports", 100)
-        self.ports = self.config.get("ports", "80,443")
         self.rate = self.config.get("rate", 600)
         self.wait = self.config.get("wait", 10)
         self.ping_first = self.config.get("ping_first", False)
         self.ping_only = self.config.get("ping_only", False)
-        self.alive_hosts = dict()
-        # make a quick dry run to validate ports etc.
-        self._target_findkey = "9.8.7.6"
-        if not self.helpers.in_tests:
+        self.use_cache = self.config.get("use_cache", False)
+        self.ports = self.config.get("ports", "")
+        if self.ports:
             try:
-                dry_run_command = self._build_masscan_command(self._target_findkey, dry_run=True)
-                dry_run_result = await self.helpers.run(dry_run_command)
-                self.masscan_config = dry_run_result.stdout
-                self.masscan_config = "\n".join(l for l in self.masscan_config.splitlines() if "nocapture" not in l)
-            except subprocess.CalledProcessError as e:
-                self.warning(f"Error in masscan: {e.stderr}")
-                return False
+                self.helpers.parse_port_string(self.ports)
+            except ValueError as e:
+                return False, f"Error parsing ports: {e}"
+        self.alive_hosts = dict()
+
+        _, invalid_targets = self._build_targets(self.scan.target)
+        if invalid_targets > 0:
+            self.warning(
+                f"Masscan can only accept IP addresses or IP ranges as target ({invalid_targets:,} targets were hostnames)"
+            )
 
         self.run_time = self.helpers.make_date()
-        self.use_cache = self.config.get("use_cache", False)
         self.ping_cache = self.scan.home / f"masscan_ping.txt"
         self.syn_cache = self.scan.home / f"masscan_syn.txt"
         if self.use_cache:
@@ -101,23 +100,14 @@ async def setup(self):
             self.helpers.depsinstaller.ensure_root(message="Masscan requires root privileges")
         self.ping_cache_fd = None
         self.syn_cache_fd = None
-        return True
 
-    async def handle_event(self, event):
+        return await super().setup()
+
+    async def handle_batch(self, *events):
         if self.use_cache:
             self.emit_from_cache()
         else:
-            exclude, invalid_exclude = self._build_targets(self.scan.blacklist)
-            targets, invalid_targets = self._build_targets(self.scan.whitelist)
-            if invalid_exclude > 0:
-                self.warning(
-                    f"Masscan can only accept IP addresses or IP ranges for blacklist ({invalid_exclude:,} blacklisted were hostnames)"
-                )
-            if invalid_targets > 0:
-                self.warning(
-                    f"Masscan can only accept IP addresses or IP ranges as target ({invalid_targets:,} targets were hostnames)"
-                )
-
+            targets = [str(e.data) for e in events]
             if not targets:
                 self.warning("No targets specified")
                 return
@@ -126,7 +116,7 @@ async def handle_event(self, event):
             if self.ping_first or self.ping_only:
                 self.verbose("Starting masscan (ping scan)")
 
-                await self.masscan(targets, result_callback=self.append_alive_host, exclude=exclude, ping=True)
+                await self.masscan(targets, result_callback=self.append_alive_host, ping=True)
                 targets = ",".join(str(h) for h in self.alive_hosts)
                 if not targets:
                     self.warning("No hosts responded to pings")
@@ -135,48 +125,46 @@ async def handle_event(self, event):
             # TCP SYN scan
             if not self.ping_only:
                 self.verbose("Starting masscan (TCP SYN scan)")
-                await self.masscan(targets, result_callback=self.emit_open_tcp_port, exclude=exclude)
+                await self.masscan(targets, result_callback=self.emit_open_tcp_port)
             else:
                 self.verbose("Only ping sweep was requested, skipping TCP SYN scan")
             # save memory
             self.alive_hosts.clear()
 
-    async def masscan(self, targets, result_callback, exclude=None, ping=False):
-        # config file
-        masscan_config = self.masscan_config.replace(self._target_findkey, targets)
-        self.debug("Masscan config:")
-        for line in masscan_config.splitlines():
-            self.debug(line)
-        config_file = self.helpers.tempfile(masscan_config)
-        # output file
-        # process_output = functools.partial(self.process_output, result_callback=result_callback)
-        # json_output_file = self.helpers.tempfile_tail(process_output)
-        # command
-        command = self._build_masscan_command(config=config_file, exclude=exclude, ping=ping)
-        # execute
+    async def masscan(self, targets, result_callback, ping=False):
+        target_file = self.helpers.tempfile(targets, pipe=False)
+        command = self._build_masscan_command(target_file, ping=ping)
         stats_file = self.helpers.tempfile_tail(callback=self.verbose)
         try:
             with open(stats_file, "w") as stats_fh:
                 async for line in self.helpers.run_live(command, sudo=True, stderr=stats_fh):
                     self.process_output(line, result_callback=result_callback)
         finally:
-            stats_file.unlink()
+            for file in (stats_file, target_file):
+                file.unlink()
 
-    def _build_masscan_command(self, targets=None, config=None, exclude=None, dry_run=False, ping=False):
-        command = ("masscan", "--rate", self.rate, "--wait", self.wait, "--open-only", "-oJ", "-")
-        if targets is not None:
-            command += (targets,)
-        if config is not None:
-            command += ("-c", config)
+    def _build_masscan_command(self, target_file=None, dry_run=False, ping=False):
+        command = (
+            "masscan",
+            "--excludefile",
+            str(self.exclude_file),
+            "--rate",
+            self.rate,
+            "--wait",
+            self.wait,
+            "--open-only",
+            "-oJ",
+            "-",
+        )
+        if target_file is not None:
+            command += ("-iL", str(target_file))
         if ping:
             command += ("--ping",)
-        elif not dry_run:
+        else:
             if self.ports:
                 command += ("-p", self.ports)
             else:
                 command += ("--top-ports", str(self.top_ports))
-        if exclude is not None:
-            command += ("--exclude", exclude)
         if dry_run:
             command += ("--echo",)
         return command
@@ -258,23 +246,14 @@ def get_source_event(self, host):
             source_event = self.scan.root_event
         return source_event
 
-    def _build_targets(self, target):
-        invalid_targets = 0
-        targets = []
-        for t in target:
-            t = self.helpers.make_ip_type(t.data)
-            if isinstance(t, str):
-                invalid_targets += 1
-            else:
-                targets.append(t)
-        return ",".join(str(t) for t in targets), invalid_targets
-
     async def cleanup(self):
         if self.ping_first:
             with suppress(Exception):
                 self.ping_cache_fd.close()
         with suppress(Exception):
             self.syn_cache_fd.close()
+        with suppress(Exception):
+            self.exclude_file.unlink()
 
     def _write_ping_result(self, host):
         if self.ping_cache_fd is None:

bbot/modules/nmap.py

@@ -1,21 +1,21 @@
 from lxml import etree
-from bbot.modules.base import BaseModule
+from bbot.modules.templates.portscanner import portscanner
 
 
-class nmap(BaseModule):
-    watched_events = ["IP_ADDRESS", "DNS_NAME"]
+class nmap(portscanner):
+    watched_events = ["IP_ADDRESS", "DNS_NAME", "IP_RANGE"]
     produced_events = ["OPEN_TCP_PORT"]
     flags = ["active", "portscan", "aggressive", "web-thorough"]
-    meta = {"description": "Execute port scans with nmap"}
+    meta = {"description": "Port scan with nmap. By default, scans top 100 ports."}
     options = {
-        "ports": "",
         "top_ports": 100,
+        "ports": "",
         "timing": "T4",
         "skip_host_discovery": True,
     }
     options_desc = {
-        "ports": "ports to scan",
-        "top_ports": "top ports to scan",
+        "top_ports": "Top ports to scan (default 100) (to override, specify 'ports')",
+        "ports": "Ports to scan",
         "timing": "-T<0-5>: Set timing template (higher is faster)",
         "skip_host_discovery": "skip host discovery (-Pn)",
     }
@@ -32,7 +32,7 @@ async def setup(self):
         self.timing = self.config.get("timing", "T4")
         self.top_ports = self.config.get("top_ports", 100)
         self.skip_host_discovery = self.config.get("skip_host_discovery", True)
-        return True
+        return await super().setup()
 
     async def handle_batch(self, *events):
         target = self.helpers.make_target(*events)
@@ -65,6 +65,8 @@ def construct_command(self, targets):
         temp_filename = self.helpers.temp_filename(extension="xml")
         command = [
             "nmap",
+            "--excludefile",
+            str(self.exclude_file),
             "-n",
             "--resolve-all",
             f"-{self.timing}",