Merge branch 'dev' into lightfuzz

.github/dependabot.yml

@@ -14,3 +14,4 @@ updates:
           - "*"  # Group all Actions updates into a single larger pull request
     schedule:
       interval: weekly
+    target-branch: "dev" 

README.md

@@ -226,8 +226,6 @@ config:
     baddns:
       enable_references: True
 
-
-
 ```
 
 </details>

bbot-docker.sh

@@ -1,2 +1,3 @@
-# run the docker image
-docker run --rm -it -v "$HOME/.bbot:/root/.bbot" -v "$HOME/.config/bbot:/root/.config/bbot" blacklanternsecurity/bbot:stable "$@"
+# OUTPUTS SCAN DATA TO ~/.bbot/scans
+
+docker run --rm -it -v "$HOME/.bbot/scans:/root/.bbot/scans" -v "$HOME/.config/bbot:/root/.config/bbot" blacklanternsecurity/bbot:stable "$@"

bbot/cli.py

@@ -78,7 +78,7 @@ async def _main():
             return
 
         # if we're listing modules or their options
-        if options.list_modules or options.list_module_options:
+        if options.list_modules or options.list_output_modules or options.list_module_options:
             # if no modules or flags are specified, enable everything
             if not (options.modules or options.output_modules or options.flags):
                 for module, preloaded in preset.module_loader.preloaded().items():
@@ -96,7 +96,17 @@ async def _main():
                 print("")
                 print("### MODULES ###")
                 print("")
-                for row in preset.module_loader.modules_table(preset.modules).splitlines():
+                modules = sorted(set(preset.scan_modules + preset.internal_modules))
+                for row in preset.module_loader.modules_table(modules).splitlines():
+                    print(row)
+                return
+
+            # --list-output-modules
+            if options.list_output_modules:
+                print("")
+                print("### OUTPUT MODULES ###")
+                print("")
+                for row in preset.module_loader.modules_table(preset.output_modules).splitlines():
                     print(row)
                 return
 
@@ -250,9 +260,7 @@ async def akeyboard_listen():
     finally:
         # save word cloud
         with suppress(BaseException):
-            save_success, filename = scan.helpers.word_cloud.save()
-            if save_success:
-                log_to_stderr(f"Saved word cloud ({len(scan.helpers.word_cloud):,} words) to {filename}")
+            scan.helpers.word_cloud.save()
         # remove output directory if empty
         with suppress(BaseException):
             scan.home.rmdir()

bbot/core/helpers/names_generator.py

@@ -293,6 +293,7 @@
     "alyssa",
     "amanda",
     "amber",
+    "amir",
     "amy",
     "andrea",
     "andrew",

bbot/modules/base.py

@@ -51,7 +51,7 @@ class BaseModule:
 
         target_only (bool): Accept only the initial target event(s). Default is False.
 
-        in_scope_only (bool): Accept only explicitly in-scope events. Default is False.
+        in_scope_only (bool): Accept only explicitly in-scope events, regardless of the scan's search distance. Default is False.
 
         options (Dict): Customizable options for the module, e.g., {"api_key": ""}. Empty dict by default.
 

bbot/modules/deadly/nuclei.py

@@ -15,7 +15,7 @@ class nuclei(BaseModule):
     }
 
     options = {
-        "version": "3.3.6",
+        "version": "3.3.7",
         "tags": "",
         "templates": "",
         "severity": "",

bbot/modules/extractous.py

@@ -112,7 +112,7 @@ def extract_text(file_path):
         result = ""
         buffer = reader.read(4096)
         while len(buffer) > 0:
-            result += buffer.decode("utf-8")
+            result += buffer.decode("utf-8", errors="ignore")
             buffer = reader.read(4096)
 
         return result.strip()

bbot/modules/httpx.py

@@ -1,5 +1,5 @@
 import re
-import json
+import orjson
 import tempfile
 import subprocess
 from pathlib import Path
@@ -156,11 +156,11 @@ async def handle_batch(self, *events):
         proxy = self.scan.http_proxy
         if proxy:
             command += ["-http-proxy", proxy]
-        async for line in self.run_process_live(command, input=list(stdin), stderr=subprocess.DEVNULL):
+        async for line in self.run_process_live(command, text=False, input=list(stdin), stderr=subprocess.DEVNULL):
             try:
-                j = json.loads(line)
-            except json.decoder.JSONDecodeError:
-                self.debug(f"Failed to decode line: {line}")
+                j = await self.helpers.run_in_executor(orjson.loads, line)
+            except orjson.JSONDecodeError:
+                self.warning(f"httpx failed to decode line: {line}")
                 continue
 
             url = j.get("url", "")

bbot/modules/internal/cloudcheck.py

@@ -5,7 +5,11 @@
 
 class CloudCheck(BaseInterceptModule):
     watched_events = ["*"]
-    meta = {"description": "Tag events by cloud provider, identify cloud resources like storage buckets"}
+    meta = {
+        "description": "Tag events by cloud provider, identify cloud resources like storage buckets",
+        "created_date": "2024-07-07",
+        "author": "@TheTechromancer",
+    }
     scope_distance_modifier = 1
     _priority = 3
 
@@ -68,9 +72,10 @@ async def handle_event(self, event, **kwargs):
                 base_kwargs["event_type"] = event_type
                 for sig in sigs:
                     matches = []
-                    if event.type == "HTTP_RESPONSE":
-                        matches = await self.helpers.re.findall(sig, event.data.get("body", ""))
-                    elif event.type.startswith("DNS_NAME"):
+                    # TODO: convert this to an excavate YARA hook
+                    # if event.type == "HTTP_RESPONSE":
+                    #     matches = await self.helpers.re.findall(sig, event.data.get("body", ""))
+                    if event.type.startswith("DNS_NAME"):
                         for host in str_hosts_to_check:
                             match = sig.match(host)
                             if match:

bbot/modules/internal/dnsresolve.py

@@ -9,6 +9,8 @@
 
 class DNSResolve(BaseInterceptModule):
     watched_events = ["*"]
+    produced_events = ["DNS_NAME", "IP_ADDRESS", "RAW_DNS_RECORD"]
+    meta = {"description": "Perform DNS resolution", "created_date": "2022-04-08", "author": "@TheTechromancer"}
     _priority = 1
     scope_distance_modifier = None
 

bbot/modules/internal/speculate.py

@@ -104,7 +104,7 @@ async def handle_event(self, event):
             # don't act on unresolved DNS_NAMEs
             usable_dns = False
             if event.type == "DNS_NAME":
-                if self.dns_disable or ("a-record" in event.tags or "aaaa-record" in event.tags):
+                if self.dns_disable or event.resolved_hosts:
                     usable_dns = True
 
             if event.type == "IP_ADDRESS" or usable_dns:

bbot/modules/output/mysql.py

@@ -3,7 +3,11 @@
 
 class MySQL(SQLTemplate):
     watched_events = ["*"]
-    meta = {"description": "Output scan data to a MySQL database"}
+    meta = {
+        "description": "Output scan data to a MySQL database",
+        "created_date": "2024-11-13",
+        "author": "@TheTechromancer",
+    }
     options = {
         "username": "root",
         "password": "bbotislife",

bbot/modules/output/nmap_xml.py

@@ -0,0 +1,171 @@
+import sys
+from xml.dom import minidom
+from datetime import datetime
+from xml.etree.ElementTree import Element, SubElement, tostring
+
+from bbot import __version__
+from bbot.modules.output.base import BaseOutputModule
+
+
+class NmapHost:
+    __slots__ = ["hostnames", "open_ports"]
+
+    def __init__(self):
+        self.hostnames = set()
+        # a dict of {port: {protocol: banner}}
+        self.open_ports = dict()
+
+
+class Nmap_XML(BaseOutputModule):
+    watched_events = ["OPEN_TCP_PORT", "DNS_NAME", "IP_ADDRESS", "PROTOCOL", "HTTP_RESPONSE"]
+    meta = {"description": "Output to Nmap XML", "created_date": "2024-11-16", "author": "@TheTechromancer"}
+    output_filename = "output.nmap.xml"
+    in_scope_only = True
+
+    async def setup(self):
+        self.hosts = {}
+        self._prep_output_dir(self.output_filename)
+        return True
+
+    async def handle_event(self, event):
+        event_host = event.host
+
+        # we always record by IP
+        ips = []
+        for ip in event.resolved_hosts:
+            try:
+                ips.append(self.helpers.make_ip_type(ip))
+            except ValueError:
+                continue
+        if not ips and self.helpers.is_ip(event_host):
+            ips = [event_host]
+
+        for ip in ips:
+            try:
+                nmap_host = self.hosts[ip]
+            except KeyError:
+                nmap_host = NmapHost()
+                self.hosts[ip] = nmap_host
+
+            event_port = getattr(event, "port", None)
+            if event.type == "OPEN_TCP_PORT":
+                if event_port not in nmap_host.open_ports:
+                    nmap_host.open_ports[event.port] = {}
+            elif event.type in ("PROTOCOL", "HTTP_RESPONSE"):
+                if event_port is not None:
+                    try:
+                        existing_services = nmap_host.open_ports[event.port]
+                    except KeyError:
+                        existing_services = {}
+                        nmap_host.open_ports[event.port] = existing_services
+                    if event.type == "PROTOCOL":
+                        protocol = event.data["protocol"].lower()
+                        banner = event.data.get("banner", None)
+                    elif event.type == "HTTP_RESPONSE":
+                        protocol = event.parsed_url.scheme.lower()
+                        banner = event.http_title
+                    if protocol not in existing_services:
+                        existing_services[protocol] = banner
+
+            if self.helpers.is_ip(event_host):
+                if str(event.module) == "PTR":
+                    nmap_host.hostnames.add(event.parent.data)
+            else:
+                nmap_host.hostnames.add(event_host)
+
+    async def report(self):
+        scan_start_time = str(int(self.scan.start_time.timestamp()))
+        scan_start_time_str = self.scan.start_time.strftime("%a %b %d %H:%M:%S %Y")
+        scan_end_time = datetime.now()
+        scan_end_time_str = scan_end_time.strftime("%a %b %d %H:%M:%S %Y")
+        scan_end_time_timestamp = str(scan_end_time.timestamp())
+        scan_duration = scan_end_time - self.scan.start_time
+        num_hosts_up = len(self.hosts)
+
+        # Create the root element
+        nmaprun = Element(
+            "nmaprun",
+            {
+                "scanner": "bbot",
+                "args": " ".join(sys.argv),
+                "start": scan_start_time,
+                "startstr": scan_start_time_str,
+                "version": str(__version__),
+                "xmloutputversion": "1.05",
+            },
+        )
+
+        ports_scanned = []
+        speculate_module = self.scan.modules.get("speculate", None)
+        if speculate_module is not None:
+            ports_scanned = speculate_module.ports
+        portscan_module = self.scan.modules.get("portscan", None)
+        if portscan_module is not None:
+            ports_scanned = self.helpers.parse_port_string(str(portscan_module.ports))
+        num_ports_scanned = len(sorted(ports_scanned))
+        ports_scanned = ",".join(str(x) for x in sorted(ports_scanned))
+
+        # Add scaninfo
+        SubElement(
+            nmaprun,
+            "scaninfo",
+            {"type": "syn", "protocol": "tcp", "numservices": str(num_ports_scanned), "services": ports_scanned},
+        )
+
+        # Add host information
+        for ip, nmap_host in self.hosts.items():
+            hostnames = sorted(nmap_host.hostnames)
+            ports = sorted(nmap_host.open_ports)
+
+            host_elem = SubElement(nmaprun, "host")
+            SubElement(host_elem, "status", {"state": "up", "reason": "user-set", "reason_ttl": "0"})
+            SubElement(host_elem, "address", {"addr": str(ip), "addrtype": f"ipv{ip.version}"})
+
+            if hostnames:
+                hostnames_elem = SubElement(host_elem, "hostnames")
+                for hostname in hostnames:
+                    SubElement(hostnames_elem, "hostname", {"name": hostname, "type": "user"})
+
+            ports = SubElement(host_elem, "ports")
+            for port, protocols in nmap_host.open_ports.items():
+                port_elem = SubElement(ports, "port", {"protocol": "tcp", "portid": str(port)})
+                SubElement(port_elem, "state", {"state": "open", "reason": "syn-ack", "reason_ttl": "0"})
+                # <port protocol="tcp" portid="443"><state state="open" reason="syn-ack" reason_ttl="53"/><service name="http" product="AkamaiGHost" extrainfo="Akamai&apos;s HTTP Acceleration/Mirror service" tunnel="ssl" method="probed" conf="10"/></port>
+                for protocol, banner in protocols.items():
+                    attrs = {"name": protocol, "method": "probed", "conf": "10"}
+                    if banner is not None:
+                        attrs["product"] = banner
+                        attrs["extrainfo"] = banner
+                    SubElement(port_elem, "service", attrs)
+
+        # Add runstats
+        runstats = SubElement(nmaprun, "runstats")
+        SubElement(
+            runstats,
+            "finished",
+            {
+                "time": scan_end_time_timestamp,
+                "timestr": scan_end_time_str,
+                "summary": f"BBOT done at {scan_end_time_str}; {num_hosts_up} scanned in {scan_duration} seconds",
+                "elapsed": str(scan_duration.total_seconds()),
+                "exit": "success",
+            },
+        )
+        SubElement(runstats, "hosts", {"up": str(num_hosts_up), "down": "0", "total": str(num_hosts_up)})
+
+        # make backup of the file
+        self.helpers.backup_file(self.output_file)
+
+        # Pretty-format the XML
+        rough_string = tostring(nmaprun, encoding="utf-8")
+        reparsed = minidom.parseString(rough_string)
+
+        # Create a new document with the doctype
+        doctype = minidom.DocumentType("nmaprun")
+        reparsed.insertBefore(doctype, reparsed.documentElement)
+
+        pretty_xml = reparsed.toprettyxml(indent="  ")
+
+        with open(self.output_file, "w") as f:
+            f.write(pretty_xml)
+        self.info(f"Saved Nmap XML output to {self.output_file}")

bbot/modules/output/postgres.py

@@ -3,7 +3,11 @@
 
 class Postgres(SQLTemplate):
     watched_events = ["*"]
-    meta = {"description": "Output scan data to a SQLite database"}
+    meta = {
+        "description": "Output scan data to a SQLite database",
+        "created_date": "2024-11-08",
+        "author": "@TheTechromancer",
+    }
     options = {
         "username": "postgres",
         "password": "bbotislife",

bbot/modules/output/sqlite.py

@@ -5,7 +5,11 @@
 
 class SQLite(SQLTemplate):
     watched_events = ["*"]
-    meta = {"description": "Output scan data to a SQLite database"}
+    meta = {
+        "description": "Output scan data to a SQLite database",
+        "created_date": "2024-11-07",
+        "author": "@TheTechromancer",
+    }
     options = {
         "database": "",
     }

bbot/modules/output/stdout.py

@@ -6,7 +6,7 @@
 
 class Stdout(BaseOutputModule):
     watched_events = ["*"]
-    meta = {"description": "Output to text"}
+    meta = {"description": "Output to text", "created_date": "2024-04-03", "author": "@TheTechromancer"}
     options = {"format": "text", "event_types": [], "event_fields": [], "in_scope_only": False, "accept_dupes": True}
     options_desc = {
         "format": "Which text format to display, choices: text,json",

bbot/modules/output/txt.py

@@ -5,7 +5,7 @@
 
 class TXT(BaseOutputModule):
     watched_events = ["*"]
-    meta = {"description": "Output to text"}
+    meta = {"description": "Output to text", "created_date": "2024-04-03", "author": "@TheTechromancer"}
     options = {"output_file": ""}
     options_desc = {"output_file": "Output to file"}
 

bbot/modules/trufflehog.py

@@ -13,7 +13,7 @@ class trufflehog(BaseModule):
     }
 
     options = {
-        "version": "3.84.1",
+        "version": "3.87.0",
         "config": "",
         "only_verified": True,
         "concurrency": 8,