From 8b1bd58cb0788fa08f86d2b3bd39f3dc6dcf9474 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 18 Nov 2024 17:50:02 -0500 Subject: [PATCH 01/12] fix speculate conflict --- bbot/cli.py | 14 +- bbot/modules/base.py | 2 +- bbot/modules/internal/speculate.py | 2 +- bbot/modules/output/nmap_xml.py | 161 ++++++++++++++++++ bbot/modules/output/stdout.py | 2 +- bbot/scanner/preset/args.py | 18 +- bbot/test/test_step_1/test_modules_basic.py | 20 +-- .../module_tests/test_module_nmap_xml.py | 85 +++++++++ 8 files changed, 282 insertions(+), 22 deletions(-) create mode 100644 bbot/modules/output/nmap_xml.py create mode 100644 bbot/test/test_step_2/module_tests/test_module_nmap_xml.py diff --git a/bbot/cli.py b/bbot/cli.py index 877f2bcaa5..4ffd3398b2 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -80,7 +80,7 @@ async def _main(): return # if we're listing modules or their options - if options.list_modules or options.list_module_options: + if options.list_modules or options.list_output_modules or options.list_module_options: # if no modules or flags are specified, enable everything if not (options.modules or options.output_modules or options.flags): @@ -99,7 +99,17 @@ async def _main(): print("") print("### MODULES ###") print("") - for row in preset.module_loader.modules_table(preset.modules).splitlines(): + modules = sorted(set(preset.scan_modules + preset.internal_modules)) + for row in preset.module_loader.modules_table(modules).splitlines(): + print(row) + return + + # --list-output-modules + if options.list_output_modules: + print("") + print("### OUTPUT MODULES ###") + print("") + for row in preset.module_loader.modules_table(preset.output_modules).splitlines(): print(row) return diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 956d59c98c..93593b9a34 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -51,7 +51,7 @@ class BaseModule: target_only (bool): Accept only the initial target event(s). Default is False. - in_scope_only (bool): Accept only explicitly in-scope events. Default is False. + in_scope_only (bool): Accept only explicitly in-scope events, regardless of the scan's search distance. Default is False. options (Dict): Customizable options for the module, e.g., {"api_key": ""}. Empty dict by default. diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index e52e4e1bb2..ef0f0ea46f 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -104,7 +104,7 @@ async def handle_event(self, event): # don't act on unresolved DNS_NAMEs usable_dns = False if event.type == "DNS_NAME": - if self.dns_disable or ("a-record" in event.tags or "aaaa-record" in event.tags): + if self.dns_disable or event.resolved_hosts: usable_dns = True if event.type == "IP_ADDRESS" or usable_dns: diff --git a/bbot/modules/output/nmap_xml.py b/bbot/modules/output/nmap_xml.py new file mode 100644 index 0000000000..5279a5a3c8 --- /dev/null +++ b/bbot/modules/output/nmap_xml.py @@ -0,0 +1,161 @@ +import sys +from xml.dom import minidom +from datetime import datetime +from xml.etree.ElementTree import Element, SubElement, tostring + +from bbot import __version__ +from bbot.modules.output.base import BaseOutputModule + + +class NmapHost: + __slots__ = ["hostnames", "open_ports"] + + def __init__(self): + self.hostnames = set() + # a dict of {port: {protocol: banner}} + self.open_ports = dict() + + +class Nmap_XML(BaseOutputModule): + watched_events = ["OPEN_TCP_PORT", "DNS_NAME", "IP_ADDRESS", "PROTOCOL"] + meta = {"description": "Output to Nmap XML", "created_date": "2024-11-16", "author": "@TheTechromancer"} + output_filename = "output.nmap.xml" + in_scope_only = True + + async def setup(self): + self.hosts = {} + self._prep_output_dir(self.output_filename) + return True + + async def handle_event(self, event): + event_host = event.host + + # we always record by IP + ips = [] + for ip in event.resolved_hosts: + try: + ips.append(self.helpers.make_ip_type(ip)) + except ValueError: + continue + if not ips and self.helpers.is_ip(event_host): + ips = [event_host] + + for ip in ips: + try: + nmap_host = self.hosts[ip] + except KeyError: + nmap_host = NmapHost() + self.hosts[ip] = nmap_host + + event_port = getattr(event, "port", None) + if event.type == "OPEN_TCP_PORT": + if event_port not in nmap_host.open_ports: + nmap_host.open_ports[event.port] = {} + elif event.type == "PROTOCOL": + if event_port is not None: + try: + existing_services = nmap_host.open_ports[event.port] + except KeyError: + existing_services = {} + nmap_host.open_ports[event.port] = existing_services + protocol = event.data["protocol"].lower() + if protocol not in existing_services: + existing_services[protocol] = event.data.get("banner", None) + + if self.helpers.is_ip(event_host): + if str(event.module) == "PTR": + nmap_host.hostnames.add(event.parent.data) + else: + nmap_host.hostnames.add(event_host) + + async def report(self): + scan_start_time = str(int(self.scan.start_time.timestamp())) + scan_start_time_str = self.scan.start_time.strftime("%a %b %d %H:%M:%S %Y") + scan_end_time = datetime.now() + scan_end_time_str = scan_end_time.strftime("%a %b %d %H:%M:%S %Y") + scan_end_time_timestamp = str(scan_end_time.timestamp()) + scan_duration = scan_end_time - self.scan.start_time + num_hosts_up = len(self.hosts) + + # Create the root element + nmaprun = Element( + "nmaprun", + { + "scanner": "bbot", + "args": " ".join(sys.argv), + "start": scan_start_time, + "startstr": scan_start_time_str, + "version": str(__version__), + "xmloutputversion": "1.05", + }, + ) + + ports_scanned = [] + speculate_module = self.scan.modules.get("speculate", None) + if speculate_module is not None: + ports_scanned = speculate_module.ports + portscan_module = self.scan.modules.get("portscan", None) + if portscan_module is not None: + ports_scanned = self.helpers.parse_port_string(str(portscan_module.ports)) + num_ports_scanned = len(sorted(ports_scanned)) + ports_scanned = ",".join(str(x) for x in sorted(ports_scanned)) + + # Add scaninfo + SubElement( + nmaprun, + "scaninfo", + {"type": "syn", "protocol": "tcp", "numservices": str(num_ports_scanned), "services": ports_scanned}, + ) + + # Add host information + for ip, nmap_host in self.hosts.items(): + hostnames = sorted(nmap_host.hostnames) + ports = sorted(nmap_host.open_ports) + + host_elem = SubElement(nmaprun, "host") + SubElement(host_elem, "status", {"state": "up", "reason": "user-set", "reason_ttl": "0"}) + SubElement(host_elem, "address", {"addr": str(ip), "addrtype": f"ipv{ip.version}"}) + + if hostnames: + hostnames_elem = SubElement(host_elem, "hostnames") + for hostname in hostnames: + SubElement(hostnames_elem, "hostname", {"name": hostname, "type": "user"}) + + ports = SubElement(host_elem, "ports") + for port, protocols in nmap_host.open_ports.items(): + port_elem = SubElement(ports, "port", {"protocol": "tcp", "portid": str(port)}) + SubElement(port_elem, "state", {"state": "open", "reason": "syn-ack", "reason_ttl": "0"}) + # + for protocol, banner in protocols.items(): + attrs = {"name": protocol, "method": "probed", "conf": "10"} + if banner is not None: + attrs["product"] = banner + attrs["extrainfo"] = banner + SubElement(port_elem, "service", attrs) + + # Add runstats + runstats = SubElement(nmaprun, "runstats") + SubElement( + runstats, + "finished", + { + "time": scan_end_time_timestamp, + "timestr": scan_end_time_str, + "summary": f"BBOT done at {scan_end_time_str}; {num_hosts_up} scanned in {scan_duration} seconds", + "elapsed": str(scan_duration.total_seconds()), + "exit": "success", + }, + ) + SubElement(runstats, "hosts", {"up": str(num_hosts_up), "down": "0", "total": str(num_hosts_up)}) + + # make backup of the file + self.helpers.backup_file(self.output_file) + + # Pretty-format the XML + rough_string = tostring(nmaprun, encoding="utf-8") + reparsed = minidom.parseString(rough_string) + pretty_xml = reparsed.toprettyxml(indent=" ") + + with open(self.output_file, "w") as f: + f.write(pretty_xml) + self.info(f"Saved Nmap XML output to {self.output_file}") diff --git a/bbot/modules/output/stdout.py b/bbot/modules/output/stdout.py index 6e4ccf5bea..33ffad5da2 100644 --- a/bbot/modules/output/stdout.py +++ b/bbot/modules/output/stdout.py @@ -6,7 +6,7 @@ class Stdout(BaseOutputModule): watched_events = ["*"] - meta = {"description": "Output to text"} + meta = {"description": "Output to text", "created_date": "2024-04-03", "author": "@TheTechromancer"} options = {"format": "text", "event_types": [], "event_fields": [], "in_scope_only": False, "accept_dupes": True} options_desc = { "format": "Which text format to display, choices: text,json", diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index cf48dd4b9f..9f9dc61ed1 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -53,6 +53,11 @@ class BBOTArgs: "", "bbot -l", ), + ( + "List output modules", + "", + "bbot -lo", + ), ( "List presets", "", @@ -289,12 +294,6 @@ def create_parser(self, *args, **kwargs): ) output = p.add_argument_group(title="Output") - output.add_argument( - "-o", - "--output-dir", - help="Directory to output scan results", - metavar="DIR", - ) output.add_argument( "-om", "--output-modules", @@ -303,6 +302,13 @@ def create_parser(self, *args, **kwargs): help=f'Output module(s). Choices: {",".join(self.preset.module_loader.output_module_choices)}', metavar="MODULE", ) + output.add_argument("-lo", "--list-output-modules", action="store_true", help="List available output modules") + output.add_argument( + "-o", + "--output-dir", + help="Directory to output scan results", + metavar="DIR", + ) output.add_argument("--json", "-j", action="store_true", help="Output scan data in JSON format") output.add_argument("--brief", "-br", action="store_true", help="Output only the data itself") output.add_argument("--event-types", nargs="+", default=[], help="Choose which event types to display") diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index 4212340695..863fdff05f 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -156,17 +156,15 @@ async def test_modules_basic_checks(events, httpx_mock): assert not ( "web-basic" in flags and "web-thorough" in flags ), f'module "{module_name}" should have either "web-basic" or "web-thorough" flags, not both' - meta = preloaded.get("meta", {}) - # make sure every module has a description - assert meta.get("description", ""), f"{module_name} must have a description" - # make sure every module has an author - assert meta.get("author", ""), f"{module_name} must have an author" - # make sure every module has a created date - created_date = meta.get("created_date", "") - assert created_date, f"{module_name} must have a created date" - assert created_date_regex.match( - created_date - ), f"{module_name}'s created_date must match the format YYYY-MM-DD" + meta = preloaded.get("meta", {}) + # make sure every module has a description + assert meta.get("description", ""), f"{module_name} must have a description" + # make sure every module has an author + assert meta.get("author", ""), f"{module_name} must have an author" + # make sure every module has a created date + created_date = meta.get("created_date", "") + assert created_date, f"{module_name} must have a created date" + assert created_date_regex.match(created_date), f"{module_name}'s created_date must match the format YYYY-MM-DD" # attribute checks watched_events = preloaded.get("watched_events") diff --git a/bbot/test/test_step_2/module_tests/test_module_nmap_xml.py b/bbot/test/test_step_2/module_tests/test_module_nmap_xml.py new file mode 100644 index 0000000000..b88595be01 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_nmap_xml.py @@ -0,0 +1,85 @@ +import xml.etree.ElementTree as ET + +from bbot.modules.base import BaseModule +from .base import ModuleTestBase + + +class TestNmap_XML(ModuleTestBase): + modules_overrides = ["nmap_xml", "speculate"] + targets = ["blacklanternsecurity.com", "127.0.0.3"] + config_overrides = {"dns": {"minimal": False}} + + class DummyModule(BaseModule): + watched_events = ["OPEN_TCP_PORT"] + _name = "dummy_module" + + async def handle_event(self, event): + if event.port == 80: + await self.emit_event( + {"host": str(event.host), "port": event.port, "protocol": "http", "banner": "Apache"}, + "PROTOCOL", + parent=event, + ) + elif event.port == 443: + await self.emit_event( + {"host": str(event.host), "port": event.port, "protocol": "https"}, "PROTOCOL", parent=event + ) + + async def setup_before_prep(self, module_test): + self.dummy_module = self.DummyModule(module_test.scan) + module_test.scan.modules["dummy_module"] = self.dummy_module + await module_test.mock_dns( + { + "blacklanternsecurity.com": {"A": ["127.0.0.1", "127.0.0.2"]}, + "3.0.0.127.in-addr.arpa": {"PTR": ["www.blacklanternsecurity.com"]}, + "www.blacklanternsecurity.com": {"A": ["127.0.0.1"]}, + } + ) + + def check(self, module_test, events): + nmap_xml_file = module_test.scan.modules["nmap_xml"].output_file + nmap_xml = open(nmap_xml_file).read() + + # Parse the XML + root = ET.fromstring(nmap_xml) + + # Expected IP addresses + expected_ips = {"127.0.0.1", "127.0.0.2", "127.0.0.3"} + found_ips = set() + + # Iterate over each host in the XML + for host in root.findall("host"): + # Get the IP address + address = host.find("address").get("addr") + found_ips.add(address) + + # Get hostnames if available + hostnames = sorted([hostname.get("name") for hostname in host.findall(".//hostname")]) + + # Get open ports and services + ports = [] + for port in host.findall(".//port"): + port_id = port.get("portid") + state = port.find("state").get("state") + if state == "open": + service_name = port.find("service").get("name") + service_product = port.find("service").get("product", "") + service_extrainfo = port.find("service").get("extrainfo", "") + ports.append((port_id, service_name, service_product, service_extrainfo)) + + # Sort ports for consistency + ports.sort() + + # Assertions + if address == "127.0.0.1": + assert hostnames == ["blacklanternsecurity.com", "www.blacklanternsecurity.com"] + assert ports == sorted([("80", "http", "Apache", "Apache"), ("443", "https", "", "")]) + elif address == "127.0.0.2": + assert hostnames == sorted(["blacklanternsecurity.com"]) + assert ports == sorted([("80", "http", "Apache", "Apache"), ("443", "https", "", "")]) + elif address == "127.0.0.3": + assert hostnames == [] # No hostnames for this IP + assert ports == sorted([("80", "http", "Apache", "Apache"), ("443", "https", "", "")]) + + # Assert that all expected IPs were found + assert found_ips == expected_ips From a410392ff0e893e34e4c70abffd302361f6acab1 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 17 Nov 2024 00:03:05 -0500 Subject: [PATCH 02/12] fix cli tests --- bbot/modules/output/nmap_xml.py | 5 +++++ bbot/test/test_step_1/test_cli.py | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/bbot/modules/output/nmap_xml.py b/bbot/modules/output/nmap_xml.py index 5279a5a3c8..38a5d8a035 100644 --- a/bbot/modules/output/nmap_xml.py +++ b/bbot/modules/output/nmap_xml.py @@ -154,6 +154,11 @@ async def report(self): # Pretty-format the XML rough_string = tostring(nmaprun, encoding="utf-8") reparsed = minidom.parseString(rough_string) + + # Create a new document with the doctype + doctype = minidom.DocumentType("nmaprun") + reparsed.insertBefore(doctype, reparsed.documentElement) + pretty_xml = reparsed.toprettyxml(indent=" ") with open(self.output_file, "w") as f: diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 47db12d2ab..0841216f09 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -150,11 +150,23 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config): out, err = capsys.readouterr() # internal modules assert "| excavate " in out - # output modules - assert "| csv " in out + # no output modules + assert not "| csv " in out # scan modules assert "| wayback " in out + # list output modules + monkeypatch.setattr("sys.argv", ["bbot", "--list-output-modules"]) + result = await cli._main() + assert result == None + out, err = capsys.readouterr() + # no internal modules + assert not "| excavate " in out + # output modules + assert "| csv " in out + # no scan modules + assert not "| wayback " in out + # output dir and scan name output_dir = bbot_test_dir / "bbot_cli_args_output" scan_name = "bbot_cli_args_scan_name" From fa044f10b836d602a1e00b5663a0c4002fc40525 Mon Sep 17 00:00:00 2001 From: github-actions Date: Sun, 17 Nov 2024 01:25:58 -0500 Subject: [PATCH 03/12] update module descriptions --- bbot/modules/internal/cloudcheck.py | 6 +++++- bbot/modules/internal/dnsresolve.py | 2 ++ bbot/modules/output/sqlite.py | 6 +++++- bbot/modules/output/txt.py | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/bbot/modules/internal/cloudcheck.py b/bbot/modules/internal/cloudcheck.py index 392c8e0c5a..2cd748601e 100644 --- a/bbot/modules/internal/cloudcheck.py +++ b/bbot/modules/internal/cloudcheck.py @@ -3,7 +3,11 @@ class CloudCheck(BaseInterceptModule): watched_events = ["*"] - meta = {"description": "Tag events by cloud provider, identify cloud resources like storage buckets"} + meta = { + "description": "Tag events by cloud provider, identify cloud resources like storage buckets", + "created_date": "2024-07-07", + "author": "@TheTechromancer", + } scope_distance_modifier = 1 _priority = 3 diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 9b68b7bb9d..f6a08b60c5 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -9,6 +9,8 @@ class DNSResolve(BaseInterceptModule): watched_events = ["*"] + produced_events = ["DNS_NAME", "IP_ADDRESS", "RAW_DNS_RECORD"] + meta = {"description": "Perform DNS resolution", "created_date": "2022-04-08", "author": "@TheTechromancer"} _priority = 1 scope_distance_modifier = None diff --git a/bbot/modules/output/sqlite.py b/bbot/modules/output/sqlite.py index 68ac60dafd..0377caaa3d 100644 --- a/bbot/modules/output/sqlite.py +++ b/bbot/modules/output/sqlite.py @@ -5,7 +5,11 @@ class SQLite(SQLTemplate): watched_events = ["*"] - meta = {"description": "Output scan data to a SQLite database"} + meta = { + "description": "Output scan data to a SQLite database", + "created_date": "2024-11-07", + "author": "@TheTechromancer", + } options = { "database": "", } diff --git a/bbot/modules/output/txt.py b/bbot/modules/output/txt.py index 68f86864da..2dfb14c106 100644 --- a/bbot/modules/output/txt.py +++ b/bbot/modules/output/txt.py @@ -5,7 +5,7 @@ class TXT(BaseOutputModule): watched_events = ["*"] - meta = {"description": "Output to text"} + meta = {"description": "Output to text", "created_date": "2024-04-03", "author": "@TheTechromancer"} options = {"output_file": ""} options_desc = {"output_file": "Output to file"} From 211d651ed017a20705e363967173782a5b560b7f Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 18 Nov 2024 17:49:02 -0500 Subject: [PATCH 04/12] support http_response --- bbot/modules/output/nmap_xml.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/bbot/modules/output/nmap_xml.py b/bbot/modules/output/nmap_xml.py index 38a5d8a035..9e540f4242 100644 --- a/bbot/modules/output/nmap_xml.py +++ b/bbot/modules/output/nmap_xml.py @@ -17,7 +17,7 @@ def __init__(self): class Nmap_XML(BaseOutputModule): - watched_events = ["OPEN_TCP_PORT", "DNS_NAME", "IP_ADDRESS", "PROTOCOL"] + watched_events = ["OPEN_TCP_PORT", "DNS_NAME", "IP_ADDRESS", "PROTOCOL", "HTTP_RESPONSE"] meta = {"description": "Output to Nmap XML", "created_date": "2024-11-16", "author": "@TheTechromancer"} output_filename = "output.nmap.xml" in_scope_only = True @@ -28,6 +28,7 @@ async def setup(self): return True async def handle_event(self, event): + self.hugesuccess(event) event_host = event.host # we always record by IP @@ -51,16 +52,21 @@ async def handle_event(self, event): if event.type == "OPEN_TCP_PORT": if event_port not in nmap_host.open_ports: nmap_host.open_ports[event.port] = {} - elif event.type == "PROTOCOL": + elif event.type in ("PROTOCOL", "HTTP_RESPONSE"): if event_port is not None: try: existing_services = nmap_host.open_ports[event.port] except KeyError: existing_services = {} nmap_host.open_ports[event.port] = existing_services - protocol = event.data["protocol"].lower() + if event.type == "PROTOCOL": + protocol = event.data["protocol"].lower() + banner = event.data.get("banner", None) + elif event.type == "HTTP_RESPONSE": + protocol = event.parsed_url.scheme.lower() + banner = event.http_title if protocol not in existing_services: - existing_services[protocol] = event.data.get("banner", None) + existing_services[protocol] = banner if self.helpers.is_ip(event_host): if str(event.module) == "PTR": From 820c6e41ea03ed11aa600dc9506a89abc431366f Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 19 Nov 2024 19:44:51 -0500 Subject: [PATCH 05/12] fix tests, remove debug statement --- bbot/modules/output/nmap_xml.py | 1 - bbot/modules/output/postgres.py | 6 +++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bbot/modules/output/nmap_xml.py b/bbot/modules/output/nmap_xml.py index 9e540f4242..52698e0de8 100644 --- a/bbot/modules/output/nmap_xml.py +++ b/bbot/modules/output/nmap_xml.py @@ -28,7 +28,6 @@ async def setup(self): return True async def handle_event(self, event): - self.hugesuccess(event) event_host = event.host # we always record by IP diff --git a/bbot/modules/output/postgres.py b/bbot/modules/output/postgres.py index b1c8c26598..45beb7c7bc 100644 --- a/bbot/modules/output/postgres.py +++ b/bbot/modules/output/postgres.py @@ -3,7 +3,11 @@ class Postgres(SQLTemplate): watched_events = ["*"] - meta = {"description": "Output scan data to a SQLite database"} + meta = { + "description": "Output scan data to a SQLite database", + "created_date": "2024-11-08", + "author": "@TheTechromancer", + } options = { "username": "postgres", "password": "bbotislife", From cb1b96789748be2fa9b92caf296c71c5f884a015 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 20 Nov 2024 11:48:24 -0500 Subject: [PATCH 06/12] fix portscan test --- .../test_step_2/module_tests/test_module_portscan.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bbot/test/test_step_2/module_tests/test_module_portscan.py b/bbot/test/test_step_2/module_tests/test_module_portscan.py index 56536cb5dd..d9f55c27f9 100644 --- a/bbot/test/test_step_2/module_tests/test_module_portscan.py +++ b/bbot/test/test_step_2/module_tests/test_module_portscan.py @@ -109,10 +109,12 @@ def check(self, module_test, events): if e.type == "DNS_NAME" and e.data == "dummy.asdf.evilcorp.net" and str(e.module) == "dummy_module" ] ) - assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.8.8"]) <= 3 - assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.4"]) <= 3 - assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.5"]) <= 3 - assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.6"]) <= 3 + # the reason these numbers aren't exactly predictable is because we can't predict which one arrives first + # to the portscan module. Sometimes, one that would normally be deduped is force-emitted because it led to a new open port. + assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.8.8"]) <= 4 + assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.4"]) <= 4 + assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.5"]) <= 4 + assert 2 <= len([e for e in events if e.type == "IP_ADDRESS" and e.data == "8.8.4.6"]) <= 4 assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.8.8:443"]) assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.4.5:80"]) assert 1 == len([e for e in events if e.type == "OPEN_TCP_PORT" and e.data == "8.8.4.6:631"]) From 2133597b5481be9d9cf01aab1695d357cc31415f Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 4 Dec 2024 11:00:29 -0500 Subject: [PATCH 07/12] ruff --- bbot/cli.py | 1 - bbot/modules/internal/dnsresolve.py | 9 +++++++-- bbot/modules/internal/excavate.py | 2 ++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/bbot/cli.py b/bbot/cli.py index d556e7413a..d06810ee70 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -79,7 +79,6 @@ async def _main(): # if we're listing modules or their options if options.list_modules or options.list_output_modules or options.list_module_options: - # if no modules or flags are specified, enable everything if not (options.modules or options.output_modules or options.flags): for module, preloaded in preset.module_loader.preloaded().items(): diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index fbf8747406..3dddd289a4 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -85,9 +85,14 @@ async def handle_event(self, event, **kwargs): event_data_changed = await self.handle_wildcard_event(main_host_event) if event_data_changed: # since data has changed, we check again whether it's a duplicate - if event.type == "DNS_NAME" and self.scan.ingress_module.is_incoming_duplicate(event, add=True): + if event.type == "DNS_NAME" and self.scan.ingress_module.is_incoming_duplicate( + event, add=True + ): if not event._graph_important: - return False, "it's a DNS wildcard, and its module already emitted a similar wildcard event" + return ( + False, + "it's a DNS wildcard, and its module already emitted a similar wildcard event", + ) else: self.debug( f"Event {event} was already emitted by its module, but it's graph-important so it gets a pass" diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py index 9d33621815..209b96eefb 100644 --- a/bbot/modules/internal/excavate.py +++ b/bbot/modules/internal/excavate.py @@ -656,8 +656,10 @@ async def process(self, yara_results, event, yara_rule_settings, discovery_conte continue if parsed_url.scheme in ["http", "https"]: continue + def abort_if(e): return e.scope_distance > 0 + finding_data = {"host": str(host), "description": f"Non-HTTP URI: {parsed_url.geturl()}"} await self.report(finding_data, event, yara_rule_settings, discovery_context, abort_if=abort_if) protocol_data = {"protocol": parsed_url.scheme, "host": str(host)} From d52da462c4b1bb39d57cc1064e8038eb0d7d7781 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 4 Dec 2024 11:25:36 -0500 Subject: [PATCH 08/12] ruff go away --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 34974e1d3b..2ad06885d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -102,7 +102,7 @@ skip = "./docs/javascripts/vega*.js,./bbot/wordlists/*" [tool.ruff] line-length = 119 format.exclude = ["bbot/test/test_step_1/test_manager_*"] -lint.ignore = ["E402", "E721", "E741", "F401", "F403", "F405"] +lint.ignore = ["E402", "E711", "E713", "E721", "E741", "F401", "F403", "F405"] [tool.poetry-dynamic-versioning] enable = true From 865fd1717b32bb1cacbcc3ff0e5da3eb53fb9f04 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 4 Dec 2024 13:07:38 -0500 Subject: [PATCH 09/12] mysql author/created date --- bbot/modules/output/mysql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bbot/modules/output/mysql.py b/bbot/modules/output/mysql.py index 69856a8a33..6099d18ceb 100644 --- a/bbot/modules/output/mysql.py +++ b/bbot/modules/output/mysql.py @@ -3,7 +3,7 @@ class MySQL(SQLTemplate): watched_events = ["*"] - meta = {"description": "Output scan data to a MySQL database"} + meta = {"description": "Output scan data to a MySQL database", "created_date": "2024-11-13", "author": "@TheTechromancer"} options = { "username": "root", "password": "bbotislife", From 6608c9dd2e86e9666b66c63c651f48fb88b37485 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 4 Dec 2024 17:00:40 -0500 Subject: [PATCH 10/12] amir --- bbot/core/helpers/names_generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index a0a569e530..e2d9b74311 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -293,6 +293,7 @@ "alyssa", "amanda", "amber", + "amir", "amy", "andrea", "andrew", From 3d40f6b550fab575c8d9da053e08a9c3a622d27b Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 10:41:41 -0500 Subject: [PATCH 11/12] update docker --- bbot-docker.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bbot-docker.sh b/bbot-docker.sh index e4e0bb9e43..3db958f94a 100755 --- a/bbot-docker.sh +++ b/bbot-docker.sh @@ -1,2 +1,3 @@ -# run the docker image -docker run --rm -it -v "$HOME/.bbot:/root/.bbot" -v "$HOME/.config/bbot:/root/.config/bbot" blacklanternsecurity/bbot:stable "$@" +# OUTPUTS SCAN DATA TO ~/.bbot/scans + +docker run --rm -it -v "$HOME/.bbot/scans:/root/.bbot/scans" -v "$HOME/.config/bbot:/root/.config/bbot" blacklanternsecurity/bbot:stable "$@" From 4d08937221b23661e84f55356b2c5b56eb357d48 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 11:12:34 -0500 Subject: [PATCH 12/12] better docker documentation --- docs/index.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/index.md b/docs/index.md index 3d6c5ef267..355d58e8b2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,6 +34,8 @@ bbot --help Docker images are provided, along with helper script `bbot-docker.sh` to persist your scan data. +Scans are output to `~/.bbot/scans` (the usual place for BBOT scan data). + ```bash # bleeding edge (dev) docker run -it blacklanternsecurity/bbot --help @@ -46,6 +48,16 @@ git clone https://github.com/blacklanternsecurity/bbot && cd bbot ./bbot-docker.sh --help ``` +Note: If you need to pass in a custom preset, you can do so by mapping the preset into the container: + +```bash +# use the preset `my_preset.yml` from the current directory +docker run --rm -it \ + -v "$HOME/.bbot/scans:/root/.bbot/scans" \ + -v "$PWD/my_preset.yml:/my_preset.yml" \ + blacklanternsecurity/bbot -p /my_preset.yml +``` + ## Example Commands Below are some examples of common scans.