diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 18e2537a56..1ad88ccb68 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,3 +14,4 @@ updates: - "*" # Group all Actions updates into a single larger pull request schedule: interval: weekly + target-branch: "dev" diff --git a/README.md b/README.md index a9823b73f6..9ceaf336d9 100644 --- a/README.md +++ b/README.md @@ -226,8 +226,6 @@ config: baddns: enable_references: True - - ``` diff --git a/bbot-docker.sh b/bbot-docker.sh index e4e0bb9e43..3db958f94a 100755 --- a/bbot-docker.sh +++ b/bbot-docker.sh @@ -1,2 +1,3 @@ -# run the docker image -docker run --rm -it -v "$HOME/.bbot:/root/.bbot" -v "$HOME/.config/bbot:/root/.config/bbot" blacklanternsecurity/bbot:stable "$@" +# OUTPUTS SCAN DATA TO ~/.bbot/scans + +docker run --rm -it -v "$HOME/.bbot/scans:/root/.bbot/scans" -v "$HOME/.config/bbot:/root/.config/bbot" blacklanternsecurity/bbot:stable "$@" diff --git a/bbot/cli.py b/bbot/cli.py index 97d7488088..db595dfc1d 100755 --- a/bbot/cli.py +++ b/bbot/cli.py @@ -78,7 +78,7 @@ async def _main(): return # if we're listing modules or their options - if options.list_modules or options.list_module_options: + if options.list_modules or options.list_output_modules or options.list_module_options: # if no modules or flags are specified, enable everything if not (options.modules or options.output_modules or options.flags): for module, preloaded in preset.module_loader.preloaded().items(): @@ -96,7 +96,17 @@ async def _main(): print("") print("### MODULES ###") print("") - for row in preset.module_loader.modules_table(preset.modules).splitlines(): + modules = sorted(set(preset.scan_modules + preset.internal_modules)) + for row in preset.module_loader.modules_table(modules).splitlines(): + print(row) + return + + # --list-output-modules + if options.list_output_modules: + print("") + print("### OUTPUT MODULES ###") + print("") + for row in preset.module_loader.modules_table(preset.output_modules).splitlines(): print(row) return @@ -250,9 +260,7 @@ async def akeyboard_listen(): finally: # save word cloud with suppress(BaseException): - save_success, filename = scan.helpers.word_cloud.save() - if save_success: - log_to_stderr(f"Saved word cloud ({len(scan.helpers.word_cloud):,} words) to {filename}") + scan.helpers.word_cloud.save() # remove output directory if empty with suppress(BaseException): scan.home.rmdir() diff --git a/bbot/core/helpers/names_generator.py b/bbot/core/helpers/names_generator.py index a0a569e530..e2d9b74311 100644 --- a/bbot/core/helpers/names_generator.py +++ b/bbot/core/helpers/names_generator.py @@ -293,6 +293,7 @@ "alyssa", "amanda", "amber", + "amir", "amy", "andrea", "andrew", diff --git a/bbot/modules/base.py b/bbot/modules/base.py index 4ab2da1528..2362983897 100644 --- a/bbot/modules/base.py +++ b/bbot/modules/base.py @@ -51,7 +51,7 @@ class BaseModule: target_only (bool): Accept only the initial target event(s). Default is False. - in_scope_only (bool): Accept only explicitly in-scope events. Default is False. + in_scope_only (bool): Accept only explicitly in-scope events, regardless of the scan's search distance. Default is False. options (Dict): Customizable options for the module, e.g., {"api_key": ""}. Empty dict by default. diff --git a/bbot/modules/deadly/nuclei.py b/bbot/modules/deadly/nuclei.py index 5de7aa465d..4f2d73c986 100644 --- a/bbot/modules/deadly/nuclei.py +++ b/bbot/modules/deadly/nuclei.py @@ -15,7 +15,7 @@ class nuclei(BaseModule): } options = { - "version": "3.3.6", + "version": "3.3.7", "tags": "", "templates": "", "severity": "", diff --git a/bbot/modules/extractous.py b/bbot/modules/extractous.py index 5b2e2cdec7..9d2ae153b4 100644 --- a/bbot/modules/extractous.py +++ b/bbot/modules/extractous.py @@ -112,7 +112,7 @@ def extract_text(file_path): result = "" buffer = reader.read(4096) while len(buffer) > 0: - result += buffer.decode("utf-8") + result += buffer.decode("utf-8", errors="ignore") buffer = reader.read(4096) return result.strip() diff --git a/bbot/modules/httpx.py b/bbot/modules/httpx.py index 664ad697b7..21fa48d63d 100644 --- a/bbot/modules/httpx.py +++ b/bbot/modules/httpx.py @@ -1,5 +1,5 @@ import re -import json +import orjson import tempfile import subprocess from pathlib import Path @@ -156,11 +156,11 @@ async def handle_batch(self, *events): proxy = self.scan.http_proxy if proxy: command += ["-http-proxy", proxy] - async for line in self.run_process_live(command, input=list(stdin), stderr=subprocess.DEVNULL): + async for line in self.run_process_live(command, text=False, input=list(stdin), stderr=subprocess.DEVNULL): try: - j = json.loads(line) - except json.decoder.JSONDecodeError: - self.debug(f"Failed to decode line: {line}") + j = await self.helpers.run_in_executor(orjson.loads, line) + except orjson.JSONDecodeError: + self.warning(f"httpx failed to decode line: {line}") continue url = j.get("url", "") diff --git a/bbot/modules/internal/cloudcheck.py b/bbot/modules/internal/cloudcheck.py index 86b6130d71..c45acfe954 100644 --- a/bbot/modules/internal/cloudcheck.py +++ b/bbot/modules/internal/cloudcheck.py @@ -5,7 +5,11 @@ class CloudCheck(BaseInterceptModule): watched_events = ["*"] - meta = {"description": "Tag events by cloud provider, identify cloud resources like storage buckets"} + meta = { + "description": "Tag events by cloud provider, identify cloud resources like storage buckets", + "created_date": "2024-07-07", + "author": "@TheTechromancer", + } scope_distance_modifier = 1 _priority = 3 @@ -68,9 +72,10 @@ async def handle_event(self, event, **kwargs): base_kwargs["event_type"] = event_type for sig in sigs: matches = [] - if event.type == "HTTP_RESPONSE": - matches = await self.helpers.re.findall(sig, event.data.get("body", "")) - elif event.type.startswith("DNS_NAME"): + # TODO: convert this to an excavate YARA hook + # if event.type == "HTTP_RESPONSE": + # matches = await self.helpers.re.findall(sig, event.data.get("body", "")) + if event.type.startswith("DNS_NAME"): for host in str_hosts_to_check: match = sig.match(host) if match: diff --git a/bbot/modules/internal/dnsresolve.py b/bbot/modules/internal/dnsresolve.py index 5bb5c5bc40..3dddd289a4 100644 --- a/bbot/modules/internal/dnsresolve.py +++ b/bbot/modules/internal/dnsresolve.py @@ -9,6 +9,8 @@ class DNSResolve(BaseInterceptModule): watched_events = ["*"] + produced_events = ["DNS_NAME", "IP_ADDRESS", "RAW_DNS_RECORD"] + meta = {"description": "Perform DNS resolution", "created_date": "2022-04-08", "author": "@TheTechromancer"} _priority = 1 scope_distance_modifier = None diff --git a/bbot/modules/internal/speculate.py b/bbot/modules/internal/speculate.py index 2c9cb55ad7..2555cd7d7e 100644 --- a/bbot/modules/internal/speculate.py +++ b/bbot/modules/internal/speculate.py @@ -104,7 +104,7 @@ async def handle_event(self, event): # don't act on unresolved DNS_NAMEs usable_dns = False if event.type == "DNS_NAME": - if self.dns_disable or ("a-record" in event.tags or "aaaa-record" in event.tags): + if self.dns_disable or event.resolved_hosts: usable_dns = True if event.type == "IP_ADDRESS" or usable_dns: diff --git a/bbot/modules/output/mysql.py b/bbot/modules/output/mysql.py index 69856a8a33..8d9a1f7f4c 100644 --- a/bbot/modules/output/mysql.py +++ b/bbot/modules/output/mysql.py @@ -3,7 +3,11 @@ class MySQL(SQLTemplate): watched_events = ["*"] - meta = {"description": "Output scan data to a MySQL database"} + meta = { + "description": "Output scan data to a MySQL database", + "created_date": "2024-11-13", + "author": "@TheTechromancer", + } options = { "username": "root", "password": "bbotislife", diff --git a/bbot/modules/output/nmap_xml.py b/bbot/modules/output/nmap_xml.py new file mode 100644 index 0000000000..52698e0de8 --- /dev/null +++ b/bbot/modules/output/nmap_xml.py @@ -0,0 +1,171 @@ +import sys +from xml.dom import minidom +from datetime import datetime +from xml.etree.ElementTree import Element, SubElement, tostring + +from bbot import __version__ +from bbot.modules.output.base import BaseOutputModule + + +class NmapHost: + __slots__ = ["hostnames", "open_ports"] + + def __init__(self): + self.hostnames = set() + # a dict of {port: {protocol: banner}} + self.open_ports = dict() + + +class Nmap_XML(BaseOutputModule): + watched_events = ["OPEN_TCP_PORT", "DNS_NAME", "IP_ADDRESS", "PROTOCOL", "HTTP_RESPONSE"] + meta = {"description": "Output to Nmap XML", "created_date": "2024-11-16", "author": "@TheTechromancer"} + output_filename = "output.nmap.xml" + in_scope_only = True + + async def setup(self): + self.hosts = {} + self._prep_output_dir(self.output_filename) + return True + + async def handle_event(self, event): + event_host = event.host + + # we always record by IP + ips = [] + for ip in event.resolved_hosts: + try: + ips.append(self.helpers.make_ip_type(ip)) + except ValueError: + continue + if not ips and self.helpers.is_ip(event_host): + ips = [event_host] + + for ip in ips: + try: + nmap_host = self.hosts[ip] + except KeyError: + nmap_host = NmapHost() + self.hosts[ip] = nmap_host + + event_port = getattr(event, "port", None) + if event.type == "OPEN_TCP_PORT": + if event_port not in nmap_host.open_ports: + nmap_host.open_ports[event.port] = {} + elif event.type in ("PROTOCOL", "HTTP_RESPONSE"): + if event_port is not None: + try: + existing_services = nmap_host.open_ports[event.port] + except KeyError: + existing_services = {} + nmap_host.open_ports[event.port] = existing_services + if event.type == "PROTOCOL": + protocol = event.data["protocol"].lower() + banner = event.data.get("banner", None) + elif event.type == "HTTP_RESPONSE": + protocol = event.parsed_url.scheme.lower() + banner = event.http_title + if protocol not in existing_services: + existing_services[protocol] = banner + + if self.helpers.is_ip(event_host): + if str(event.module) == "PTR": + nmap_host.hostnames.add(event.parent.data) + else: + nmap_host.hostnames.add(event_host) + + async def report(self): + scan_start_time = str(int(self.scan.start_time.timestamp())) + scan_start_time_str = self.scan.start_time.strftime("%a %b %d %H:%M:%S %Y") + scan_end_time = datetime.now() + scan_end_time_str = scan_end_time.strftime("%a %b %d %H:%M:%S %Y") + scan_end_time_timestamp = str(scan_end_time.timestamp()) + scan_duration = scan_end_time - self.scan.start_time + num_hosts_up = len(self.hosts) + + # Create the root element + nmaprun = Element( + "nmaprun", + { + "scanner": "bbot", + "args": " ".join(sys.argv), + "start": scan_start_time, + "startstr": scan_start_time_str, + "version": str(__version__), + "xmloutputversion": "1.05", + }, + ) + + ports_scanned = [] + speculate_module = self.scan.modules.get("speculate", None) + if speculate_module is not None: + ports_scanned = speculate_module.ports + portscan_module = self.scan.modules.get("portscan", None) + if portscan_module is not None: + ports_scanned = self.helpers.parse_port_string(str(portscan_module.ports)) + num_ports_scanned = len(sorted(ports_scanned)) + ports_scanned = ",".join(str(x) for x in sorted(ports_scanned)) + + # Add scaninfo + SubElement( + nmaprun, + "scaninfo", + {"type": "syn", "protocol": "tcp", "numservices": str(num_ports_scanned), "services": ports_scanned}, + ) + + # Add host information + for ip, nmap_host in self.hosts.items(): + hostnames = sorted(nmap_host.hostnames) + ports = sorted(nmap_host.open_ports) + + host_elem = SubElement(nmaprun, "host") + SubElement(host_elem, "status", {"state": "up", "reason": "user-set", "reason_ttl": "0"}) + SubElement(host_elem, "address", {"addr": str(ip), "addrtype": f"ipv{ip.version}"}) + + if hostnames: + hostnames_elem = SubElement(host_elem, "hostnames") + for hostname in hostnames: + SubElement(hostnames_elem, "hostname", {"name": hostname, "type": "user"}) + + ports = SubElement(host_elem, "ports") + for port, protocols in nmap_host.open_ports.items(): + port_elem = SubElement(ports, "port", {"protocol": "tcp", "portid": str(port)}) + SubElement(port_elem, "state", {"state": "open", "reason": "syn-ack", "reason_ttl": "0"}) + # + for protocol, banner in protocols.items(): + attrs = {"name": protocol, "method": "probed", "conf": "10"} + if banner is not None: + attrs["product"] = banner + attrs["extrainfo"] = banner + SubElement(port_elem, "service", attrs) + + # Add runstats + runstats = SubElement(nmaprun, "runstats") + SubElement( + runstats, + "finished", + { + "time": scan_end_time_timestamp, + "timestr": scan_end_time_str, + "summary": f"BBOT done at {scan_end_time_str}; {num_hosts_up} scanned in {scan_duration} seconds", + "elapsed": str(scan_duration.total_seconds()), + "exit": "success", + }, + ) + SubElement(runstats, "hosts", {"up": str(num_hosts_up), "down": "0", "total": str(num_hosts_up)}) + + # make backup of the file + self.helpers.backup_file(self.output_file) + + # Pretty-format the XML + rough_string = tostring(nmaprun, encoding="utf-8") + reparsed = minidom.parseString(rough_string) + + # Create a new document with the doctype + doctype = minidom.DocumentType("nmaprun") + reparsed.insertBefore(doctype, reparsed.documentElement) + + pretty_xml = reparsed.toprettyxml(indent=" ") + + with open(self.output_file, "w") as f: + f.write(pretty_xml) + self.info(f"Saved Nmap XML output to {self.output_file}") diff --git a/bbot/modules/output/postgres.py b/bbot/modules/output/postgres.py index b1c8c26598..45beb7c7bc 100644 --- a/bbot/modules/output/postgres.py +++ b/bbot/modules/output/postgres.py @@ -3,7 +3,11 @@ class Postgres(SQLTemplate): watched_events = ["*"] - meta = {"description": "Output scan data to a SQLite database"} + meta = { + "description": "Output scan data to a SQLite database", + "created_date": "2024-11-08", + "author": "@TheTechromancer", + } options = { "username": "postgres", "password": "bbotislife", diff --git a/bbot/modules/output/sqlite.py b/bbot/modules/output/sqlite.py index 5926c961eb..261b13b6e2 100644 --- a/bbot/modules/output/sqlite.py +++ b/bbot/modules/output/sqlite.py @@ -5,7 +5,11 @@ class SQLite(SQLTemplate): watched_events = ["*"] - meta = {"description": "Output scan data to a SQLite database"} + meta = { + "description": "Output scan data to a SQLite database", + "created_date": "2024-11-07", + "author": "@TheTechromancer", + } options = { "database": "", } diff --git a/bbot/modules/output/stdout.py b/bbot/modules/output/stdout.py index 520a90204b..59a121bd47 100644 --- a/bbot/modules/output/stdout.py +++ b/bbot/modules/output/stdout.py @@ -6,7 +6,7 @@ class Stdout(BaseOutputModule): watched_events = ["*"] - meta = {"description": "Output to text"} + meta = {"description": "Output to text", "created_date": "2024-04-03", "author": "@TheTechromancer"} options = {"format": "text", "event_types": [], "event_fields": [], "in_scope_only": False, "accept_dupes": True} options_desc = { "format": "Which text format to display, choices: text,json", diff --git a/bbot/modules/output/txt.py b/bbot/modules/output/txt.py index 68f86864da..2dfb14c106 100644 --- a/bbot/modules/output/txt.py +++ b/bbot/modules/output/txt.py @@ -5,7 +5,7 @@ class TXT(BaseOutputModule): watched_events = ["*"] - meta = {"description": "Output to text"} + meta = {"description": "Output to text", "created_date": "2024-04-03", "author": "@TheTechromancer"} options = {"output_file": ""} options_desc = {"output_file": "Output to file"} diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py index 80a4385c2e..8441c73648 100644 --- a/bbot/modules/trufflehog.py +++ b/bbot/modules/trufflehog.py @@ -13,7 +13,7 @@ class trufflehog(BaseModule): } options = { - "version": "3.84.1", + "version": "3.87.0", "config": "", "only_verified": True, "concurrency": 8, diff --git a/bbot/scanner/preset/args.py b/bbot/scanner/preset/args.py index 88219000ec..ffb0dbe3a8 100644 --- a/bbot/scanner/preset/args.py +++ b/bbot/scanner/preset/args.py @@ -52,6 +52,11 @@ class BBOTArgs: "", "bbot -l", ), + ( + "List output modules", + "", + "bbot -lo", + ), ( "List presets", "", @@ -130,14 +135,17 @@ def preset_from_args(self): args_preset.core.merge_custom({"modules": {"stdout": {"event_types": self.parsed.event_types}}}) # dependencies + deps_config = args_preset.core.custom_config.get("deps", {}) if self.parsed.retry_deps: - args_preset.core.custom_config["deps_behavior"] = "retry_failed" + deps_config["behavior"] = "retry_failed" elif self.parsed.force_deps: - args_preset.core.custom_config["deps_behavior"] = "force_install" + deps_config["behavior"] = "force_install" elif self.parsed.no_deps: - args_preset.core.custom_config["deps_behavior"] = "disable" + deps_config["behavior"] = "disable" elif self.parsed.ignore_failed_deps: - args_preset.core.custom_config["deps_behavior"] = "ignore_failed" + deps_config["behavior"] = "ignore_failed" + if deps_config: + args_preset.core.merge_custom({"deps": deps_config}) # other scan options if self.parsed.name is not None: @@ -307,6 +315,7 @@ def create_parser(self, *args, **kwargs): help=f'Output module(s). Choices: {",".join(sorted(self.preset.module_loader.output_module_choices))}', metavar="MODULE", ) + output.add_argument("-lo", "--list-output-modules", action="store_true", help="List available output modules") output.add_argument("--json", "-j", action="store_true", help="Output scan data in JSON format") output.add_argument("--brief", "-br", action="store_true", help="Output only the data itself") output.add_argument("--event-types", nargs="+", default=[], help="Choose which event types to display") diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py index b275cc1f72..1ea9ebb2cf 100644 --- a/bbot/scanner/preset/preset.py +++ b/bbot/scanner/preset/preset.py @@ -798,7 +798,7 @@ def to_dict(self, include_target=False, full_config=False, redact_secrets=False) # misc scan options if self.scan_name: preset_dict["scan_name"] = self.scan_name - if self.scan_name: + if self.scan_name and self.output_dir is not None: preset_dict["output_dir"] = self.output_dir # conditions diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py index c541277082..2036b9d1ff 100644 --- a/bbot/scanner/scanner.py +++ b/bbot/scanner/scanner.py @@ -124,6 +124,7 @@ def __init__( self.duration_seconds = None self._success = False + self._scan_finish_status_message = None if scan_id is not None: self.id = str(scan_id) @@ -431,14 +432,19 @@ async def async_start(self): self._stop_log_handlers() + if self._scan_finish_status_message: + log_fn = self.hugesuccess + if self.status.startswith("ABORT"): + log_fn = self.hugewarning + elif not self._success: + log_fn = self.critical + log_fn(self._scan_finish_status_message) + async def _mark_finished(self): - log_fn = self.hugesuccess if self.status == "ABORTING": status = "ABORTED" - log_fn = self.hugewarning elif not self._success: status = "FAILED" - log_fn = self.critical else: status = "FINISHED" @@ -447,9 +453,9 @@ async def _mark_finished(self): self.duration_seconds = self.duration.total_seconds() self.duration_human = self.helpers.human_timedelta(self.duration) - status_message = f"Scan {self.name} completed in {self.duration_human} with status {status}" + self._scan_finish_status_message = f"Scan {self.name} completed in {self.duration_human} with status {status}" - scan_finish_event = self.finish_event(status_message, status) + scan_finish_event = self.finish_event(self._scan_finish_status_message, status) # queue final scan event with output modules output_modules = [m for m in self.modules.values() if m._type == "output" and m.name != "python"] @@ -463,7 +469,6 @@ async def _mark_finished(self): await asyncio.sleep(0.05) self.status = status - log_fn(status_message) return scan_finish_event def _start_modules(self): diff --git a/bbot/scanner/target.py b/bbot/scanner/target.py index 5608fdf4fe..f86b0de15b 100644 --- a/bbot/scanner/target.py +++ b/bbot/scanner/target.py @@ -101,7 +101,7 @@ def add(self, targets): events.add(event) # sort by host size to ensure consistency - events = sorted(events, key=lambda e: (0 if not e.host else host_size_key(e.host))) + events = sorted(events, key=lambda e: ((0, 0) if not e.host else host_size_key(e.host))) for event in events: self.events.add(event) self._add(event.host, data=event) @@ -192,7 +192,6 @@ def __init__(self, *args, **kwargs): @special_target_type(r"^(?:RE|REGEX):(.*)") def handle_regex(self, match): pattern = match.group(1) - log.info(f"Blacklisting by custom regex: {pattern}") blacklist_regex = re.compile(pattern, re.IGNORECASE) self.blacklist_regexes.add(blacklist_regex) return [] @@ -225,6 +224,12 @@ def _hash_value(self): hosts = [str(h).encode() for h in self.sorted_hosts] return hosts + regex_patterns + def __len__(self): + return super().__len__() + len(self.blacklist_regexes) + + def __bool__(self): + return bool(len(self)) + class BBOTTarget: """ diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py index 5ee272b635..c700cfaad8 100644 --- a/bbot/test/test_step_1/test_cli.py +++ b/bbot/test/test_step_1/test_cli.py @@ -1,3 +1,5 @@ +import yaml + from ..bbot_fixtures import * from bbot import cli @@ -143,6 +145,20 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config): assert len(out.splitlines()) == 1 assert out.count(".") > 1 + # deps behavior + monkeypatch.setattr("sys.argv", ["bbot", "-n", "depstest", "--retry-deps", "--current-preset"]) + result = await cli._main() + assert result is None + out, err = capsys.readouterr() + print(out) + # parse YAML output + preset = yaml.safe_load(out) + assert preset == { + "description": "depstest", + "scan_name": "depstest", + "config": {"deps": {"behavior": "retry_failed"}}, + } + # list modules monkeypatch.setattr("sys.argv", ["bbot", "--list-modules"]) result = await cli._main() @@ -150,11 +166,23 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config): out, err = capsys.readouterr() # internal modules assert "| excavate " in out - # output modules - assert "| csv " in out + # no output modules + assert not "| csv " in out # scan modules assert "| wayback " in out + # list output modules + monkeypatch.setattr("sys.argv", ["bbot", "--list-output-modules"]) + result = await cli._main() + assert result == None + out, err = capsys.readouterr() + # no internal modules + assert not "| excavate " in out + # output modules + assert "| csv " in out + # no scan modules + assert not "| wayback " in out + # output dir and scan name output_dir = bbot_test_dir / "bbot_cli_args_output" scan_name = "bbot_cli_args_scan_name" @@ -389,7 +417,6 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config): async def test_cli_customheaders(monkeypatch, caplog, capsys): monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True) monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True) - import yaml # test custom headers monkeypatch.setattr( diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py index e8052464c6..fbb8b35f15 100644 --- a/bbot/test/test_step_1/test_modules_basic.py +++ b/bbot/test/test_step_1/test_modules_basic.py @@ -156,17 +156,15 @@ async def test_modules_basic_checks(events, httpx_mock): assert not ( "web-basic" in flags and "web-thorough" in flags ), f'module "{module_name}" should have either "web-basic" or "web-thorough" flags, not both' - meta = preloaded.get("meta", {}) - # make sure every module has a description - assert meta.get("description", ""), f"{module_name} must have a description" - # make sure every module has an author - assert meta.get("author", ""), f"{module_name} must have an author" - # make sure every module has a created date - created_date = meta.get("created_date", "") - assert created_date, f"{module_name} must have a created date" - assert created_date_regex.match( - created_date - ), f"{module_name}'s created_date must match the format YYYY-MM-DD" + meta = preloaded.get("meta", {}) + # make sure every module has a description + assert meta.get("description", ""), f"{module_name} must have a description" + # make sure every module has an author + assert meta.get("author", ""), f"{module_name} must have an author" + # make sure every module has a created date + created_date = meta.get("created_date", "") + assert created_date, f"{module_name} must have a created date" + assert created_date_regex.match(created_date), f"{module_name}'s created_date must match the format YYYY-MM-DD" # attribute checks watched_events = preloaded.get("watched_events") diff --git a/bbot/test/test_step_1/test_target.py b/bbot/test/test_step_1/test_target.py index 8f2a6bf91f..4bc269595a 100644 --- a/bbot/test/test_step_1/test_target.py +++ b/bbot/test/test_step_1/test_target.py @@ -2,7 +2,7 @@ @pytest.mark.asyncio -async def test_target(bbot_scanner): +async def test_target_basic(bbot_scanner): from radixtarget import RadixTarget from ipaddress import ip_address, ip_network from bbot.scanner.target import BBOTTarget, ScanSeeds @@ -245,6 +245,17 @@ async def test_target(bbot_scanner): assert len(events) == 3 assert {e.type for e in events} == {"SCAN", "USERNAME"} + # users + orgs + domains + scan = bbot_scanner("USER:evilcorp", "ORG:evilcorp", "evilcorp.com") + await scan.helpers.dns._mock_dns( + { + "evilcorp.com": {"A": ["1.2.3.4"]}, + }, + ) + events = [e async for e in scan.async_start()] + assert len(events) == 5 + assert {e.type for e in events} == {"SCAN", "USERNAME", "ORG_STUB", "DNS_NAME"} + # verify hash values bbottarget = BBOTTarget( "1.2.3.0/24", @@ -395,6 +406,7 @@ async def test_blacklist_regex(bbot_scanner, bbot_httpserver): config={"excavate": True}, debug=True, ) + assert len(scan.target.blacklist) == 2 assert scan.target.blacklist.blacklist_regexes assert {r.pattern for r in scan.target.blacklist.blacklist_regexes} == { r"evil[0-9]{3}", diff --git a/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py b/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py index 902f2df35f..0ce93ec00d 100644 --- a/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py +++ b/bbot/test/test_step_2/module_tests/test_module_cloudcheck.py @@ -8,7 +8,7 @@ class TestCloudCheck(ModuleTestBase): modules_overrides = ["httpx", "excavate", "cloudcheck"] async def setup_after_prep(self, module_test): - module_test.set_expect_requests({"uri": "/"}, {"response_data": ""}) + module_test.set_expect_requests({"uri": "/"}, {"response_data": ""}) scan = Scanner(config={"cloudcheck": True}) await scan._prep() diff --git a/bbot/test/test_step_2/module_tests/test_module_nmap_xml.py b/bbot/test/test_step_2/module_tests/test_module_nmap_xml.py new file mode 100644 index 0000000000..b88595be01 --- /dev/null +++ b/bbot/test/test_step_2/module_tests/test_module_nmap_xml.py @@ -0,0 +1,85 @@ +import xml.etree.ElementTree as ET + +from bbot.modules.base import BaseModule +from .base import ModuleTestBase + + +class TestNmap_XML(ModuleTestBase): + modules_overrides = ["nmap_xml", "speculate"] + targets = ["blacklanternsecurity.com", "127.0.0.3"] + config_overrides = {"dns": {"minimal": False}} + + class DummyModule(BaseModule): + watched_events = ["OPEN_TCP_PORT"] + _name = "dummy_module" + + async def handle_event(self, event): + if event.port == 80: + await self.emit_event( + {"host": str(event.host), "port": event.port, "protocol": "http", "banner": "Apache"}, + "PROTOCOL", + parent=event, + ) + elif event.port == 443: + await self.emit_event( + {"host": str(event.host), "port": event.port, "protocol": "https"}, "PROTOCOL", parent=event + ) + + async def setup_before_prep(self, module_test): + self.dummy_module = self.DummyModule(module_test.scan) + module_test.scan.modules["dummy_module"] = self.dummy_module + await module_test.mock_dns( + { + "blacklanternsecurity.com": {"A": ["127.0.0.1", "127.0.0.2"]}, + "3.0.0.127.in-addr.arpa": {"PTR": ["www.blacklanternsecurity.com"]}, + "www.blacklanternsecurity.com": {"A": ["127.0.0.1"]}, + } + ) + + def check(self, module_test, events): + nmap_xml_file = module_test.scan.modules["nmap_xml"].output_file + nmap_xml = open(nmap_xml_file).read() + + # Parse the XML + root = ET.fromstring(nmap_xml) + + # Expected IP addresses + expected_ips = {"127.0.0.1", "127.0.0.2", "127.0.0.3"} + found_ips = set() + + # Iterate over each host in the XML + for host in root.findall("host"): + # Get the IP address + address = host.find("address").get("addr") + found_ips.add(address) + + # Get hostnames if available + hostnames = sorted([hostname.get("name") for hostname in host.findall(".//hostname")]) + + # Get open ports and services + ports = [] + for port in host.findall(".//port"): + port_id = port.get("portid") + state = port.find("state").get("state") + if state == "open": + service_name = port.find("service").get("name") + service_product = port.find("service").get("product", "") + service_extrainfo = port.find("service").get("extrainfo", "") + ports.append((port_id, service_name, service_product, service_extrainfo)) + + # Sort ports for consistency + ports.sort() + + # Assertions + if address == "127.0.0.1": + assert hostnames == ["blacklanternsecurity.com", "www.blacklanternsecurity.com"] + assert ports == sorted([("80", "http", "Apache", "Apache"), ("443", "https", "", "")]) + elif address == "127.0.0.2": + assert hostnames == sorted(["blacklanternsecurity.com"]) + assert ports == sorted([("80", "http", "Apache", "Apache"), ("443", "https", "", "")]) + elif address == "127.0.0.3": + assert hostnames == [] # No hostnames for this IP + assert ports == sorted([("80", "http", "Apache", "Apache"), ("443", "https", "", "")]) + + # Assert that all expected IPs were found + assert found_ips == expected_ips diff --git a/docs/data/chord_graph/entities.json b/docs/data/chord_graph/entities.json index 88242097ed..00633c95fa 100644 --- a/docs/data/chord_graph/entities.json +++ b/docs/data/chord_graph/entities.json @@ -23,11 +23,11 @@ ] }, { - "id": 129, + "id": 131, "name": "AZURE_TENANT", "parent": 88888888, "consumes": [ - 128 + 130 ], "produces": [] }, @@ -36,20 +36,20 @@ "name": "CODE_REPOSITORY", "parent": 88888888, "consumes": [ - 62, - 82, - 85, - 87, - 117, - 136 + 63, + 83, + 86, + 88, + 119, + 138 ], "produces": [ 42, - 63, - 83, + 64, 84, - 86, - 116 + 85, + 87, + 118 ] }, { @@ -87,36 +87,37 @@ 58, 59, 60, - 61, - 67, - 79, - 83, - 90, - 94, - 96, - 102, + 62, + 68, + 80, + 84, + 91, + 95, + 97, 103, - 107, - 108, - 112, - 113, + 104, + 106, + 109, + 110, 114, - 118, - 121, - 122, + 115, + 116, + 120, 123, 124, 125, - 128, - 131, - 132, + 126, + 127, + 130, 133, + 134, 135, - 138, - 141, - 142, - 145, - 148 + 137, + 140, + 143, + 144, + 147, + 150 ], "produces": [ 6, @@ -137,31 +138,32 @@ 58, 59, 60, - 79, - 90, - 94, - 96, - 102, + 61, + 80, + 91, + 95, + 97, 103, - 105, + 104, 107, - 108, - 112, - 118, - 121, + 109, + 110, + 114, + 120, 123, - 124, - 128, + 125, + 126, 130, - 131, 132, - 135, - 138, - 139, + 133, + 134, + 137, + 140, 141, - 142, - 145, - 148 + 143, + 144, + 147, + 150 ] }, { @@ -170,8 +172,8 @@ "parent": 88888888, "consumes": [ 21, - 128, - 133 + 130, + 135 ], "produces": [] }, @@ -180,19 +182,19 @@ "name": "EMAIL_ADDRESS", "parent": 88888888, "consumes": [ - 68 + 69 ], "produces": [ 45, 52, 58, - 61, - 67, - 94, - 113, - 122, - 125, - 130 + 62, + 68, + 95, + 115, + 124, + 127, + 132 ] }, { @@ -200,18 +202,18 @@ "name": "FILESYSTEM", "parent": 88888888, "consumes": [ - 72, - 101, - 136 + 73, + 102, + 138 ], "produces": [ 8, - 62, - 76, - 82, - 85, - 101, - 117 + 63, + 77, + 83, + 86, + 102, + 119 ] }, { @@ -220,7 +222,7 @@ "parent": 88888888, "consumes": [ 14, - 146 + 148 ], "produces": [ 1, @@ -235,33 +237,33 @@ 34, 37, 51, - 81, - 86, - 91, - 93, - 96, - 104, + 82, + 87, + 92, + 94, + 97, 105, - 106, - 109, - 110, - 120, - 126, + 107, + 108, + 111, + 112, + 122, 128, - 134, + 130, 136, - 137, - 147 + 138, + 139, + 149 ] }, { - "id": 98, + "id": 99, "name": "GEOLOCATION", "parent": 88888888, "consumes": [], "produces": [ - 97, - 100 + 98, + 101 ] }, { @@ -283,24 +285,25 @@ 14, 26, 51, - 66, - 69, - 76, - 86, - 91, - 104, + 67, + 70, + 77, + 87, + 92, 105, - 109, - 110, + 106, + 107, 111, - 120, - 128, - 134, - 144, - 147 + 112, + 113, + 122, + 130, + 136, + 146, + 149 ], "produces": [ - 92 + 93 ] }, { @@ -310,26 +313,28 @@ "consumes": [ 11, 14, - 96, 97, - 99, + 98, 100, - 114, - 128 + 101, + 106, + 116, + 130 ], "produces": [ 14, - 99, - 128 + 61, + 100, + 130 ] }, { - "id": 115, + "id": 117, "name": "IP_RANGE", "parent": 88888888, "consumes": [ - 114, - 128 + 116, + 130 ], "produces": [] }, @@ -341,7 +346,7 @@ 8 ], "produces": [ - 87 + 88 ] }, { @@ -350,29 +355,30 @@ "parent": 88888888, "consumes": [ 14, - 77, - 92, - 130 + 78, + 93, + 106, + 132 ], "produces": [ 14, - 96, - 114, - 128 + 97, + 116, + 130 ] }, { - "id": 64, + "id": 65, "name": "ORG_STUB", "parent": 88888888, "consumes": [ - 63, - 84, - 87, - 116 + 64, + 85, + 88, + 118 ], "produces": [ - 128 + 130 ] }, { @@ -386,12 +392,14 @@ ] }, { - "id": 78, + "id": 79, "name": "PROTOCOL", "parent": 88888888, - "consumes": [], + "consumes": [ + 106 + ], "produces": [ - 77 + 78 ] }, { @@ -401,36 +409,37 @@ "consumes": [], "produces": [ 54, - 61 + 61, + 62 ] }, { - "id": 70, + "id": 71, "name": "RAW_TEXT", "parent": 88888888, "consumes": [ - 69 + 70 ], "produces": [ - 72 + 73 ] }, { - "id": 65, + "id": 66, "name": "SOCIAL", "parent": 88888888, "consumes": [ - 63, - 84, - 86, - 88, - 116, - 128 + 64, + 85, + 87, + 89, + 118, + 130 ], "produces": [ - 63, - 86, - 127 + 64, + 87, + 129 ] }, { @@ -445,7 +454,7 @@ 32, 33, 34, - 128 + 130 ], "produces": [ 29, @@ -461,19 +470,19 @@ "parent": 88888888, "consumes": [ 14, - 86, - 146, - 147 + 87, + 148, + 149 ], "produces": [ 26, - 66, - 86, - 88, - 96, - 106, - 144, - 147 + 67, + 87, + 89, + 97, + 108, + 146, + 149 ] }, { @@ -485,37 +494,37 @@ 14, 23, 37, - 73, - 80, + 74, 81, - 88, - 92, - 95, - 105, - 106, - 119, - 126, + 82, + 89, + 93, + 96, + 107, + 108, + 121, 128, - 134, - 137, + 130, + 136, 139, - 143, - 146 + 141, + 145, + 148 ], "produces": [ - 88, - 92 + 89, + 93 ] }, { - "id": 75, + "id": 76, "name": "URL_HINT", "parent": 88888888, "consumes": [ - 74 + 75 ], "produces": [ - 95 + 96 ] }, { @@ -524,11 +533,11 @@ "parent": 88888888, "consumes": [ 42, - 76, - 92, - 107, - 127, - 128 + 77, + 93, + 109, + 129, + 130 ], "produces": [ 18, @@ -536,19 +545,19 @@ 32, 54, 58, - 61, - 63, - 69, - 73, + 62, + 64, + 70, 74, - 83, - 88, - 94, - 119, - 122, - 138, - 145, - 147 + 75, + 84, + 89, + 95, + 121, + 124, + 140, + 147, + 149 ] }, { @@ -556,7 +565,7 @@ "name": "USERNAME", "parent": 88888888, "consumes": [ - 128 + 130 ], "produces": [ 45, @@ -564,14 +573,14 @@ ] }, { - "id": 140, + "id": 142, "name": "VHOST", "parent": 88888888, "consumes": [ - 146 + 148 ], "produces": [ - 139 + 141 ] }, { @@ -580,7 +589,7 @@ "parent": 88888888, "consumes": [ 14, - 146 + 148 ], "produces": [ 1, @@ -589,13 +598,13 @@ 25, 26, 51, - 66, - 80, - 96, - 106, - 134, + 67, + 81, + 97, + 108, 136, - 147 + 138, + 149 ] }, { @@ -606,33 +615,33 @@ 14 ], "produces": [ - 143 + 145 ] }, { - "id": 89, + "id": 90, "name": "WEBSCREENSHOT", "parent": 88888888, "consumes": [], "produces": [ - 88 + 89 ] }, { - "id": 71, + "id": 72, "name": "WEB_PARAMETER", "parent": 88888888, "consumes": [ - 93, - 109, - 110, - 111 + 94, + 111, + 112, + 113 ], "produces": [ - 69, - 109, - 110, - 111 + 70, + 111, + 112, + 113 ] }, { @@ -1105,6 +1114,17 @@ }, { "id": 61, + "name": "dnsresolve", + "parent": 99999999, + "consumes": [], + "produces": [ + 7, + 12, + 55 + ] + }, + { + "id": 62, "name": "dnstlsrpt", "parent": 99999999, "consumes": [ @@ -1117,7 +1137,7 @@ ] }, { - "id": 62, + "id": 63, "name": "docker_pull", "parent": 99999999, "consumes": [ @@ -1128,21 +1148,21 @@ ] }, { - "id": 63, + "id": 64, "name": "dockerhub", "parent": 99999999, "consumes": [ - 64, - 65 + 65, + 66 ], "produces": [ 43, - 65, + 66, 19 ] }, { - "id": 66, + "id": 67, "name": "dotnetnuke", "parent": 99999999, "consumes": [ @@ -1154,7 +1174,7 @@ ] }, { - "id": 67, + "id": 68, "name": "emailformat", "parent": 99999999, "consumes": [ @@ -1165,7 +1185,7 @@ ] }, { - "id": 68, + "id": 69, "name": "emails", "parent": 99999999, "consumes": [ @@ -1174,31 +1194,31 @@ "produces": [] }, { - "id": 69, + "id": 70, "name": "excavate", "parent": 99999999, "consumes": [ 2, - 70 + 71 ], "produces": [ 19, - 71 + 72 ] }, { - "id": 72, + "id": 73, "name": "extractous", "parent": 99999999, "consumes": [ 10 ], "produces": [ - 70 + 71 ] }, { - "id": 73, + "id": 74, "name": "ffuf", "parent": 99999999, "consumes": [ @@ -1209,18 +1229,18 @@ ] }, { - "id": 74, + "id": 75, "name": "ffuf_shortnames", "parent": 99999999, "consumes": [ - 75 + 76 ], "produces": [ 19 ] }, { - "id": 76, + "id": 77, "name": "filedownload", "parent": 99999999, "consumes": [ @@ -1232,18 +1252,18 @@ ] }, { - "id": 77, + "id": 78, "name": "fingerprintx", "parent": 99999999, "consumes": [ 15 ], "produces": [ - 78 + 79 ] }, { - "id": 79, + "id": 80, "name": "fullhunt", "parent": 99999999, "consumes": [ @@ -1254,7 +1274,7 @@ ] }, { - "id": 80, + "id": 81, "name": "generic_ssrf", "parent": 99999999, "consumes": [ @@ -1265,7 +1285,7 @@ ] }, { - "id": 81, + "id": 82, "name": "git", "parent": 99999999, "consumes": [ @@ -1276,7 +1296,7 @@ ] }, { - "id": 82, + "id": 83, "name": "git_clone", "parent": 99999999, "consumes": [ @@ -1287,7 +1307,7 @@ ] }, { - "id": 83, + "id": 84, "name": "github_codesearch", "parent": 99999999, "consumes": [ @@ -1299,19 +1319,19 @@ ] }, { - "id": 84, + "id": 85, "name": "github_org", "parent": 99999999, "consumes": [ - 64, - 65 + 65, + 66 ], "produces": [ 43 ] }, { - "id": 85, + "id": 86, "name": "github_workflows", "parent": 99999999, "consumes": [ @@ -1322,50 +1342,50 @@ ] }, { - "id": 86, + "id": 87, "name": "gitlab", "parent": 99999999, "consumes": [ 2, - 65, + 66, 16 ], "produces": [ 43, 4, - 65, + 66, 16 ] }, { - "id": 87, + "id": 88, "name": "google_playstore", "parent": 99999999, "consumes": [ 43, - 64 + 65 ], "produces": [ 9 ] }, { - "id": 88, + "id": 89, "name": "gowitness", "parent": 99999999, "consumes": [ - 65, + 66, 3 ], "produces": [ 16, 3, 19, - 89 + 90 ] }, { - "id": 90, + "id": 91, "name": "hackertarget", "parent": 99999999, "consumes": [ @@ -1376,7 +1396,7 @@ ] }, { - "id": 91, + "id": 92, "name": "host_header", "parent": 99999999, "consumes": [ @@ -1387,7 +1407,7 @@ ] }, { - "id": 92, + "id": 93, "name": "httpx", "parent": 99999999, "consumes": [ @@ -1401,18 +1421,18 @@ ] }, { - "id": 93, + "id": 94, "name": "hunt", "parent": 99999999, "consumes": [ - 71 + 72 ], "produces": [ 4 ] }, { - "id": 94, + "id": 95, "name": "hunterio", "parent": 99999999, "consumes": [ @@ -1425,18 +1445,18 @@ ] }, { - "id": 95, + "id": 96, "name": "iis_shortnames", "parent": 99999999, "consumes": [ 3 ], "produces": [ - 75 + 76 ] }, { - "id": 96, + "id": 97, "name": "internetdb", "parent": 99999999, "consumes": [ @@ -1452,18 +1472,18 @@ ] }, { - "id": 97, + "id": 98, "name": "ip2location", "parent": 99999999, "consumes": [ 12 ], "produces": [ - 98 + 99 ] }, { - "id": 99, + "id": 100, "name": "ipneighbor", "parent": 99999999, "consumes": [ @@ -1474,18 +1494,18 @@ ] }, { - "id": 100, + "id": 101, "name": "ipstack", "parent": 99999999, "consumes": [ 12 ], "produces": [ - 98 + 99 ] }, { - "id": 101, + "id": 102, "name": "jadx", "parent": 99999999, "consumes": [ @@ -1496,7 +1516,7 @@ ] }, { - "id": 102, + "id": 103, "name": "leakix", "parent": 99999999, "consumes": [ @@ -1507,7 +1527,7 @@ ] }, { - "id": 103, + "id": 104, "name": "myssl", "parent": 99999999, "consumes": [ @@ -1518,7 +1538,7 @@ ] }, { - "id": 104, + "id": 105, "name": "newsletters", "parent": 99999999, "consumes": [ @@ -1529,7 +1549,20 @@ ] }, { - "id": 105, + "id": 106, + "name": "nmap_xml", + "parent": 99999999, + "consumes": [ + 7, + 2, + 12, + 15, + 79 + ], + "produces": [] + }, + { + "id": 107, "name": "ntlm", "parent": 99999999, "consumes": [ @@ -1542,7 +1575,7 @@ ] }, { - "id": 106, + "id": 108, "name": "nuclei", "parent": 99999999, "consumes": [ @@ -1555,7 +1588,7 @@ ] }, { - "id": 107, + "id": 109, "name": "oauth", "parent": 99999999, "consumes": [ @@ -1567,7 +1600,7 @@ ] }, { - "id": 108, + "id": 110, "name": "otx", "parent": 99999999, "consumes": [ @@ -1578,45 +1611,45 @@ ] }, { - "id": 109, + "id": 111, "name": "paramminer_cookies", "parent": 99999999, "consumes": [ 2, - 71 + 72 ], "produces": [ 4, - 71 + 72 ] }, { - "id": 110, + "id": 112, "name": "paramminer_getparams", "parent": 99999999, "consumes": [ 2, - 71 + 72 ], "produces": [ 4, - 71 + 72 ] }, { - "id": 111, + "id": 113, "name": "paramminer_headers", "parent": 99999999, "consumes": [ 2, - 71 + 72 ], "produces": [ - 71 + 72 ] }, { - "id": 112, + "id": 114, "name": "passivetotal", "parent": 99999999, "consumes": [ @@ -1627,7 +1660,7 @@ ] }, { - "id": 113, + "id": 115, "name": "pgp", "parent": 99999999, "consumes": [ @@ -1638,32 +1671,32 @@ ] }, { - "id": 114, + "id": 116, "name": "portscan", "parent": 99999999, "consumes": [ 7, 12, - 115 + 117 ], "produces": [ 15 ] }, { - "id": 116, + "id": 118, "name": "postman", "parent": 99999999, "consumes": [ - 64, - 65 + 65, + 66 ], "produces": [ 43 ] }, { - "id": 117, + "id": 119, "name": "postman_download", "parent": 99999999, "consumes": [ @@ -1674,7 +1707,7 @@ ] }, { - "id": 118, + "id": 120, "name": "rapiddns", "parent": 99999999, "consumes": [ @@ -1685,7 +1718,7 @@ ] }, { - "id": 119, + "id": 121, "name": "robots", "parent": 99999999, "consumes": [ @@ -1696,7 +1729,7 @@ ] }, { - "id": 120, + "id": 122, "name": "secretsdb", "parent": 99999999, "consumes": [ @@ -1707,7 +1740,7 @@ ] }, { - "id": 121, + "id": 123, "name": "securitytrails", "parent": 99999999, "consumes": [ @@ -1718,7 +1751,7 @@ ] }, { - "id": 122, + "id": 124, "name": "securitytxt", "parent": 99999999, "consumes": [ @@ -1730,7 +1763,7 @@ ] }, { - "id": 123, + "id": 125, "name": "shodan_dns", "parent": 99999999, "consumes": [ @@ -1741,7 +1774,7 @@ ] }, { - "id": 124, + "id": 126, "name": "sitedossier", "parent": 99999999, "consumes": [ @@ -1752,7 +1785,7 @@ ] }, { - "id": 125, + "id": 127, "name": "skymem", "parent": 99999999, "consumes": [ @@ -1763,7 +1796,7 @@ ] }, { - "id": 126, + "id": 128, "name": "smuggler", "parent": 99999999, "consumes": [ @@ -1774,28 +1807,28 @@ ] }, { - "id": 127, + "id": 129, "name": "social", "parent": 99999999, "consumes": [ 19 ], "produces": [ - 65 + 66 ] }, { - "id": 128, + "id": 130, "name": "speculate", "parent": 99999999, "consumes": [ - 129, + 131, 7, 22, 2, 12, - 115, - 65, + 117, + 66, 24, 3, 19, @@ -1806,11 +1839,11 @@ 4, 12, 15, - 64 + 65 ] }, { - "id": 130, + "id": 132, "name": "sslcert", "parent": 99999999, "consumes": [ @@ -1822,7 +1855,7 @@ ] }, { - "id": 131, + "id": 133, "name": "subdomaincenter", "parent": 99999999, "consumes": [ @@ -1833,7 +1866,7 @@ ] }, { - "id": 132, + "id": 134, "name": "subdomainradar", "parent": 99999999, "consumes": [ @@ -1844,7 +1877,7 @@ ] }, { - "id": 133, + "id": 135, "name": "subdomains", "parent": 99999999, "consumes": [ @@ -1854,7 +1887,7 @@ "produces": [] }, { - "id": 134, + "id": 136, "name": "telerik", "parent": 99999999, "consumes": [ @@ -1867,7 +1900,7 @@ ] }, { - "id": 135, + "id": 137, "name": "trickest", "parent": 99999999, "consumes": [ @@ -1878,7 +1911,7 @@ ] }, { - "id": 136, + "id": 138, "name": "trufflehog", "parent": 99999999, "consumes": [ @@ -1891,7 +1924,7 @@ ] }, { - "id": 137, + "id": 139, "name": "url_manipulation", "parent": 99999999, "consumes": [ @@ -1902,7 +1935,7 @@ ] }, { - "id": 138, + "id": 140, "name": "urlscan", "parent": 99999999, "consumes": [ @@ -1914,7 +1947,7 @@ ] }, { - "id": 139, + "id": 141, "name": "vhost", "parent": 99999999, "consumes": [ @@ -1922,11 +1955,11 @@ ], "produces": [ 7, - 140 + 142 ] }, { - "id": 141, + "id": 143, "name": "viewdns", "parent": 99999999, "consumes": [ @@ -1937,7 +1970,7 @@ ] }, { - "id": 142, + "id": 144, "name": "virustotal", "parent": 99999999, "consumes": [ @@ -1948,7 +1981,7 @@ ] }, { - "id": 143, + "id": 145, "name": "wafw00f", "parent": 99999999, "consumes": [ @@ -1959,7 +1992,7 @@ ] }, { - "id": 144, + "id": 146, "name": "wappalyzer", "parent": 99999999, "consumes": [ @@ -1970,7 +2003,7 @@ ] }, { - "id": 145, + "id": 147, "name": "wayback", "parent": 99999999, "consumes": [ @@ -1982,20 +2015,20 @@ ] }, { - "id": 146, + "id": 148, "name": "web_report", "parent": 99999999, "consumes": [ 4, 16, 3, - 140, + 142, 5 ], "produces": [] }, { - "id": 147, + "id": 149, "name": "wpscan", "parent": 99999999, "consumes": [ @@ -2010,7 +2043,7 @@ ] }, { - "id": 148, + "id": 150, "name": "zoomeye", "parent": 99999999, "consumes": [ @@ -2020,4 +2053,4 @@ 7 ] } -] +] \ No newline at end of file diff --git a/docs/data/chord_graph/rels.json b/docs/data/chord_graph/rels.json index 43a646026a..f25e491f3e 100644 --- a/docs/data/chord_graph/rels.json +++ b/docs/data/chord_graph/rels.json @@ -585,1138 +585,1178 @@ "type": "produces" }, { - "source": 61, + "source": 7, + "target": 61, + "type": "produces" + }, + { + "source": 12, + "target": 61, + "type": "produces" + }, + { + "source": 55, + "target": 61, + "type": "produces" + }, + { + "source": 62, "target": 7, "type": "consumes" }, { "source": 46, - "target": 61, + "target": 62, "type": "produces" }, { "source": 55, - "target": 61, + "target": 62, "type": "produces" }, { "source": 19, - "target": 61, + "target": 62, "type": "produces" }, { - "source": 62, + "source": 63, "target": 43, "type": "consumes" }, { "source": 10, - "target": 62, + "target": 63, "type": "produces" }, { - "source": 63, - "target": 64, + "source": 64, + "target": 65, "type": "consumes" }, { - "source": 63, - "target": 65, + "source": 64, + "target": 66, "type": "consumes" }, { "source": 43, - "target": 63, + "target": 64, "type": "produces" }, { - "source": 65, - "target": 63, + "source": 66, + "target": 64, "type": "produces" }, { "source": 19, - "target": 63, + "target": 64, "type": "produces" }, { - "source": 66, + "source": 67, "target": 2, "type": "consumes" }, { "source": 16, - "target": 66, + "target": 67, "type": "produces" }, { "source": 5, - "target": 66, + "target": 67, "type": "produces" }, { - "source": 67, + "source": 68, "target": 7, "type": "consumes" }, { "source": 46, - "target": 67, + "target": 68, "type": "produces" }, { - "source": 68, + "source": 69, "target": 46, "type": "consumes" }, { - "source": 69, + "source": 70, "target": 2, "type": "consumes" }, { - "source": 69, - "target": 70, + "source": 70, + "target": 71, "type": "consumes" }, { "source": 19, - "target": 69, + "target": 70, "type": "produces" }, { - "source": 71, - "target": 69, + "source": 72, + "target": 70, "type": "produces" }, { - "source": 72, + "source": 73, "target": 10, "type": "consumes" }, { - "source": 70, - "target": 72, + "source": 71, + "target": 73, "type": "produces" }, { - "source": 73, + "source": 74, "target": 3, "type": "consumes" }, { "source": 19, - "target": 73, + "target": 74, "type": "produces" }, { - "source": 74, - "target": 75, + "source": 75, + "target": 76, "type": "consumes" }, { "source": 19, - "target": 74, + "target": 75, "type": "produces" }, { - "source": 76, + "source": 77, "target": 2, "type": "consumes" }, { - "source": 76, + "source": 77, "target": 19, "type": "consumes" }, { "source": 10, - "target": 76, + "target": 77, "type": "produces" }, { - "source": 77, + "source": 78, "target": 15, "type": "consumes" }, { - "source": 78, - "target": 77, + "source": 79, + "target": 78, "type": "produces" }, { - "source": 79, + "source": 80, "target": 7, "type": "consumes" }, { "source": 7, - "target": 79, + "target": 80, "type": "produces" }, { - "source": 80, + "source": 81, "target": 3, "type": "consumes" }, { "source": 5, - "target": 80, + "target": 81, "type": "produces" }, { - "source": 81, + "source": 82, "target": 3, "type": "consumes" }, { "source": 4, - "target": 81, + "target": 82, "type": "produces" }, { - "source": 82, + "source": 83, "target": 43, "type": "consumes" }, { "source": 10, - "target": 82, + "target": 83, "type": "produces" }, { - "source": 83, + "source": 84, "target": 7, "type": "consumes" }, { "source": 43, - "target": 83, + "target": 84, "type": "produces" }, { "source": 19, - "target": 83, + "target": 84, "type": "produces" }, { - "source": 84, - "target": 64, + "source": 85, + "target": 65, "type": "consumes" }, { - "source": 84, - "target": 65, + "source": 85, + "target": 66, "type": "consumes" }, { "source": 43, - "target": 84, + "target": 85, "type": "produces" }, { - "source": 85, + "source": 86, "target": 43, "type": "consumes" }, { "source": 10, - "target": 85, + "target": 86, "type": "produces" }, { - "source": 86, + "source": 87, "target": 2, "type": "consumes" }, { - "source": 86, - "target": 65, + "source": 87, + "target": 66, "type": "consumes" }, { - "source": 86, + "source": 87, "target": 16, "type": "consumes" }, { "source": 43, - "target": 86, + "target": 87, "type": "produces" }, { "source": 4, - "target": 86, + "target": 87, "type": "produces" }, { - "source": 65, - "target": 86, + "source": 66, + "target": 87, "type": "produces" }, { "source": 16, - "target": 86, + "target": 87, "type": "produces" }, { - "source": 87, + "source": 88, "target": 43, "type": "consumes" }, { - "source": 87, - "target": 64, + "source": 88, + "target": 65, "type": "consumes" }, { "source": 9, - "target": 87, + "target": 88, "type": "produces" }, { - "source": 88, - "target": 65, + "source": 89, + "target": 66, "type": "consumes" }, { - "source": 88, + "source": 89, "target": 3, "type": "consumes" }, { "source": 16, - "target": 88, + "target": 89, "type": "produces" }, { "source": 3, - "target": 88, + "target": 89, "type": "produces" }, { "source": 19, - "target": 88, + "target": 89, "type": "produces" }, { - "source": 89, - "target": 88, + "source": 90, + "target": 89, "type": "produces" }, { - "source": 90, + "source": 91, "target": 7, "type": "consumes" }, { "source": 7, - "target": 90, + "target": 91, "type": "produces" }, { - "source": 91, + "source": 92, "target": 2, "type": "consumes" }, { "source": 4, - "target": 91, + "target": 92, "type": "produces" }, { - "source": 92, + "source": 93, "target": 15, "type": "consumes" }, { - "source": 92, + "source": 93, "target": 3, "type": "consumes" }, { - "source": 92, + "source": 93, "target": 19, "type": "consumes" }, { "source": 2, - "target": 92, + "target": 93, "type": "produces" }, { "source": 3, - "target": 92, + "target": 93, "type": "produces" }, { - "source": 93, - "target": 71, + "source": 94, + "target": 72, "type": "consumes" }, { "source": 4, - "target": 93, + "target": 94, "type": "produces" }, { - "source": 94, + "source": 95, "target": 7, "type": "consumes" }, { "source": 7, - "target": 94, + "target": 95, "type": "produces" }, { "source": 46, - "target": 94, + "target": 95, "type": "produces" }, { "source": 19, - "target": 94, + "target": 95, "type": "produces" }, { - "source": 95, + "source": 96, "target": 3, "type": "consumes" }, { - "source": 75, - "target": 95, + "source": 76, + "target": 96, "type": "produces" }, { - "source": 96, + "source": 97, "target": 7, "type": "consumes" }, { - "source": 96, + "source": 97, "target": 12, "type": "consumes" }, { "source": 7, - "target": 96, + "target": 97, "type": "produces" }, { "source": 4, - "target": 96, + "target": 97, "type": "produces" }, { "source": 15, - "target": 96, + "target": 97, "type": "produces" }, { "source": 16, - "target": 96, + "target": 97, "type": "produces" }, { "source": 5, - "target": 96, + "target": 97, "type": "produces" }, { - "source": 97, + "source": 98, "target": 12, "type": "consumes" }, { - "source": 98, - "target": 97, + "source": 99, + "target": 98, "type": "produces" }, { - "source": 99, + "source": 100, "target": 12, "type": "consumes" }, { "source": 12, - "target": 99, + "target": 100, "type": "produces" }, { - "source": 100, + "source": 101, "target": 12, "type": "consumes" }, { - "source": 98, - "target": 100, + "source": 99, + "target": 101, "type": "produces" }, { - "source": 101, + "source": 102, "target": 10, "type": "consumes" }, { "source": 10, - "target": 101, + "target": 102, "type": "produces" }, { - "source": 102, + "source": 103, "target": 7, "type": "consumes" }, { "source": 7, - "target": 102, + "target": 103, "type": "produces" }, { - "source": 103, + "source": 104, "target": 7, "type": "consumes" }, { "source": 7, - "target": 103, + "target": 104, "type": "produces" }, { - "source": 104, + "source": 105, "target": 2, "type": "consumes" }, { "source": 4, - "target": 104, + "target": 105, "type": "produces" }, { - "source": 105, + "source": 106, + "target": 7, + "type": "consumes" + }, + { + "source": 106, "target": 2, "type": "consumes" }, { - "source": 105, + "source": 106, + "target": 12, + "type": "consumes" + }, + { + "source": 106, + "target": 15, + "type": "consumes" + }, + { + "source": 106, + "target": 79, + "type": "consumes" + }, + { + "source": 107, + "target": 2, + "type": "consumes" + }, + { + "source": 107, "target": 3, "type": "consumes" }, { "source": 7, - "target": 105, + "target": 107, "type": "produces" }, { "source": 4, - "target": 105, + "target": 107, "type": "produces" }, { - "source": 106, + "source": 108, "target": 3, "type": "consumes" }, { "source": 4, - "target": 106, + "target": 108, "type": "produces" }, { "source": 16, - "target": 106, + "target": 108, "type": "produces" }, { "source": 5, - "target": 106, + "target": 108, "type": "produces" }, { - "source": 107, + "source": 109, "target": 7, "type": "consumes" }, { - "source": 107, + "source": 109, "target": 19, "type": "consumes" }, { "source": 7, - "target": 107, + "target": 109, "type": "produces" }, { - "source": 108, + "source": 110, "target": 7, "type": "consumes" }, { "source": 7, - "target": 108, + "target": 110, "type": "produces" }, { - "source": 109, + "source": 111, "target": 2, "type": "consumes" }, { - "source": 109, - "target": 71, + "source": 111, + "target": 72, "type": "consumes" }, { "source": 4, - "target": 109, + "target": 111, "type": "produces" }, { - "source": 71, - "target": 109, + "source": 72, + "target": 111, "type": "produces" }, { - "source": 110, + "source": 112, "target": 2, "type": "consumes" }, { - "source": 110, - "target": 71, + "source": 112, + "target": 72, "type": "consumes" }, { "source": 4, - "target": 110, + "target": 112, "type": "produces" }, { - "source": 71, - "target": 110, + "source": 72, + "target": 112, "type": "produces" }, { - "source": 111, + "source": 113, "target": 2, "type": "consumes" }, { - "source": 111, - "target": 71, + "source": 113, + "target": 72, "type": "consumes" }, { - "source": 71, - "target": 111, + "source": 72, + "target": 113, "type": "produces" }, { - "source": 112, + "source": 114, "target": 7, "type": "consumes" }, { "source": 7, - "target": 112, + "target": 114, "type": "produces" }, { - "source": 113, + "source": 115, "target": 7, "type": "consumes" }, { "source": 46, - "target": 113, + "target": 115, "type": "produces" }, { - "source": 114, + "source": 116, "target": 7, "type": "consumes" }, { - "source": 114, + "source": 116, "target": 12, "type": "consumes" }, { - "source": 114, - "target": 115, + "source": 116, + "target": 117, "type": "consumes" }, { "source": 15, - "target": 114, + "target": 116, "type": "produces" }, { - "source": 116, - "target": 64, + "source": 118, + "target": 65, "type": "consumes" }, { - "source": 116, - "target": 65, + "source": 118, + "target": 66, "type": "consumes" }, { "source": 43, - "target": 116, + "target": 118, "type": "produces" }, { - "source": 117, + "source": 119, "target": 43, "type": "consumes" }, { "source": 10, - "target": 117, + "target": 119, "type": "produces" }, { - "source": 118, + "source": 120, "target": 7, "type": "consumes" }, { "source": 7, - "target": 118, + "target": 120, "type": "produces" }, { - "source": 119, + "source": 121, "target": 3, "type": "consumes" }, { "source": 19, - "target": 119, + "target": 121, "type": "produces" }, { - "source": 120, + "source": 122, "target": 2, "type": "consumes" }, { "source": 4, - "target": 120, + "target": 122, "type": "produces" }, { - "source": 121, + "source": 123, "target": 7, "type": "consumes" }, { "source": 7, - "target": 121, + "target": 123, "type": "produces" }, { - "source": 122, + "source": 124, "target": 7, "type": "consumes" }, { "source": 46, - "target": 122, + "target": 124, "type": "produces" }, { "source": 19, - "target": 122, + "target": 124, "type": "produces" }, { - "source": 123, + "source": 125, "target": 7, "type": "consumes" }, { "source": 7, - "target": 123, + "target": 125, "type": "produces" }, { - "source": 124, + "source": 126, "target": 7, "type": "consumes" }, { "source": 7, - "target": 124, + "target": 126, "type": "produces" }, { - "source": 125, + "source": 127, "target": 7, "type": "consumes" }, { "source": 46, - "target": 125, + "target": 127, "type": "produces" }, { - "source": 126, + "source": 128, "target": 3, "type": "consumes" }, { "source": 4, - "target": 126, + "target": 128, "type": "produces" }, { - "source": 127, + "source": 129, "target": 19, "type": "consumes" }, { - "source": 65, - "target": 127, + "source": 66, + "target": 129, "type": "produces" }, { - "source": 128, - "target": 129, + "source": 130, + "target": 131, "type": "consumes" }, { - "source": 128, + "source": 130, "target": 7, "type": "consumes" }, { - "source": 128, + "source": 130, "target": 22, "type": "consumes" }, { - "source": 128, + "source": 130, "target": 2, "type": "consumes" }, { - "source": 128, + "source": 130, "target": 12, "type": "consumes" }, { - "source": 128, - "target": 115, + "source": 130, + "target": 117, "type": "consumes" }, { - "source": 128, - "target": 65, + "source": 130, + "target": 66, "type": "consumes" }, { - "source": 128, + "source": 130, "target": 24, "type": "consumes" }, { - "source": 128, + "source": 130, "target": 3, "type": "consumes" }, { - "source": 128, + "source": 130, "target": 19, "type": "consumes" }, { - "source": 128, + "source": 130, "target": 49, "type": "consumes" }, { "source": 7, - "target": 128, + "target": 130, "type": "produces" }, { "source": 4, - "target": 128, + "target": 130, "type": "produces" }, { "source": 12, - "target": 128, + "target": 130, "type": "produces" }, { "source": 15, - "target": 128, + "target": 130, "type": "produces" }, { - "source": 64, - "target": 128, + "source": 65, + "target": 130, "type": "produces" }, { - "source": 130, + "source": 132, "target": 15, "type": "consumes" }, { "source": 7, - "target": 130, + "target": 132, "type": "produces" }, { "source": 46, - "target": 130, + "target": 132, "type": "produces" }, { - "source": 131, + "source": 133, "target": 7, "type": "consumes" }, { "source": 7, - "target": 131, + "target": 133, "type": "produces" }, { - "source": 132, + "source": 134, "target": 7, "type": "consumes" }, { "source": 7, - "target": 132, + "target": 134, "type": "produces" }, { - "source": 133, + "source": 135, "target": 7, "type": "consumes" }, { - "source": 133, + "source": 135, "target": 22, "type": "consumes" }, { - "source": 134, + "source": 136, "target": 2, "type": "consumes" }, { - "source": 134, + "source": 136, "target": 3, "type": "consumes" }, { "source": 4, - "target": 134, + "target": 136, "type": "produces" }, { "source": 5, - "target": 134, + "target": 136, "type": "produces" }, { - "source": 135, + "source": 137, "target": 7, "type": "consumes" }, { "source": 7, - "target": 135, + "target": 137, "type": "produces" }, { - "source": 136, + "source": 138, "target": 43, "type": "consumes" }, { - "source": 136, + "source": 138, "target": 10, "type": "consumes" }, { "source": 4, - "target": 136, + "target": 138, "type": "produces" }, { "source": 5, - "target": 136, + "target": 138, "type": "produces" }, { - "source": 137, + "source": 139, "target": 3, "type": "consumes" }, { "source": 4, - "target": 137, + "target": 139, "type": "produces" }, { - "source": 138, + "source": 140, "target": 7, "type": "consumes" }, { "source": 7, - "target": 138, + "target": 140, "type": "produces" }, { "source": 19, - "target": 138, + "target": 140, "type": "produces" }, { - "source": 139, + "source": 141, "target": 3, "type": "consumes" }, { "source": 7, - "target": 139, + "target": 141, "type": "produces" }, { - "source": 140, - "target": 139, + "source": 142, + "target": 141, "type": "produces" }, { - "source": 141, + "source": 143, "target": 7, "type": "consumes" }, { "source": 7, - "target": 141, + "target": 143, "type": "produces" }, { - "source": 142, + "source": 144, "target": 7, "type": "consumes" }, { "source": 7, - "target": 142, + "target": 144, "type": "produces" }, { - "source": 143, + "source": 145, "target": 3, "type": "consumes" }, { "source": 17, - "target": 143, + "target": 145, "type": "produces" }, { - "source": 144, + "source": 146, "target": 2, "type": "consumes" }, { "source": 16, - "target": 144, + "target": 146, "type": "produces" }, { - "source": 145, + "source": 147, "target": 7, "type": "consumes" }, { "source": 7, - "target": 145, + "target": 147, "type": "produces" }, { "source": 19, - "target": 145, + "target": 147, "type": "produces" }, { - "source": 146, + "source": 148, "target": 4, "type": "consumes" }, { - "source": 146, + "source": 148, "target": 16, "type": "consumes" }, { - "source": 146, + "source": 148, "target": 3, "type": "consumes" }, { - "source": 146, - "target": 140, + "source": 148, + "target": 142, "type": "consumes" }, { - "source": 146, + "source": 148, "target": 5, "type": "consumes" }, { - "source": 147, + "source": 149, "target": 2, "type": "consumes" }, { - "source": 147, + "source": 149, "target": 16, "type": "consumes" }, { "source": 4, - "target": 147, + "target": 149, "type": "produces" }, { "source": 16, - "target": 147, + "target": 149, "type": "produces" }, { "source": 19, - "target": 147, + "target": 149, "type": "produces" }, { "source": 5, - "target": 147, + "target": 149, "type": "produces" }, { - "source": 148, + "source": 150, "target": 7, "type": "consumes" }, { "source": 7, - "target": 148, + "target": 150, "type": "produces" } -] +] \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 3d6c5ef267..355d58e8b2 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,6 +34,8 @@ bbot --help Docker images are provided, along with helper script `bbot-docker.sh` to persist your scan data. +Scans are output to `~/.bbot/scans` (the usual place for BBOT scan data). + ```bash # bleeding edge (dev) docker run -it blacklanternsecurity/bbot --help @@ -46,6 +48,16 @@ git clone https://github.com/blacklanternsecurity/bbot && cd bbot ./bbot-docker.sh --help ``` +Note: If you need to pass in a custom preset, you can do so by mapping the preset into the container: + +```bash +# use the preset `my_preset.yml` from the current directory +docker run --rm -it \ + -v "$HOME/.bbot/scans:/root/.bbot/scans" \ + -v "$PWD/my_preset.yml:/my_preset.yml" \ + blacklanternsecurity/bbot -p /my_preset.yml +``` + ## Example Commands Below are some examples of common scans. diff --git a/docs/modules/list_of_modules.md b/docs/modules/list_of_modules.md index 44d57a0fbe..f8eae6b501 100644 --- a/docs/modules/list_of_modules.md +++ b/docs/modules/list_of_modules.md @@ -120,21 +120,22 @@ | emails | output | No | Output any email addresses found belonging to the target domain | email-enum | EMAIL_ADDRESS | | @domwhewell-sage | 2023-12-23 | | http | output | No | Send every event to a custom URL via a web request | | * | | @TheTechromancer | 2022-04-13 | | json | output | No | Output to Newline-Delimited JSON (NDJSON) | | * | | @TheTechromancer | 2022-04-07 | -| mysql | output | No | Output scan data to a MySQL database | | * | | | | +| mysql | output | No | Output scan data to a MySQL database | | * | | @TheTechromancer | 2024-11-13 | | neo4j | output | No | Output to Neo4j | | * | | @TheTechromancer | 2022-04-07 | -| postgres | output | No | Output scan data to a SQLite database | | * | | | | +| nmap_xml | output | No | Output to Nmap XML | | DNS_NAME, HTTP_RESPONSE, IP_ADDRESS, OPEN_TCP_PORT, PROTOCOL | | @TheTechromancer | 2024-11-16 | +| postgres | output | No | Output scan data to a SQLite database | | * | | @TheTechromancer | 2024-11-08 | | python | output | No | Output via Python API | | * | | @TheTechromancer | 2022-09-13 | | slack | output | No | Message a Slack channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 | | splunk | output | No | Send every event to a splunk instance through HTTP Event Collector | | * | | @w0Tx | 2024-02-17 | -| sqlite | output | No | Output scan data to a SQLite database | | * | | | | -| stdout | output | No | Output to text | | * | | | | +| sqlite | output | No | Output scan data to a SQLite database | | * | | @TheTechromancer | 2024-11-07 | +| stdout | output | No | Output to text | | * | | @TheTechromancer | 2024-04-03 | | subdomains | output | No | Output only resolved, in-scope subdomains | subdomain-enum | DNS_NAME, DNS_NAME_UNRESOLVED | | @TheTechromancer | 2023-07-31 | | teams | output | No | Message a Teams channel when certain events are encountered | | * | | @TheTechromancer | 2023-08-14 | -| txt | output | No | Output to text | | * | | | | +| txt | output | No | Output to text | | * | | @TheTechromancer | 2024-04-03 | | web_report | output | No | Create a markdown report with web assets | | FINDING, TECHNOLOGY, URL, VHOST, VULNERABILITY | | @liquidsec | 2023-02-08 | | websocket | output | No | Output to websockets | | * | | @TheTechromancer | 2022-04-15 | -| cloudcheck | internal | No | Tag events by cloud provider, identify cloud resources like storage buckets | | * | | | | -| dnsresolve | internal | No | | | * | | | | +| cloudcheck | internal | No | Tag events by cloud provider, identify cloud resources like storage buckets | | * | | @TheTechromancer | 2024-07-07 | +| dnsresolve | internal | No | Perform DNS resolution | | * | DNS_NAME, IP_ADDRESS, RAW_DNS_RECORD | @TheTechromancer | 2022-04-08 | | aggregate | internal | No | Summarize statistics at the end of a scan | passive, safe | | | @TheTechromancer | 2022-07-25 | | excavate | internal | No | Passively extract juicy tidbits from scan data | passive | HTTP_RESPONSE, RAW_TEXT | URL_UNVERIFIED, WEB_PARAMETER | @liquidsec | 2022-06-27 | | speculate | internal | No | Derive certain event types from others by common sense | passive | AZURE_TENANT, DNS_NAME, DNS_NAME_UNRESOLVED, HTTP_RESPONSE, IP_ADDRESS, IP_RANGE, SOCIAL, STORAGE_BUCKET, URL, URL_UNVERIFIED, USERNAME | DNS_NAME, FINDING, IP_ADDRESS, OPEN_TCP_PORT, ORG_STUB | @liquidsec | 2022-05-03 | diff --git a/docs/modules/nuclei.md b/docs/modules/nuclei.md index 43231b5c7a..2849cf22f5 100644 --- a/docs/modules/nuclei.md +++ b/docs/modules/nuclei.md @@ -51,7 +51,7 @@ The Nuclei module has many configuration options: | modules.nuclei.silent | bool | Don't display nuclei's banner or status messages | False | | modules.nuclei.tags | str | execute a subset of templates that contain the provided tags | | | modules.nuclei.templates | str | template or template directory paths to include in the scan | | -| modules.nuclei.version | str | nuclei version | 3.3.6 | +| modules.nuclei.version | str | nuclei version | 3.3.7 | Most of these you probably will **NOT** want to change. In particular, we advise against changing the version of Nuclei, as it's possible the latest version won't work right with BBOT. diff --git a/docs/release_history.md b/docs/release_history.md index 211cfa8d7e..cf1f140688 100644 --- a/docs/release_history.md +++ b/docs/release_history.md @@ -1,47 +1,51 @@ +### 2.2.0 - Nov 18, 2024 +- [https://github.com/blacklanternsecurity/bbot/pull/1919](https://github.com/blacklanternsecurity/bbot/pull/1919) + ### 2.1.2 - Nov 1, 2024 -- https://github.com/blacklanternsecurity/bbot/pull/1909 +- [https://github.com/blacklanternsecurity/bbot/pull/1909](https://github.com/blacklanternsecurity/bbot/pull/1909) ### 2.1.1 - Oct 31, 2024 -- https://github.com/blacklanternsecurity/bbot/pull/1885 +- [https://github.com/blacklanternsecurity/bbot/pull/1885](https://github.com/blacklanternsecurity/bbot/pull/1885) ### 2.1.0 - Oct 18, 2024 -- https://github.com/blacklanternsecurity/bbot/pull/1724 +- [https://github.com/blacklanternsecurity/bbot/pull/1724](https://github.com/blacklanternsecurity/bbot/pull/1724) ### 2.0.1 - Aug 29, 2024 -- https://github.com/blacklanternsecurity/bbot/pull/1650 +- [https://github.com/blacklanternsecurity/bbot/pull/1650](https://github.com/blacklanternsecurity/bbot/pull/1650) ### 2.0.0 - Aug 9, 2024 -- https://github.com/blacklanternsecurity/bbot/pull/1424 +- [https://github.com/blacklanternsecurity/bbot/pull/1424](https://github.com/blacklanternsecurity/bbot/pull/1424) +- [https://github.com/blacklanternsecurity/bbot/pull/1235](https://github.com/blacklanternsecurity/bbot/pull/1235) ### 1.1.8 - May 29, 2024 -- https://github.com/blacklanternsecurity/bbot/pull/1382 +- [https://github.com/blacklanternsecurity/bbot/pull/1382](https://github.com/blacklanternsecurity/bbot/pull/1382) ### 1.1.7 - May 15, 2024 -- https://github.com/blacklanternsecurity/bbot/pull/1119 +- [https://github.com/blacklanternsecurity/bbot/pull/1119](https://github.com/blacklanternsecurity/bbot/pull/1119) ### 1.1.6 - Feb 21, 2024 -- https://github.com/blacklanternsecurity/bbot/pull/1002 +- [https://github.com/blacklanternsecurity/bbot/pull/1002](https://github.com/blacklanternsecurity/bbot/pull/1002) ### 1.1.5 - Jan 15, 2024 -- https://github.com/blacklanternsecurity/bbot/pull/996 +- [https://github.com/blacklanternsecurity/bbot/pull/996](https://github.com/blacklanternsecurity/bbot/pull/996) ### 1.1.4 - Jan 11, 2024 -- https://github.com/blacklanternsecurity/bbot/pull/837 +- [https://github.com/blacklanternsecurity/bbot/pull/837](https://github.com/blacklanternsecurity/bbot/pull/837) ### 1.1.3 - Nov 4, 2023 -- https://github.com/blacklanternsecurity/bbot/pull/823 +- [https://github.com/blacklanternsecurity/bbot/pull/823](https://github.com/blacklanternsecurity/bbot/pull/823) ### 1.1.2 - Nov 3, 2023 -- https://github.com/blacklanternsecurity/bbot/pull/777 +- [https://github.com/blacklanternsecurity/bbot/pull/777](https://github.com/blacklanternsecurity/bbot/pull/777) ### 1.1.1 - Oct 11, 2023 -- https://github.com/blacklanternsecurity/bbot/pull/668 +- [https://github.com/blacklanternsecurity/bbot/pull/668](https://github.com/blacklanternsecurity/bbot/pull/668) ### 1.1.0 - Aug 4, 2023 -- https://github.com/blacklanternsecurity/bbot/pull/598 +- [https://github.com/blacklanternsecurity/bbot/pull/598](https://github.com/blacklanternsecurity/bbot/pull/598) ### 1.0.5 - Mar 10, 2023 -- https://github.com/blacklanternsecurity/bbot/pull/352 +- [https://github.com/blacklanternsecurity/bbot/pull/352](https://github.com/blacklanternsecurity/bbot/pull/352) ### 1.0.5 - Mar 10, 2023 -- https://github.com/blacklanternsecurity/bbot/pull/352 +- [https://github.com/blacklanternsecurity/bbot/pull/352](https://github.com/blacklanternsecurity/bbot/pull/352) diff --git a/docs/scanning/advanced.md b/docs/scanning/advanced.md index b990598a1a..88f75da5c1 100644 --- a/docs/scanning/advanced.md +++ b/docs/scanning/advanced.md @@ -39,8 +39,8 @@ usage: bbot [-h] [-t TARGET [TARGET ...]] [-w WHITELIST [WHITELIST ...]] [-f FLAG [FLAG ...]] [-lf] [-rf FLAG [FLAG ...]] [-ef FLAG [FLAG ...]] [--allow-deadly] [-n SCAN_NAME] [-v] [-d] [-s] [--force] [-y] [--fast-mode] [--dry-run] - [--current-preset] [--current-preset-full] [-o DIR] - [-om MODULE [MODULE ...]] [--json] [--brief] + [--current-preset] [--current-preset-full] + [-om MODULE [MODULE ...]] [-lo] [-o DIR] [--json] [--brief] [--event-types EVENT_TYPES [EVENT_TYPES ...]] [--no-deps | --force-deps | --retry-deps | --ignore-failed-deps | --install-all-deps] [--version] [--proxy HTTP_PROXY] @@ -100,10 +100,12 @@ Scan: Show the current preset in its full form, including defaults Output: + -om MODULE [MODULE ...], --output-modules MODULE [MODULE ...] + Output module(s). Choices: asset_inventory,csv,discord,emails,http,json,mysql,neo4j,nmap_xml,postgres,python,slack,splunk,sqlite,stdout,subdomains,teams,txt,web_report,websocket + -lo, --list-output-modules + List available output modules -o DIR, --output-dir DIR Directory to output scan results - -om MODULE [MODULE ...], --output-modules MODULE [MODULE ...] - Output module(s). Choices: asset_inventory,csv,discord,emails,http,json,mysql,neo4j,postgres,python,slack,splunk,sqlite,stdout,subdomains,teams,txt,web_report,websocket --json, -j Output scan data in JSON format --brief, -br Output only the data itself --event-types EVENT_TYPES [EVENT_TYPES ...] @@ -149,6 +151,9 @@ EXAMPLES List modules: bbot -l + List output modules: + bbot -lo + List presets: bbot -lp diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md index 4ce1a92212..5808b149c5 100644 --- a/docs/scanning/configuration.md +++ b/docs/scanning/configuration.md @@ -325,7 +325,7 @@ Many modules accept their own configuration options. These options have the abil | modules.nuclei.silent | bool | Don't display nuclei's banner or status messages | False | | modules.nuclei.tags | str | execute a subset of templates that contain the provided tags | | | modules.nuclei.templates | str | template or template directory paths to include in the scan | | -| modules.nuclei.version | str | nuclei version | 3.3.6 | +| modules.nuclei.version | str | nuclei version | 3.3.7 | | modules.oauth.try_all | bool | Check for OAUTH/IODC on every subdomain and URL. | False | | modules.paramminer_cookies.recycle_words | bool | Attempt to use words found during the scan on all other endpoints | False | | modules.paramminer_cookies.skip_boring_words | bool | Remove commonly uninteresting words from the wordlist | True | @@ -436,7 +436,7 @@ Many modules accept their own configuration options. These options have the abil | modules.trufflehog.config | str | File path or URL to YAML trufflehog config | | | modules.trufflehog.deleted_forks | bool | Scan for deleted github forks. WARNING: This is SLOW. For a smaller repository, this process can take 20 minutes. For a larger repository, it could take hours. | False | | modules.trufflehog.only_verified | bool | Only report credentials that have been verified | True | -| modules.trufflehog.version | str | trufflehog version | 3.84.1 | +| modules.trufflehog.version | str | trufflehog version | 3.86.1 | | modules.urlscan.urls | bool | Emit URLs in addition to DNS_NAMEs | False | | modules.virustotal.api_key | str | VirusTotal API Key | | | modules.wayback.garbage_threshold | int | Dedupe similar urls if they are in a group of this size or higher (lower values == less garbage data) | 10 | diff --git a/docs/scanning/events.md b/docs/scanning/events.md index e837417985..f512ac770d 100644 --- a/docs/scanning/events.md +++ b/docs/scanning/events.md @@ -104,41 +104,41 @@ Below is a full list of event types along with which modules produce/consume the ## List of Event Types -| Event Type | # Consuming Modules | # Producing Modules | Consuming Modules | Producing Modules | -|---------------------|-----------------------|-----------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| * | 18 | 0 | affiliates, cloudcheck, csv, discord, dnsresolve, http, json, mysql, neo4j, postgres, python, slack, splunk, sqlite, stdout, teams, txt, websocket | | -| ASN | 0 | 1 | | asn | -| AZURE_TENANT | 1 | 0 | speculate | | -| CODE_REPOSITORY | 6 | 6 | docker_pull, git_clone, github_workflows, google_playstore, postman_download, trufflehog | code_repository, dockerhub, github_codesearch, github_org, gitlab, postman | -| DNS_NAME | 60 | 43 | anubisdb, asset_inventory, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bufferoverrun, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crt, dehashed, digitorus, dnsbimi, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dnstlsrpt, emailformat, fullhunt, github_codesearch, hackertarget, hunterio, internetdb, leakix, myssl, oauth, otx, passivetotal, pgp, portscan, rapiddns, securitytrails, securitytxt, shodan_dns, sitedossier, skymem, speculate, subdomaincenter, subdomainradar, subdomains, trickest, urlscan, viewdns, virustotal, wayback, zoomeye | anubisdb, azure_tenant, bevigil, binaryedge, bufferoverrun, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, fullhunt, hackertarget, hunterio, internetdb, leakix, myssl, ntlm, oauth, otx, passivetotal, rapiddns, securitytrails, shodan_dns, sitedossier, speculate, sslcert, subdomaincenter, subdomainradar, trickest, urlscan, vhost, viewdns, virustotal, wayback, zoomeye | -| DNS_NAME_UNRESOLVED | 3 | 0 | baddns, speculate, subdomains | | -| EMAIL_ADDRESS | 1 | 10 | emails | credshed, dehashed, dnscaa, dnstlsrpt, emailformat, hunterio, pgp, securitytxt, skymem, sslcert | -| FILESYSTEM | 3 | 7 | extractous, jadx, trufflehog | apkpure, docker_pull, filedownload, git_clone, github_workflows, jadx, postman_download | -| FINDING | 2 | 29 | asset_inventory, web_report | ajaxpro, baddns, baddns_direct, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, git, gitlab, host_header, hunt, internetdb, newsletters, ntlm, nuclei, paramminer_cookies, paramminer_getparams, secretsdb, smuggler, speculate, telerik, trufflehog, url_manipulation, wpscan | -| GEOLOCATION | 0 | 2 | | ip2location, ipstack | -| HASHED_PASSWORD | 0 | 2 | | credshed, dehashed | -| HTTP_RESPONSE | 19 | 1 | ajaxpro, asset_inventory, badsecrets, dastardly, dotnetnuke, excavate, filedownload, gitlab, host_header, newsletters, ntlm, paramminer_cookies, paramminer_getparams, paramminer_headers, secretsdb, speculate, telerik, wappalyzer, wpscan | httpx | -| IP_ADDRESS | 8 | 3 | asn, asset_inventory, internetdb, ip2location, ipneighbor, ipstack, portscan, speculate | asset_inventory, ipneighbor, speculate | -| IP_RANGE | 2 | 0 | portscan, speculate | | -| MOBILE_APP | 1 | 1 | apkpure | google_playstore | -| OPEN_TCP_PORT | 4 | 4 | asset_inventory, fingerprintx, httpx, sslcert | asset_inventory, internetdb, portscan, speculate | -| ORG_STUB | 4 | 1 | dockerhub, github_org, google_playstore, postman | speculate | -| PASSWORD | 0 | 2 | | credshed, dehashed | -| PROTOCOL | 0 | 1 | | fingerprintx | -| RAW_DNS_RECORD | 0 | 2 | | dnsbimi, dnstlsrpt | -| RAW_TEXT | 1 | 1 | excavate | extractous | -| SOCIAL | 6 | 3 | dockerhub, github_org, gitlab, gowitness, postman, speculate | dockerhub, gitlab, social | -| STORAGE_BUCKET | 8 | 5 | baddns_direct, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, speculate | bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google | -| TECHNOLOGY | 4 | 8 | asset_inventory, gitlab, web_report, wpscan | badsecrets, dotnetnuke, gitlab, gowitness, internetdb, nuclei, wappalyzer, wpscan | -| URL | 20 | 2 | ajaxpro, asset_inventory, baddns_direct, bypass403, ffuf, generic_ssrf, git, gowitness, httpx, iis_shortnames, ntlm, nuclei, robots, smuggler, speculate, telerik, url_manipulation, vhost, wafw00f, web_report | gowitness, httpx | -| URL_HINT | 1 | 1 | ffuf_shortnames | iis_shortnames | -| URL_UNVERIFIED | 6 | 18 | code_repository, filedownload, httpx, oauth, social, speculate | azure_realm, bevigil, bucket_file_enum, dnsbimi, dnscaa, dnstlsrpt, dockerhub, excavate, ffuf, ffuf_shortnames, github_codesearch, gowitness, hunterio, robots, securitytxt, urlscan, wayback, wpscan | -| USERNAME | 1 | 2 | speculate | credshed, dehashed | -| VHOST | 1 | 1 | web_report | vhost | -| VULNERABILITY | 2 | 13 | asset_inventory, web_report | ajaxpro, baddns, baddns_direct, baddns_zone, badsecrets, dastardly, dotnetnuke, generic_ssrf, internetdb, nuclei, telerik, trufflehog, wpscan | -| WAF | 1 | 1 | asset_inventory | wafw00f | -| WEBSCREENSHOT | 0 | 1 | | gowitness | -| WEB_PARAMETER | 4 | 4 | hunt, paramminer_cookies, paramminer_getparams, paramminer_headers | excavate, paramminer_cookies, paramminer_getparams, paramminer_headers | +| Event Type | # Consuming Modules | # Producing Modules | Consuming Modules | Producing Modules | +|---------------------|-----------------------|-----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| * | 18 | 0 | affiliates, cloudcheck, csv, discord, dnsresolve, http, json, mysql, neo4j, postgres, python, slack, splunk, sqlite, stdout, teams, txt, websocket | | +| ASN | 0 | 1 | | asn | +| AZURE_TENANT | 1 | 0 | speculate | | +| CODE_REPOSITORY | 6 | 6 | docker_pull, git_clone, github_workflows, google_playstore, postman_download, trufflehog | code_repository, dockerhub, github_codesearch, github_org, gitlab, postman | +| DNS_NAME | 61 | 44 | anubisdb, asset_inventory, azure_realm, azure_tenant, baddns, baddns_zone, bevigil, binaryedge, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bufferoverrun, builtwith, c99, censys, certspotter, chaos, columbus, credshed, crt, dehashed, digitorus, dnsbimi, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dnstlsrpt, emailformat, fullhunt, github_codesearch, hackertarget, hunterio, internetdb, leakix, myssl, nmap_xml, oauth, otx, passivetotal, pgp, portscan, rapiddns, securitytrails, securitytxt, shodan_dns, sitedossier, skymem, speculate, subdomaincenter, subdomainradar, subdomains, trickest, urlscan, viewdns, virustotal, wayback, zoomeye | anubisdb, azure_tenant, bevigil, binaryedge, bufferoverrun, builtwith, c99, censys, certspotter, chaos, columbus, crt, digitorus, dnsbrute, dnsbrute_mutations, dnscaa, dnscommonsrv, dnsdumpster, dnsresolve, fullhunt, hackertarget, hunterio, internetdb, leakix, myssl, ntlm, oauth, otx, passivetotal, rapiddns, securitytrails, shodan_dns, sitedossier, speculate, sslcert, subdomaincenter, subdomainradar, trickest, urlscan, vhost, viewdns, virustotal, wayback, zoomeye | +| DNS_NAME_UNRESOLVED | 3 | 0 | baddns, speculate, subdomains | | +| EMAIL_ADDRESS | 1 | 10 | emails | credshed, dehashed, dnscaa, dnstlsrpt, emailformat, hunterio, pgp, securitytxt, skymem, sslcert | +| FILESYSTEM | 3 | 7 | extractous, jadx, trufflehog | apkpure, docker_pull, filedownload, git_clone, github_workflows, jadx, postman_download | +| FINDING | 2 | 29 | asset_inventory, web_report | ajaxpro, baddns, baddns_direct, baddns_zone, badsecrets, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google, bypass403, dastardly, git, gitlab, host_header, hunt, internetdb, newsletters, ntlm, nuclei, paramminer_cookies, paramminer_getparams, secretsdb, smuggler, speculate, telerik, trufflehog, url_manipulation, wpscan | +| GEOLOCATION | 0 | 2 | | ip2location, ipstack | +| HASHED_PASSWORD | 0 | 2 | | credshed, dehashed | +| HTTP_RESPONSE | 20 | 1 | ajaxpro, asset_inventory, badsecrets, dastardly, dotnetnuke, excavate, filedownload, gitlab, host_header, newsletters, nmap_xml, ntlm, paramminer_cookies, paramminer_getparams, paramminer_headers, secretsdb, speculate, telerik, wappalyzer, wpscan | httpx | +| IP_ADDRESS | 9 | 4 | asn, asset_inventory, internetdb, ip2location, ipneighbor, ipstack, nmap_xml, portscan, speculate | asset_inventory, dnsresolve, ipneighbor, speculate | +| IP_RANGE | 2 | 0 | portscan, speculate | | +| MOBILE_APP | 1 | 1 | apkpure | google_playstore | +| OPEN_TCP_PORT | 5 | 4 | asset_inventory, fingerprintx, httpx, nmap_xml, sslcert | asset_inventory, internetdb, portscan, speculate | +| ORG_STUB | 4 | 1 | dockerhub, github_org, google_playstore, postman | speculate | +| PASSWORD | 0 | 2 | | credshed, dehashed | +| PROTOCOL | 1 | 1 | nmap_xml | fingerprintx | +| RAW_DNS_RECORD | 0 | 3 | | dnsbimi, dnsresolve, dnstlsrpt | +| RAW_TEXT | 1 | 1 | excavate | extractous | +| SOCIAL | 6 | 3 | dockerhub, github_org, gitlab, gowitness, postman, speculate | dockerhub, gitlab, social | +| STORAGE_BUCKET | 8 | 5 | baddns_direct, bucket_amazon, bucket_azure, bucket_digitalocean, bucket_file_enum, bucket_firebase, bucket_google, speculate | bucket_amazon, bucket_azure, bucket_digitalocean, bucket_firebase, bucket_google | +| TECHNOLOGY | 4 | 8 | asset_inventory, gitlab, web_report, wpscan | badsecrets, dotnetnuke, gitlab, gowitness, internetdb, nuclei, wappalyzer, wpscan | +| URL | 20 | 2 | ajaxpro, asset_inventory, baddns_direct, bypass403, ffuf, generic_ssrf, git, gowitness, httpx, iis_shortnames, ntlm, nuclei, robots, smuggler, speculate, telerik, url_manipulation, vhost, wafw00f, web_report | gowitness, httpx | +| URL_HINT | 1 | 1 | ffuf_shortnames | iis_shortnames | +| URL_UNVERIFIED | 6 | 18 | code_repository, filedownload, httpx, oauth, social, speculate | azure_realm, bevigil, bucket_file_enum, dnsbimi, dnscaa, dnstlsrpt, dockerhub, excavate, ffuf, ffuf_shortnames, github_codesearch, gowitness, hunterio, robots, securitytxt, urlscan, wayback, wpscan | +| USERNAME | 1 | 2 | speculate | credshed, dehashed | +| VHOST | 1 | 1 | web_report | vhost | +| VULNERABILITY | 2 | 13 | asset_inventory, web_report | ajaxpro, baddns, baddns_direct, baddns_zone, badsecrets, dastardly, dotnetnuke, generic_ssrf, internetdb, nuclei, telerik, trufflehog, wpscan | +| WAF | 1 | 1 | asset_inventory | wafw00f | +| WEBSCREENSHOT | 0 | 1 | | gowitness | +| WEB_PARAMETER | 4 | 4 | hunt, paramminer_cookies, paramminer_getparams, paramminer_headers | excavate, paramminer_cookies, paramminer_getparams, paramminer_headers | ## Findings Vs. Vulnerabilities diff --git a/docs/scanning/presets_list.md b/docs/scanning/presets_list.md index 416e163c52..11407fead5 100644 --- a/docs/scanning/presets_list.md +++ b/docs/scanning/presets_list.md @@ -8,13 +8,13 @@ Run all baddns modules and submodules. ??? note "`baddns-thorough.yml`" ```yaml title="~/.bbot/presets/baddns-thorough.yml" description: Run all baddns modules and submodules. - - + + modules: - baddns - baddns_zone - baddns_direct - + config: modules: baddns: @@ -32,10 +32,10 @@ Enumerate cloud resources such as storage buckets, etc. ??? note "`cloud-enum.yml`" ```yaml title="~/.bbot/presets/cloud-enum.yml" description: Enumerate cloud resources such as storage buckets, etc. - + include: - subdomain-enum - + flags: - cloud-enum ``` @@ -51,7 +51,7 @@ Enumerate Git repositories, Docker images, etc. ??? note "`code-enum.yml`" ```yaml title="~/.bbot/presets/code-enum.yml" description: Enumerate Git repositories, Docker images, etc. - + flags: - code-enum ``` @@ -67,17 +67,17 @@ Recursive web directory brute-force (aggressive) ??? note "`dirbust-heavy.yml`" ```yaml title="~/.bbot/presets/web/dirbust-heavy.yml" description: Recursive web directory brute-force (aggressive) - + include: - spider - + flags: - iis-shortnames - + modules: - ffuf - wayback - + config: modules: iis_shortnames: @@ -118,13 +118,13 @@ Basic web directory brute-force (surface-level directories only) ??? note "`dirbust-light.yml`" ```yaml title="~/.bbot/presets/web/dirbust-light.yml" description: Basic web directory brute-force (surface-level directories only) - + include: - iis-shortnames - + modules: - ffuf - + config: modules: ffuf: @@ -143,11 +143,11 @@ Comprehensive scan for all IIS/.NET specific modules and module settings ??? note "`dotnet-audit.yml`" ```yaml title="~/.bbot/presets/web/dotnet-audit.yml" description: Comprehensive scan for all IIS/.NET specific modules and module settings - - + + include: - iis-shortnames - + modules: - httpx - badsecrets @@ -156,14 +156,13 @@ Comprehensive scan for all IIS/.NET specific modules and module settings - telerik - ajaxpro - dotnetnuke - + config: modules: ffuf: extensions: asp,aspx,ashx,asmx,ascx telerik: exploit_RAU_crypto: True - ``` Category: web @@ -177,10 +176,10 @@ Enumerate email addresses from APIs, web crawling, etc. ??? note "`email-enum.yml`" ```yaml title="~/.bbot/presets/email-enum.yml" description: Enumerate email addresses from APIs, web crawling, etc. - + flags: - email-enum - + output_modules: - emails ``` @@ -196,10 +195,10 @@ Scan only the provided targets as fast as possible - no extra discovery ??? note "`fast.yml`" ```yaml title="~/.bbot/presets/fast.yml" description: Scan only the provided targets as fast as possible - no extra discovery - + exclude_modules: - excavate - + config: # only scan the exact targets specified scope: @@ -224,10 +223,10 @@ Recursively enumerate IIS shortnames ??? note "`iis-shortnames.yml`" ```yaml title="~/.bbot/presets/web/iis-shortnames.yml" description: Recursively enumerate IIS shortnames - + flags: - iis-shortnames - + config: modules: iis_shortnames: @@ -246,7 +245,7 @@ Everything everywhere all at once ??? note "`kitchen-sink.yml`" ```yaml title="~/.bbot/presets/kitchen-sink.yml" description: Everything everywhere all at once - + include: - subdomain-enum - cloud-enum @@ -258,13 +257,11 @@ Everything everywhere all at once - dirbust-light - web-screenshots - baddns-thorough - + config: modules: baddns: enable_references: True - - ``` @@ -278,13 +275,13 @@ Discover new web parameters via brute-force ??? note "`paramminer.yml`" ```yaml title="~/.bbot/presets/web/paramminer.yml" description: Discover new web parameters via brute-force - + flags: - web-paramminer - + modules: - httpx - + config: web: spider_distance: 1 @@ -302,14 +299,14 @@ Recursive web spider ??? note "`spider.yml`" ```yaml title="~/.bbot/presets/spider.yml" description: Recursive web spider - + modules: - httpx - + blacklist: # Prevent spider from invalidating sessions by logging out - "RE:/.*(sign|log)[_-]?out" - + config: web: # how many links to follow in a row @@ -331,15 +328,15 @@ Enumerate subdomains via APIs, brute-force ??? note "`subdomain-enum.yml`" ```yaml title="~/.bbot/presets/subdomain-enum.yml" description: Enumerate subdomains via APIs, brute-force - + flags: # enable every module with the subdomain-enum flag - subdomain-enum - + output_modules: # output unique subdomains to TXT file - subdomains - + config: dns: threads: 25 @@ -365,10 +362,10 @@ Quick web scan ??? note "`web-basic.yml`" ```yaml title="~/.bbot/presets/web-basic.yml" description: Quick web scan - + include: - iis-shortnames - + flags: - web-basic ``` @@ -384,10 +381,10 @@ Take screenshots of webpages ??? note "`web-screenshots.yml`" ```yaml title="~/.bbot/presets/web-screenshots.yml" description: Take screenshots of webpages - + flags: - web-screenshots - + config: modules: gowitness: @@ -410,11 +407,11 @@ Aggressive web scan ??? note "`web-thorough.yml`" ```yaml title="~/.bbot/presets/web-thorough.yml" description: Aggressive web scan - + include: # include the web-basic preset - web-basic - + flags: - web-thorough ``` diff --git a/poetry.lock b/poetry.lock index 655d449669..2e54a2edf2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "annotated-types" @@ -371,17 +371,17 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "cloudcheck" -version = "6.0.0.661" +version = "6.0.0.686" description = "Check whether an IP address belongs to a cloud provider" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "cloudcheck-6.0.0.661-py3-none-any.whl", hash = "sha256:b8c45061d76eea14aa493e9dfd087e1aefccb1632c3bb8d49c77d273f721188c"}, - {file = "cloudcheck-6.0.0.661.tar.gz", hash = "sha256:98a7b88f4784fad91faa3d6ea5749c7fe215462dbad63c34df1afc671f915795"}, + {file = "cloudcheck-6.0.0.686-py3-none-any.whl", hash = "sha256:bd2494ffc5e3ee2c6ab6c13bf2842407da02a2ff27edd9788d73e13a276f1c39"}, + {file = "cloudcheck-6.0.0.686.tar.gz", hash = "sha256:70ef41a1e03dcc35093322d7bb1cd6636ea945bdbeceda15e9120013023ee5cb"}, ] [package.dependencies] -httpx = ">=0.26,<0.28" +httpx = ">=0.26,<0.29" pydantic = ">=2.4.2,<3.0.0" radixtarget = ">=2.0.0.32,<3.0.0.0" regex = ">=2024.4.16,<2025.0.0" @@ -602,13 +602,13 @@ test = ["pytest (>=6)"] [[package]] name = "fastapi" -version = "0.115.5" +version = "0.115.6" description = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" optional = false python-versions = ">=3.8" files = [ - {file = "fastapi-0.115.5-py3-none-any.whl", hash = "sha256:596b95adbe1474da47049e802f9a65ab2ffa9c2b07e7efee70eb8a66c9f2f796"}, - {file = "fastapi-0.115.5.tar.gz", hash = "sha256:0e7a4d0dc0d01c68df21887cce0945e72d3c48b9f4f79dfe7a7d53aa08fbb289"}, + {file = "fastapi-0.115.6-py3-none-any.whl", hash = "sha256:e9240b29e36fa8f4bb7290316988e90c381e5092e0cbe84e7818cc3713bcf305"}, + {file = "fastapi-0.115.6.tar.gz", hash = "sha256:9ec46f7addc14ea472958a96aae5b5de65f39721a46aaf5705c480d9a8b76654"}, ] [package.dependencies] @@ -1214,13 +1214,13 @@ pyyaml = ">=5.1" [[package]] name = "mkdocs-material" -version = "9.5.46" +version = "9.5.48" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.46-py3-none-any.whl", hash = "sha256:98f0a2039c62e551a68aad0791a8d41324ff90c03a6e6cea381a384b84908b83"}, - {file = "mkdocs_material-9.5.46.tar.gz", hash = "sha256:ae2043f4238e572f9a40e0b577f50400d6fc31e2fef8ea141800aebf3bd273d7"}, + {file = "mkdocs_material-9.5.48-py3-none-any.whl", hash = "sha256:b695c998f4b939ce748adbc0d3bff73fa886a670ece948cf27818fa115dc16f8"}, + {file = "mkdocs_material-9.5.48.tar.gz", hash = "sha256:a582531e8b34f4c7ed38c29d5c44763053832cf2a32f7409567e0c74749a47db"}, ] [package.dependencies] @@ -1449,6 +1449,90 @@ files = [ [package.extras] dev = ["black", "mypy", "pytest"] +[[package]] +name = "orjson" +version = "3.10.12" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.10.12-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ece01a7ec71d9940cc654c482907a6b65df27251255097629d0dea781f255c6d"}, + {file = "orjson-3.10.12-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c34ec9aebc04f11f4b978dd6caf697a2df2dd9b47d35aa4cc606cabcb9df69d7"}, + {file = "orjson-3.10.12-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fd6ec8658da3480939c79b9e9e27e0db31dffcd4ba69c334e98c9976ac29140e"}, + {file = "orjson-3.10.12-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f17e6baf4cf01534c9de8a16c0c611f3d94925d1701bf5f4aff17003677d8ced"}, + {file = "orjson-3.10.12-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6402ebb74a14ef96f94a868569f5dccf70d791de49feb73180eb3c6fda2ade56"}, + {file = "orjson-3.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0000758ae7c7853e0a4a6063f534c61656ebff644391e1f81698c1b2d2fc8cd2"}, + {file = "orjson-3.10.12-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:888442dcee99fd1e5bd37a4abb94930915ca6af4db50e23e746cdf4d1e63db13"}, + {file = "orjson-3.10.12-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c1f7a3ce79246aa0e92f5458d86c54f257fb5dfdc14a192651ba7ec2c00f8a05"}, + {file = "orjson-3.10.12-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:802a3935f45605c66fb4a586488a38af63cb37aaad1c1d94c982c40dcc452e85"}, + {file = "orjson-3.10.12-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:1da1ef0113a2be19bb6c557fb0ec2d79c92ebd2fed4cfb1b26bab93f021fb885"}, + {file = "orjson-3.10.12-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:7a3273e99f367f137d5b3fecb5e9f45bcdbfac2a8b2f32fbc72129bbd48789c2"}, + {file = "orjson-3.10.12-cp310-none-win32.whl", hash = "sha256:475661bf249fd7907d9b0a2a2421b4e684355a77ceef85b8352439a9163418c3"}, + {file = "orjson-3.10.12-cp310-none-win_amd64.whl", hash = "sha256:87251dc1fb2b9e5ab91ce65d8f4caf21910d99ba8fb24b49fd0c118b2362d509"}, + {file = "orjson-3.10.12-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a734c62efa42e7df94926d70fe7d37621c783dea9f707a98cdea796964d4cf74"}, + {file = "orjson-3.10.12-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:750f8b27259d3409eda8350c2919a58b0cfcd2054ddc1bd317a643afc646ef23"}, + {file = "orjson-3.10.12-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bb52c22bfffe2857e7aa13b4622afd0dd9d16ea7cc65fd2bf318d3223b1b6252"}, + {file = "orjson-3.10.12-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:440d9a337ac8c199ff8251e100c62e9488924c92852362cd27af0e67308c16ef"}, + {file = "orjson-3.10.12-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a9e15c06491c69997dfa067369baab3bf094ecb74be9912bdc4339972323f252"}, + {file = "orjson-3.10.12-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:362d204ad4b0b8724cf370d0cd917bb2dc913c394030da748a3bb632445ce7c4"}, + {file = "orjson-3.10.12-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2b57cbb4031153db37b41622eac67329c7810e5f480fda4cfd30542186f006ae"}, + {file = "orjson-3.10.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:165c89b53ef03ce0d7c59ca5c82fa65fe13ddf52eeb22e859e58c237d4e33b9b"}, + {file = "orjson-3.10.12-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:5dee91b8dfd54557c1a1596eb90bcd47dbcd26b0baaed919e6861f076583e9da"}, + {file = "orjson-3.10.12-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:77a4e1cfb72de6f905bdff061172adfb3caf7a4578ebf481d8f0530879476c07"}, + {file = "orjson-3.10.12-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:038d42c7bc0606443459b8fe2d1f121db474c49067d8d14c6a075bbea8bf14dd"}, + {file = "orjson-3.10.12-cp311-none-win32.whl", hash = "sha256:03b553c02ab39bed249bedd4abe37b2118324d1674e639b33fab3d1dafdf4d79"}, + {file = "orjson-3.10.12-cp311-none-win_amd64.whl", hash = "sha256:8b8713b9e46a45b2af6b96f559bfb13b1e02006f4242c156cbadef27800a55a8"}, + {file = "orjson-3.10.12-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:53206d72eb656ca5ac7d3a7141e83c5bbd3ac30d5eccfe019409177a57634b0d"}, + {file = "orjson-3.10.12-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac8010afc2150d417ebda810e8df08dd3f544e0dd2acab5370cfa6bcc0662f8f"}, + {file = "orjson-3.10.12-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ed459b46012ae950dd2e17150e838ab08215421487371fa79d0eced8d1461d70"}, + {file = "orjson-3.10.12-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dcb9673f108a93c1b52bfc51b0af422c2d08d4fc710ce9c839faad25020bb69"}, + {file = "orjson-3.10.12-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:22a51ae77680c5c4652ebc63a83d5255ac7d65582891d9424b566fb3b5375ee9"}, + {file = "orjson-3.10.12-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:910fdf2ac0637b9a77d1aad65f803bac414f0b06f720073438a7bd8906298192"}, + {file = "orjson-3.10.12-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:24ce85f7100160936bc2116c09d1a8492639418633119a2224114f67f63a4559"}, + {file = "orjson-3.10.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a76ba5fc8dd9c913640292df27bff80a685bed3a3c990d59aa6ce24c352f8fc"}, + {file = "orjson-3.10.12-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:ff70ef093895fd53f4055ca75f93f047e088d1430888ca1229393a7c0521100f"}, + {file = "orjson-3.10.12-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:f4244b7018b5753ecd10a6d324ec1f347da130c953a9c88432c7fbc8875d13be"}, + {file = "orjson-3.10.12-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:16135ccca03445f37921fa4b585cff9a58aa8d81ebcb27622e69bfadd220b32c"}, + {file = "orjson-3.10.12-cp312-none-win32.whl", hash = "sha256:2d879c81172d583e34153d524fcba5d4adafbab8349a7b9f16ae511c2cee8708"}, + {file = "orjson-3.10.12-cp312-none-win_amd64.whl", hash = "sha256:fc23f691fa0f5c140576b8c365bc942d577d861a9ee1142e4db468e4e17094fb"}, + {file = "orjson-3.10.12-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:47962841b2a8aa9a258b377f5188db31ba49af47d4003a32f55d6f8b19006543"}, + {file = "orjson-3.10.12-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6334730e2532e77b6054e87ca84f3072bee308a45a452ea0bffbbbc40a67e296"}, + {file = "orjson-3.10.12-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:accfe93f42713c899fdac2747e8d0d5c659592df2792888c6c5f829472e4f85e"}, + {file = "orjson-3.10.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a7974c490c014c48810d1dede6c754c3cc46598da758c25ca3b4001ac45b703f"}, + {file = "orjson-3.10.12-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:3f250ce7727b0b2682f834a3facff88e310f52f07a5dcfd852d99637d386e79e"}, + {file = "orjson-3.10.12-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:f31422ff9486ae484f10ffc51b5ab2a60359e92d0716fcce1b3593d7bb8a9af6"}, + {file = "orjson-3.10.12-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5f29c5d282bb2d577c2a6bbde88d8fdcc4919c593f806aac50133f01b733846e"}, + {file = "orjson-3.10.12-cp313-none-win32.whl", hash = "sha256:f45653775f38f63dc0e6cd4f14323984c3149c05d6007b58cb154dd080ddc0dc"}, + {file = "orjson-3.10.12-cp313-none-win_amd64.whl", hash = "sha256:229994d0c376d5bdc91d92b3c9e6be2f1fbabd4cc1b59daae1443a46ee5e9825"}, + {file = "orjson-3.10.12-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:7d69af5b54617a5fac5c8e5ed0859eb798e2ce8913262eb522590239db6c6763"}, + {file = "orjson-3.10.12-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ed119ea7d2953365724a7059231a44830eb6bbb0cfead33fcbc562f5fd8f935"}, + {file = "orjson-3.10.12-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9c5fc1238ef197e7cad5c91415f524aaa51e004be5a9b35a1b8a84ade196f73f"}, + {file = "orjson-3.10.12-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:43509843990439b05f848539d6f6198d4ac86ff01dd024b2f9a795c0daeeab60"}, + {file = "orjson-3.10.12-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f72e27a62041cfb37a3de512247ece9f240a561e6c8662276beaf4d53d406db4"}, + {file = "orjson-3.10.12-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a904f9572092bb6742ab7c16c623f0cdccbad9eeb2d14d4aa06284867bddd31"}, + {file = "orjson-3.10.12-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:855c0833999ed5dc62f64552db26f9be767434917d8348d77bacaab84f787d7b"}, + {file = "orjson-3.10.12-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:897830244e2320f6184699f598df7fb9db9f5087d6f3f03666ae89d607e4f8ed"}, + {file = "orjson-3.10.12-cp38-cp38-musllinux_1_2_armv7l.whl", hash = "sha256:0b32652eaa4a7539f6f04abc6243619c56f8530c53bf9b023e1269df5f7816dd"}, + {file = "orjson-3.10.12-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:36b4aa31e0f6a1aeeb6f8377769ca5d125db000f05c20e54163aef1d3fe8e833"}, + {file = "orjson-3.10.12-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:5535163054d6cbf2796f93e4f0dbc800f61914c0e3c4ed8499cf6ece22b4a3da"}, + {file = "orjson-3.10.12-cp38-none-win32.whl", hash = "sha256:90a5551f6f5a5fa07010bf3d0b4ca2de21adafbbc0af6cb700b63cd767266cb9"}, + {file = "orjson-3.10.12-cp38-none-win_amd64.whl", hash = "sha256:703a2fb35a06cdd45adf5d733cf613cbc0cb3ae57643472b16bc22d325b5fb6c"}, + {file = "orjson-3.10.12-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:f29de3ef71a42a5822765def1febfb36e0859d33abf5c2ad240acad5c6a1b78d"}, + {file = "orjson-3.10.12-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:de365a42acc65d74953f05e4772c974dad6c51cfc13c3240899f534d611be967"}, + {file = "orjson-3.10.12-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:91a5a0158648a67ff0004cb0df5df7dcc55bfc9ca154d9c01597a23ad54c8d0c"}, + {file = "orjson-3.10.12-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c47ce6b8d90fe9646a25b6fb52284a14ff215c9595914af63a5933a49972ce36"}, + {file = "orjson-3.10.12-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0eee4c2c5bfb5c1b47a5db80d2ac7aaa7e938956ae88089f098aff2c0f35d5d8"}, + {file = "orjson-3.10.12-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35d3081bbe8b86587eb5c98a73b97f13d8f9fea685cf91a579beddacc0d10566"}, + {file = "orjson-3.10.12-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:73c23a6e90383884068bc2dba83d5222c9fcc3b99a0ed2411d38150734236755"}, + {file = "orjson-3.10.12-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5472be7dc3269b4b52acba1433dac239215366f89dc1d8d0e64029abac4e714e"}, + {file = "orjson-3.10.12-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:7319cda750fca96ae5973efb31b17d97a5c5225ae0bc79bf5bf84df9e1ec2ab6"}, + {file = "orjson-3.10.12-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:74d5ca5a255bf20b8def6a2b96b1e18ad37b4a122d59b154c458ee9494377f80"}, + {file = "orjson-3.10.12-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:ff31d22ecc5fb85ef62c7d4afe8301d10c558d00dd24274d4bbe464380d3cd69"}, + {file = "orjson-3.10.12-cp39-none-win32.whl", hash = "sha256:c22c3ea6fba91d84fcb4cda30e64aff548fcf0c44c876e681f47d61d24b12e6b"}, + {file = "orjson-3.10.12-cp39-none-win_amd64.whl", hash = "sha256:be604f60d45ace6b0b33dd990a66b4526f1a7a186ac411c942674625456ca548"}, + {file = "orjson-3.10.12.tar.gz", hash = "sha256:0a78bbda3aea0f9f079057ee1ee8a1ecf790d4f1af88dd67493c6b8ee52506ff"}, +] + [[package]] name = "packaging" version = "24.2" @@ -1712,13 +1796,13 @@ files = [ [[package]] name = "pydantic" -version = "2.10.1" +version = "2.10.3" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.10.1-py3-none-any.whl", hash = "sha256:a8d20db84de64cf4a7d59e899c2caf0fe9d660c7cfc482528e7020d7dd189a7e"}, - {file = "pydantic-2.10.1.tar.gz", hash = "sha256:a4daca2dc0aa429555e0656d6bf94873a7dc5f54ee42b1f5873d666fb3f35560"}, + {file = "pydantic-2.10.3-py3-none-any.whl", hash = "sha256:be04d85bbc7b65651c5f8e6b9976ed9c6f41782a55524cef079a34a0bb82144d"}, + {file = "pydantic-2.10.3.tar.gz", hash = "sha256:cb5ac360ce894ceacd69c403187900a02c4b20b693a9dd1d643e1effab9eadf9"}, ] [package.dependencies] @@ -1858,13 +1942,13 @@ windows-terminal = ["colorama (>=0.4.6)"] [[package]] name = "pyjwt" -version = "2.10.0" +version = "2.10.1" description = "JSON Web Token implementation in Python" optional = false python-versions = ">=3.9" files = [ - {file = "PyJWT-2.10.0-py3-none-any.whl", hash = "sha256:543b77207db656de204372350926bed5a86201c4cbff159f623f79c7bb487a15"}, - {file = "pyjwt-2.10.0.tar.gz", hash = "sha256:7628a7eb7938959ac1b26e819a1df0fd3259505627b575e4bad6d08f76db695c"}, + {file = "PyJWT-2.10.1-py3-none-any.whl", hash = "sha256:dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb"}, + {file = "pyjwt-2.10.1.tar.gz", hash = "sha256:3cc5772eb20009233caf06e9d8a0577824723b44e6648ee0a2aedb6cf9381953"}, ] [package.extras] @@ -1907,13 +1991,13 @@ diagrams = ["jinja2", "railroad-diagrams"] [[package]] name = "pytest" -version = "8.3.3" +version = "8.3.4" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.8" files = [ - {file = "pytest-8.3.3-py3-none-any.whl", hash = "sha256:a6853c7375b2663155079443d2e45de913a911a11d669df02a50814944db57b2"}, - {file = "pytest-8.3.3.tar.gz", hash = "sha256:70b98107bd648308a7952b06e6ca9a50bc660be218d53c257cc1fc94fda10181"}, + {file = "pytest-8.3.4-py3-none-any.whl", hash = "sha256:50e16d954148559c9a74109af1eaf0c945ba2d8f30f0a3d3335edde19788b6f6"}, + {file = "pytest-8.3.4.tar.gz", hash = "sha256:965370d062bce11e73868e0335abac31b4d3de0e82f4007408d242b4f8610761"}, ] [package.dependencies] @@ -1929,20 +2013,20 @@ dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments [[package]] name = "pytest-asyncio" -version = "0.24.0" +version = "0.25.0" description = "Pytest support for asyncio" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "pytest_asyncio-0.24.0-py3-none-any.whl", hash = "sha256:a811296ed596b69bf0b6f3dc40f83bcaf341b155a269052d82efa2b25ac7037b"}, - {file = "pytest_asyncio-0.24.0.tar.gz", hash = "sha256:d081d828e576d85f875399194281e92bf8a68d60d72d1a2faf2feddb6c46b276"}, + {file = "pytest_asyncio-0.25.0-py3-none-any.whl", hash = "sha256:db5432d18eac6b7e28b46dcd9b69921b55c3b1086e85febfe04e70b18d9e81b3"}, + {file = "pytest_asyncio-0.25.0.tar.gz", hash = "sha256:8c0610303c9e0442a5db8604505fc0f545456ba1528824842b37b4a626cbf609"}, ] [package.dependencies] pytest = ">=8.2,<9" [package.extras] -docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1)"] testing = ["coverage (>=6.2)", "hypothesis (>=5.7.1)"] [[package]] @@ -2278,13 +2362,13 @@ cffi = {version = "*", markers = "implementation_name == \"pypy\""} [[package]] name = "radixtarget" -version = "2.0.0.50" +version = "2.0.0.58" description = "Check whether an IP address belongs to a cloud provider" optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "radixtarget-2.0.0.50-py3-none-any.whl", hash = "sha256:fe1670a382d1ddaebc2cba3b16607d32085987eb5d71074cc0535e19a02406b7"}, - {file = "radixtarget-2.0.0.50.tar.gz", hash = "sha256:73519eebb0596a67d4e9347a5e4602c95c9ff9dc8be4c64e6ab0247bc69a13e8"}, + {file = "radixtarget-2.0.0.58-py3-none-any.whl", hash = "sha256:da1feb277012a115c26b370f5e2102dd31ff8745294fddc75f2d2664cc8820ad"}, + {file = "radixtarget-2.0.0.58.tar.gz", hash = "sha256:2d909608503698495b135cf1c2446c23c1d6f9dc4dfc6e6ed5517fcb5f7ee46e"}, ] [[package]] @@ -2444,29 +2528,29 @@ test = ["commentjson", "packaging", "pytest"] [[package]] name = "ruff" -version = "0.8.0" +version = "0.8.3" description = "An extremely fast Python linter and code formatter, written in Rust." optional = false python-versions = ">=3.7" files = [ - {file = "ruff-0.8.0-py3-none-linux_armv6l.whl", hash = "sha256:fcb1bf2cc6706adae9d79c8d86478677e3bbd4ced796ccad106fd4776d395fea"}, - {file = "ruff-0.8.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:295bb4c02d58ff2ef4378a1870c20af30723013f441c9d1637a008baaf928c8b"}, - {file = "ruff-0.8.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:7b1f1c76b47c18fa92ee78b60d2d20d7e866c55ee603e7d19c1e991fad933a9a"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eb0d4f250a7711b67ad513fde67e8870109e5ce590a801c3722580fe98c33a99"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e55cce9aa93c5d0d4e3937e47b169035c7e91c8655b0974e61bb79cf398d49c"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3f4cd64916d8e732ce6b87f3f5296a8942d285bbbc161acee7fe561134af64f9"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:c5c1466be2a2ebdf7c5450dd5d980cc87c8ba6976fb82582fea18823da6fa362"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2dabfd05b96b7b8f2da00d53c514eea842bff83e41e1cceb08ae1966254a51df"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:facebdfe5a5af6b1588a1d26d170635ead6892d0e314477e80256ef4a8470cf3"}, - {file = "ruff-0.8.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87a8e86bae0dbd749c815211ca11e3a7bd559b9710746c559ed63106d382bd9c"}, - {file = "ruff-0.8.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:85e654f0ded7befe2d61eeaf3d3b1e4ef3894469cd664ffa85006c7720f1e4a2"}, - {file = "ruff-0.8.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:83a55679c4cb449fa527b8497cadf54f076603cc36779b2170b24f704171ce70"}, - {file = "ruff-0.8.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:812e2052121634cf13cd6fddf0c1871d0ead1aad40a1a258753c04c18bb71bbd"}, - {file = "ruff-0.8.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:780d5d8523c04202184405e60c98d7595bdb498c3c6abba3b6d4cdf2ca2af426"}, - {file = "ruff-0.8.0-py3-none-win32.whl", hash = "sha256:5fdb6efecc3eb60bba5819679466471fd7d13c53487df7248d6e27146e985468"}, - {file = "ruff-0.8.0-py3-none-win_amd64.whl", hash = "sha256:582891c57b96228d146725975fbb942e1f30a0c4ba19722e692ca3eb25cc9b4f"}, - {file = "ruff-0.8.0-py3-none-win_arm64.whl", hash = "sha256:ba93e6294e9a737cd726b74b09a6972e36bb511f9a102f1d9a7e1ce94dd206a6"}, - {file = "ruff-0.8.0.tar.gz", hash = "sha256:a7ccfe6331bf8c8dad715753e157457faf7351c2b69f62f32c165c2dbcbacd44"}, + {file = "ruff-0.8.3-py3-none-linux_armv6l.whl", hash = "sha256:8d5d273ffffff0acd3db5bf626d4b131aa5a5ada1276126231c4174543ce20d6"}, + {file = "ruff-0.8.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e4d66a21de39f15c9757d00c50c8cdd20ac84f55684ca56def7891a025d7e939"}, + {file = "ruff-0.8.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c356e770811858bd20832af696ff6c7e884701115094f427b64b25093d6d932d"}, + {file = "ruff-0.8.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c0a60a825e3e177116c84009d5ebaa90cf40dfab56e1358d1df4e29a9a14b13"}, + {file = "ruff-0.8.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:75fb782f4db39501210ac093c79c3de581d306624575eddd7e4e13747e61ba18"}, + {file = "ruff-0.8.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7f26bc76a133ecb09a38b7868737eded6941b70a6d34ef53a4027e83913b6502"}, + {file = "ruff-0.8.3-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:01b14b2f72a37390c1b13477c1c02d53184f728be2f3ffc3ace5b44e9e87b90d"}, + {file = "ruff-0.8.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:53babd6e63e31f4e96ec95ea0d962298f9f0d9cc5990a1bbb023a6baf2503a82"}, + {file = "ruff-0.8.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1ae441ce4cf925b7f363d33cd6570c51435972d697e3e58928973994e56e1452"}, + {file = "ruff-0.8.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7c65bc0cadce32255e93c57d57ecc2cca23149edd52714c0c5d6fa11ec328cd"}, + {file = "ruff-0.8.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:5be450bb18f23f0edc5a4e5585c17a56ba88920d598f04a06bd9fd76d324cb20"}, + {file = "ruff-0.8.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:8faeae3827eaa77f5721f09b9472a18c749139c891dbc17f45e72d8f2ca1f8fc"}, + {file = "ruff-0.8.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:db503486e1cf074b9808403991663e4277f5c664d3fe237ee0d994d1305bb060"}, + {file = "ruff-0.8.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:6567be9fb62fbd7a099209257fef4ad2c3153b60579818b31a23c886ed4147ea"}, + {file = "ruff-0.8.3-py3-none-win32.whl", hash = "sha256:19048f2f878f3ee4583fc6cb23fb636e48c2635e30fb2022b3a1cd293402f964"}, + {file = "ruff-0.8.3-py3-none-win_amd64.whl", hash = "sha256:f7df94f57d7418fa7c3ffb650757e0c2b96cf2501a0b192c18e4fb5571dfada9"}, + {file = "ruff-0.8.3-py3-none-win_arm64.whl", hash = "sha256:fe2756edf68ea79707c8d68b78ca9a58ed9af22e430430491ee03e718b5e4936"}, + {file = "ruff-0.8.3.tar.gz", hash = "sha256:5e7558304353b84279042fc584a4f4cb8a07ae79b2bf3da1a7551d960b5626d3"}, ] [[package]] @@ -2766,13 +2850,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "uvicorn" -version = "0.32.1" +version = "0.34.0" description = "The lightning-fast ASGI server." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "uvicorn-0.32.1-py3-none-any.whl", hash = "sha256:82ad92fd58da0d12af7482ecdb5f2470a04c9c9a53ced65b9bbb4a205377602e"}, - {file = "uvicorn-0.32.1.tar.gz", hash = "sha256:ee9519c246a72b1c084cea8d3b44ed6026e78a4a309cbedae9c37e4cb9fbb175"}, + {file = "uvicorn-0.34.0-py3-none-any.whl", hash = "sha256:023dc038422502fa28a09c7a30bf2b6991512da7dcdb8fd35fe57cfc154126f4"}, + {file = "uvicorn-0.34.0.tar.gz", hash = "sha256:404051050cd7e905de2c9a7e61790943440b3416f49cb409f965d9dcd0fa73e9"}, ] [package.dependencies] @@ -3103,4 +3187,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "23405abad735433843bc67ad91a5d8cd41fca56a0f2a73d908ba9819d1ff95be" +content-hash = "e47772530c41a0fad08b26e205811fd6c14628ab167f6e852899927c7c812e00" diff --git a/pyproject.toml b/pyproject.toml index e0f9222186..873274158a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ pyahocorasick = "^2.1.0" puremagic = "^1.28" cloudcheck = "^6.0.0.602" radixtarget = "^2.0.0.50" +orjson = "^3.10.12" [tool.poetry.group.dev.dependencies] poetry-dynamic-versioning = ">=0.21.4,<1.5.0" @@ -69,8 +70,8 @@ pytest-rerunfailures = ">=14,<16" pytest-timeout = "^2.3.1" pytest-httpserver = "^1.0.11" pytest = "^8.3.1" -pytest-asyncio = "0.24.0" -uvicorn = "^0.32.0" +pytest-asyncio = "0.25.0" +uvicorn = ">=0.32,<0.35" fastapi = "^0.115.5" pytest-httpx = ">=0.33,<0.35" ruff = "^0.8.0" @@ -103,7 +104,8 @@ skip = "./docs/javascripts/vega*.js,./bbot/wordlists/*" [tool.ruff] line-length = 119 format.exclude = ["bbot/test/test_step_1/test_manager_*"] -lint.ignore = ["E402", "E721", "E741", "F401", "F403", "F405", "E713"] +lint.ignore = ["E402", "E721", "E741", "F401", "F403", "F405", "E713", "E711"] + [tool.poetry-dynamic-versioning] enable = true